From 6c70022dcc33ae6ce88642f95a8f891653dd183c Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Fri, 22 Oct 2021 15:53:19 -0400 Subject: [PATCH 01/53] check for intersecting ip ranges correctly, add unit test --- llarp/net/ip_range.hpp | 7 +++++++ llarp/net/net.cpp | 4 ++-- test/net/test_llarp_net.cpp | 11 +++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/llarp/net/ip_range.hpp b/llarp/net/ip_range.hpp index 5bc1bf449..7a3f9e6df 100644 --- a/llarp/net/ip_range.hpp +++ b/llarp/net/ip_range.hpp @@ -63,6 +63,13 @@ namespace llarp return bits::count_bits(netmask_bits); } + /// return true if our range and other intersect + constexpr bool + operator*(const IPRange& other) const + { + return Contains(other) or other.Contains(*this); + } + /// return true if the other range is inside our range constexpr bool Contains(const IPRange& other) const diff --git a/llarp/net/net.cpp b/llarp/net/net.cpp index 05479d8dd..fe2b883c1 100644 --- a/llarp/net/net.cpp +++ b/llarp/net/net.cpp @@ -503,10 +503,10 @@ namespace llarp IPRange{net::ExpandV4(xntohl(ifaddr)), net::ExpandV4(xntohl(ifmask))}); } }); - auto ownsRange = [¤tRanges](IPRange range) -> bool { + auto ownsRange = [¤tRanges](const IPRange& range) -> bool { for (const auto& ownRange : currentRanges) { - if (ownRange.Contains(range)) + if (ownRange * range) return true; } return false; diff --git a/test/net/test_llarp_net.cpp b/test/net/test_llarp_net.cpp index 4e5b92902..c383684c1 100644 --- a/test/net/test_llarp_net.cpp +++ b/test/net/test_llarp_net.cpp @@ -60,6 +60,17 @@ TEST_CASE("Range") REQUIRE(!llarp::IPRange::FromIPv4(192, 168, 0, 1, 24) .Contains(llarp::ipaddr_ipv4_bits(10, 200, 0, 253))); } + SECTION("Intersecting networks") + { + const auto range_16 = llarp::IPRange::FromIPv4(10,9,0,1, 16); + const auto range_24a = llarp::IPRange::FromIPv4(10,9,0,1, 24); + const auto range_24b = llarp::IPRange::FromIPv4(10,9,1,1, 24); + const auto range_unrelated = llarp::IPRange::FromIPv4(1,9,1,1, 8); + REQUIRE(range_16 * range_24a); + REQUIRE(range_16 * range_24b); + REQUIRE(not(range_24a * range_24b)); + REQUIRE(not(range_16 * range_unrelated)); + } } TEST_CASE("IPv4 netmask") From 78e829a03fc2728e727839d6b9a8edb82b134c6c Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 22 Oct 2021 17:22:42 -0300 Subject: [PATCH 02/53] Add python build deps to ci images --- contrib/ci/docker/10-debian-bullseye.dockerfile | 5 +++++ contrib/ci/docker/10-debian-buster.dockerfile | 5 +++++ contrib/ci/docker/10-debian-sid.dockerfile | 5 +++++ contrib/ci/docker/10-debian-stable.dockerfile | 5 +++++ contrib/ci/docker/10-ubuntu-bionic.dockerfile | 5 +++++ contrib/ci/docker/10-ubuntu-focal.dockerfile | 5 +++++ contrib/ci/docker/10-ubuntu-impish.dockerfile | 5 +++++ contrib/ci/docker/10-ubuntu-lts.dockerfile | 5 +++++ contrib/ci/docker/10-ubuntu-rolling.dockerfile | 5 +++++ 9 files changed, 45 insertions(+) diff --git a/contrib/ci/docker/10-debian-bullseye.dockerfile b/contrib/ci/docker/10-debian-bullseye.dockerfile index d92ea1b97..bcb7d8678 100644 --- a/contrib/ci/docker/10-debian-bullseye.dockerfile +++ b/contrib/ci/docker/10-debian-bullseye.dockerfile @@ -35,5 +35,10 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ openssh-client \ patch \ pkg-config \ + pybind11-dev \ python3-dev \ + python3-pip \ + python3-pybind11 \ + python3-pytest \ + python3-setuptools \ qttools5-dev diff --git a/contrib/ci/docker/10-debian-buster.dockerfile b/contrib/ci/docker/10-debian-buster.dockerfile index f77b9deda..75791af19 100644 --- a/contrib/ci/docker/10-debian-buster.dockerfile +++ b/contrib/ci/docker/10-debian-buster.dockerfile @@ -35,6 +35,11 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ openssh-client \ patch \ pkg-config \ + pybind11-dev \ python3-dev \ + python3-pip \ + python3-pybind11 \ + python3-pytest \ + python3-setuptools \ qttools5-dev \ xz-utils diff --git a/contrib/ci/docker/10-debian-sid.dockerfile b/contrib/ci/docker/10-debian-sid.dockerfile index 7bb2f8c3c..c7a97d881 100644 --- a/contrib/ci/docker/10-debian-sid.dockerfile +++ b/contrib/ci/docker/10-debian-sid.dockerfile @@ -35,5 +35,10 @@ RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ openssh-client \ patch \ pkg-config \ + pybind11-dev \ python3-dev \ + python3-pip \ + python3-pybind11 \ + python3-pytest \ + python3-setuptools \ qttools5-dev diff --git a/contrib/ci/docker/10-debian-stable.dockerfile b/contrib/ci/docker/10-debian-stable.dockerfile index cd50a4651..b00351dbe 100644 --- a/contrib/ci/docker/10-debian-stable.dockerfile +++ b/contrib/ci/docker/10-debian-stable.dockerfile @@ -35,5 +35,10 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ openssh-client \ patch \ pkg-config \ + pybind11-dev \ python3-dev \ + python3-pip \ + python3-pybind11 \ + python3-pytest \ + python3-setuptools \ qttools5-dev diff --git a/contrib/ci/docker/10-ubuntu-bionic.dockerfile b/contrib/ci/docker/10-ubuntu-bionic.dockerfile index 767d2f3d1..936eac28b 100644 --- a/contrib/ci/docker/10-ubuntu-bionic.dockerfile +++ b/contrib/ci/docker/10-ubuntu-bionic.dockerfile @@ -26,6 +26,11 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ openssh-client \ patch \ pkg-config \ + pybind11-dev \ python3-dev \ + python3-pip \ + python3-pybind11 \ + python3-pytest \ + python3-setuptools \ qttools5-dev \ && mkdir -p /usr/lib/x86_64-linux-gnu/pgm-5.2/include diff --git a/contrib/ci/docker/10-ubuntu-focal.dockerfile b/contrib/ci/docker/10-ubuntu-focal.dockerfile index a9d38599e..7b0ab0f91 100644 --- a/contrib/ci/docker/10-ubuntu-focal.dockerfile +++ b/contrib/ci/docker/10-ubuntu-focal.dockerfile @@ -38,5 +38,10 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ openssh-client \ patch \ pkg-config \ + pybind11-dev \ python3-dev \ + python3-pip \ + python3-pybind11 \ + python3-pytest \ + python3-setuptools \ qttools5-dev diff --git a/contrib/ci/docker/10-ubuntu-impish.dockerfile b/contrib/ci/docker/10-ubuntu-impish.dockerfile index 0e7c354e0..598e52f3e 100644 --- a/contrib/ci/docker/10-ubuntu-impish.dockerfile +++ b/contrib/ci/docker/10-ubuntu-impish.dockerfile @@ -38,5 +38,10 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ openssh-client \ patch \ pkg-config \ + pybind11-dev \ python3-dev \ + python3-pip \ + python3-pybind11 \ + python3-pytest \ + python3-setuptools \ qttools5-dev diff --git a/contrib/ci/docker/10-ubuntu-lts.dockerfile b/contrib/ci/docker/10-ubuntu-lts.dockerfile index 3b2dfc413..1a0aab36f 100644 --- a/contrib/ci/docker/10-ubuntu-lts.dockerfile +++ b/contrib/ci/docker/10-ubuntu-lts.dockerfile @@ -38,5 +38,10 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ openssh-client \ patch \ pkg-config \ + pybind11-dev \ python3-dev \ + python3-pip \ + python3-pybind11 \ + python3-pytest \ + python3-setuptools \ qttools5-dev diff --git a/contrib/ci/docker/10-ubuntu-rolling.dockerfile b/contrib/ci/docker/10-ubuntu-rolling.dockerfile index 989dc0a5c..04777b6b7 100644 --- a/contrib/ci/docker/10-ubuntu-rolling.dockerfile +++ b/contrib/ci/docker/10-ubuntu-rolling.dockerfile @@ -38,5 +38,10 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ openssh-client \ patch \ pkg-config \ + pybind11-dev \ python3-dev \ + python3-pip \ + python3-pybind11 \ + python3-pytest \ + python3-setuptools \ qttools5-dev From c7410660c8e979f4d5acfc48900d4df4137ae04f Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 22 Oct 2021 17:30:51 -0300 Subject: [PATCH 03/53] Make sure we apt-get update in the build The base image can get out of date. --- contrib/ci/docker/10-debian-bullseye.dockerfile | 4 +++- contrib/ci/docker/10-debian-buster.dockerfile | 4 +++- contrib/ci/docker/10-debian-sid.dockerfile | 4 +++- contrib/ci/docker/10-debian-stable.dockerfile | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/contrib/ci/docker/10-debian-bullseye.dockerfile b/contrib/ci/docker/10-debian-bullseye.dockerfile index bcb7d8678..ed0106180 100644 --- a/contrib/ci/docker/10-debian-bullseye.dockerfile +++ b/contrib/ci/docker/10-debian-bullseye.dockerfile @@ -1,6 +1,8 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-bullseye-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ automake \ ccache \ cmake \ diff --git a/contrib/ci/docker/10-debian-buster.dockerfile b/contrib/ci/docker/10-debian-buster.dockerfile index 75791af19..d653da838 100644 --- a/contrib/ci/docker/10-debian-buster.dockerfile +++ b/contrib/ci/docker/10-debian-buster.dockerfile @@ -1,6 +1,8 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-buster-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ automake \ ccache \ cmake \ diff --git a/contrib/ci/docker/10-debian-sid.dockerfile b/contrib/ci/docker/10-debian-sid.dockerfile index c7a97d881..4c662a20c 100644 --- a/contrib/ci/docker/10-debian-sid.dockerfile +++ b/contrib/ci/docker/10-debian-sid.dockerfile @@ -1,6 +1,8 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-sid-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ automake \ ccache \ cmake \ diff --git a/contrib/ci/docker/10-debian-stable.dockerfile b/contrib/ci/docker/10-debian-stable.dockerfile index b00351dbe..adc4c6db7 100644 --- a/contrib/ci/docker/10-debian-stable.dockerfile +++ b/contrib/ci/docker/10-debian-stable.dockerfile @@ -1,6 +1,8 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-stable-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ automake \ ccache \ cmake \ From f3bac4ae7219064db7a50725a69675c50f16d49b Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 22 Oct 2021 17:34:45 -0300 Subject: [PATCH 04/53] Pass DOCKER_BUILD_OPTS env var to docker build --- contrib/ci/docker/rebuild-docker-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/ci/docker/rebuild-docker-images.sh b/contrib/ci/docker/rebuild-docker-images.sh index a0773a0b1..bded2f2f4 100755 --- a/contrib/ci/docker/rebuild-docker-images.sh +++ b/contrib/ci/docker/rebuild-docker-images.sh @@ -28,7 +28,7 @@ for file in "${files[@]}"; do namearch=$registry/lokinet-ci-$name/$arch latest=$registry/lokinet-ci-$name:latest echo -e "\e[32;1mrebuilding \e[35;1m$namearch\e[0m" - docker build --pull -f $file -t $namearch --build-arg ARCH=$arch . + docker build --pull -f $file -t $namearch --build-arg ARCH=$arch $DOCKER_BUILD_OPTS . docker push $namearch manifests[$latest]="${manifests[$latest]} $namearch" From b5f5d07f03d153b2d43e9519329e371c9eadd150 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Mon, 25 Oct 2021 11:35:49 -0400 Subject: [PATCH 05/53] bump libunbound static dep hashpin --- cmake/StaticBuild.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/StaticBuild.cmake b/cmake/StaticBuild.cmake index 6b000a8ef..87ba00a81 100644 --- a/cmake/StaticBuild.cmake +++ b/cmake/StaticBuild.cmake @@ -19,10 +19,10 @@ set(EXPAT_SOURCE expat-${EXPAT_VERSION}.tar.xz) set(EXPAT_HASH SHA512=dde8a9a094b18d795a0e86ca4aa68488b352dc67019e0d669e8b910ed149628de4c2a49bc3a5b832f624319336a01f9e4debe03433a43e1c420f36356d886820 CACHE STRING "expat source hash") -set(UNBOUND_VERSION 1.13.1 CACHE STRING "unbound version") +set(UNBOUND_VERSION 1.13.2 CACHE STRING "unbound version") set(UNBOUND_MIRROR ${LOCAL_MIRROR} https://nlnetlabs.nl/downloads/unbound CACHE STRING "unbound download mirror(s)") set(UNBOUND_SOURCE unbound-${UNBOUND_VERSION}.tar.gz) -set(UNBOUND_HASH SHA256=8504d97b8fc5bd897345c95d116e0ee0ddf8c8ff99590ab2b4bd13278c9f50b8 +set(UNBOUND_HASH SHA256=0a13b547f3b92a026b5ebd0423f54c991e5718037fd9f72445817f6a040e1a83 CACHE STRING "unbound source hash") set(SQLITE3_VERSION 3350500 CACHE STRING "sqlite3 version") From cbbee5977477431496016f53a08dd640d3619888 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 25 Oct 2021 16:17:20 -0300 Subject: [PATCH 06/53] Add a bunch of base CI images for building debs Installs just the base tools (which are quite substantial), but not the deps (which get installed as part of the build). --- contrib/ci/docker/00-ubuntu-bionic-base.dockerfile | 6 ++++++ contrib/ci/docker/00-ubuntu-focal-base.dockerfile | 6 ++++++ contrib/ci/docker/00-ubuntu-hirsute-base.dockerfile | 6 ++++++ contrib/ci/docker/00-ubuntu-impish-base.dockerfile | 6 ++++++ contrib/ci/docker/00-ubuntu-lts-base.dockerfile | 6 ++++++ contrib/ci/docker/00-ubuntu-rolling-base.dockerfile | 6 ++++++ contrib/ci/docker/10-ubuntu-bionic.dockerfile | 2 +- contrib/ci/docker/10-ubuntu-focal.dockerfile | 2 +- contrib/ci/docker/10-ubuntu-impish.dockerfile | 2 +- contrib/ci/docker/10-ubuntu-lts.dockerfile | 2 +- contrib/ci/docker/10-ubuntu-rolling.dockerfile | 2 +- contrib/ci/docker/20-deb-builder-bionic.dockerfile | 10 ++++++++++ contrib/ci/docker/20-deb-builder-bullseye.dockerfile | 10 ++++++++++ contrib/ci/docker/20-deb-builder-buster.dockerfile | 10 ++++++++++ contrib/ci/docker/20-deb-builder-focal.dockerfile | 10 ++++++++++ contrib/ci/docker/20-deb-builder-hirsute.dockerfile | 10 ++++++++++ contrib/ci/docker/20-deb-builder-impish.dockerfile | 10 ++++++++++ contrib/ci/docker/20-deb-builder-sid.dockerfile | 10 ++++++++++ .../docker/arm32v7/00-debian-bullseye-base.dockerfile | 1 + .../ci/docker/arm32v7/00-debian-sid-base.dockerfile | 1 + .../ci/docker/arm32v7/00-ubuntu-bionic-base.dockerfile | 1 + .../ci/docker/arm32v7/00-ubuntu-focal-base.dockerfile | 1 + .../docker/arm32v7/00-ubuntu-hirsute-base.dockerfile | 1 + .../ci/docker/arm32v7/00-ubuntu-impish-base.dockerfile | 1 + .../ci/docker/arm32v7/20-deb-builder-bionic.dockerfile | 1 + .../docker/arm32v7/20-deb-builder-bullseye.dockerfile | 1 + .../ci/docker/arm32v7/20-deb-builder-buster.dockerfile | 1 + .../ci/docker/arm32v7/20-deb-builder-focal.dockerfile | 1 + .../docker/arm32v7/20-deb-builder-hirsute.dockerfile | 1 + .../ci/docker/arm32v7/20-deb-builder-impish.dockerfile | 1 + .../ci/docker/arm32v7/20-deb-builder-sid.dockerfile | 1 + .../docker/arm64v8/00-debian-bullseye-base.dockerfile | 1 + .../ci/docker/arm64v8/00-debian-buster-base.dockerfile | 1 + .../ci/docker/arm64v8/00-ubuntu-bionic-base.dockerfile | 1 + .../ci/docker/arm64v8/00-ubuntu-focal-base.dockerfile | 1 + .../docker/arm64v8/00-ubuntu-hirsute-base.dockerfile | 1 + .../ci/docker/arm64v8/00-ubuntu-impish-base.dockerfile | 1 + .../ci/docker/arm64v8/20-deb-builder-bionic.dockerfile | 1 + .../docker/arm64v8/20-deb-builder-bullseye.dockerfile | 1 + .../ci/docker/arm64v8/20-deb-builder-buster.dockerfile | 1 + .../ci/docker/arm64v8/20-deb-builder-focal.dockerfile | 1 + .../docker/arm64v8/20-deb-builder-hirsute.dockerfile | 1 + .../ci/docker/arm64v8/20-deb-builder-impish.dockerfile | 1 + .../ci/docker/arm64v8/20-deb-builder-sid.dockerfile | 1 + .../ci/docker/i386/00-debian-bullseye-base.dockerfile | 1 + .../ci/docker/i386/00-debian-buster-base.dockerfile | 1 + contrib/ci/docker/i386/00-debian-sid-base.dockerfile | 1 + .../ci/docker/i386/00-ubuntu-bionic-base.dockerfile | 1 + .../ci/docker/i386/20-deb-builder-bionic.dockerfile | 1 + .../ci/docker/i386/20-deb-builder-bullseye.dockerfile | 1 + .../ci/docker/i386/20-deb-builder-buster.dockerfile | 1 + contrib/ci/docker/i386/20-deb-builder-sid.dockerfile | 1 + 52 files changed, 145 insertions(+), 5 deletions(-) create mode 100644 contrib/ci/docker/00-ubuntu-bionic-base.dockerfile create mode 100644 contrib/ci/docker/00-ubuntu-focal-base.dockerfile create mode 100644 contrib/ci/docker/00-ubuntu-hirsute-base.dockerfile create mode 100644 contrib/ci/docker/00-ubuntu-impish-base.dockerfile create mode 100644 contrib/ci/docker/00-ubuntu-lts-base.dockerfile create mode 100644 contrib/ci/docker/00-ubuntu-rolling-base.dockerfile create mode 100644 contrib/ci/docker/20-deb-builder-bionic.dockerfile create mode 100644 contrib/ci/docker/20-deb-builder-bullseye.dockerfile create mode 100644 contrib/ci/docker/20-deb-builder-buster.dockerfile create mode 100644 contrib/ci/docker/20-deb-builder-focal.dockerfile create mode 100644 contrib/ci/docker/20-deb-builder-hirsute.dockerfile create mode 100644 contrib/ci/docker/20-deb-builder-impish.dockerfile create mode 100644 contrib/ci/docker/20-deb-builder-sid.dockerfile create mode 120000 contrib/ci/docker/arm32v7/00-debian-bullseye-base.dockerfile create mode 120000 contrib/ci/docker/arm32v7/00-debian-sid-base.dockerfile create mode 120000 contrib/ci/docker/arm32v7/00-ubuntu-bionic-base.dockerfile create mode 120000 contrib/ci/docker/arm32v7/00-ubuntu-focal-base.dockerfile create mode 120000 contrib/ci/docker/arm32v7/00-ubuntu-hirsute-base.dockerfile create mode 120000 contrib/ci/docker/arm32v7/00-ubuntu-impish-base.dockerfile create mode 120000 contrib/ci/docker/arm32v7/20-deb-builder-bionic.dockerfile create mode 120000 contrib/ci/docker/arm32v7/20-deb-builder-bullseye.dockerfile create mode 120000 contrib/ci/docker/arm32v7/20-deb-builder-buster.dockerfile create mode 120000 contrib/ci/docker/arm32v7/20-deb-builder-focal.dockerfile create mode 120000 contrib/ci/docker/arm32v7/20-deb-builder-hirsute.dockerfile create mode 120000 contrib/ci/docker/arm32v7/20-deb-builder-impish.dockerfile create mode 120000 contrib/ci/docker/arm32v7/20-deb-builder-sid.dockerfile create mode 120000 contrib/ci/docker/arm64v8/00-debian-bullseye-base.dockerfile create mode 120000 contrib/ci/docker/arm64v8/00-debian-buster-base.dockerfile create mode 120000 contrib/ci/docker/arm64v8/00-ubuntu-bionic-base.dockerfile create mode 120000 contrib/ci/docker/arm64v8/00-ubuntu-focal-base.dockerfile create mode 120000 contrib/ci/docker/arm64v8/00-ubuntu-hirsute-base.dockerfile create mode 120000 contrib/ci/docker/arm64v8/00-ubuntu-impish-base.dockerfile create mode 120000 contrib/ci/docker/arm64v8/20-deb-builder-bionic.dockerfile create mode 120000 contrib/ci/docker/arm64v8/20-deb-builder-bullseye.dockerfile create mode 120000 contrib/ci/docker/arm64v8/20-deb-builder-buster.dockerfile create mode 120000 contrib/ci/docker/arm64v8/20-deb-builder-focal.dockerfile create mode 120000 contrib/ci/docker/arm64v8/20-deb-builder-hirsute.dockerfile create mode 120000 contrib/ci/docker/arm64v8/20-deb-builder-impish.dockerfile create mode 120000 contrib/ci/docker/arm64v8/20-deb-builder-sid.dockerfile create mode 120000 contrib/ci/docker/i386/00-debian-bullseye-base.dockerfile create mode 120000 contrib/ci/docker/i386/00-debian-buster-base.dockerfile create mode 120000 contrib/ci/docker/i386/00-debian-sid-base.dockerfile create mode 120000 contrib/ci/docker/i386/00-ubuntu-bionic-base.dockerfile create mode 120000 contrib/ci/docker/i386/20-deb-builder-bionic.dockerfile create mode 120000 contrib/ci/docker/i386/20-deb-builder-bullseye.dockerfile create mode 120000 contrib/ci/docker/i386/20-deb-builder-buster.dockerfile create mode 120000 contrib/ci/docker/i386/20-deb-builder-sid.dockerfile diff --git a/contrib/ci/docker/00-ubuntu-bionic-base.dockerfile b/contrib/ci/docker/00-ubuntu-bionic-base.dockerfile new file mode 100644 index 000000000..99ccc1bab --- /dev/null +++ b/contrib/ci/docker/00-ubuntu-bionic-base.dockerfile @@ -0,0 +1,6 @@ +ARG ARCH=amd64 +FROM ${ARCH}/ubuntu:bionic +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-ubuntu-focal-base.dockerfile b/contrib/ci/docker/00-ubuntu-focal-base.dockerfile new file mode 100644 index 000000000..f3d190bff --- /dev/null +++ b/contrib/ci/docker/00-ubuntu-focal-base.dockerfile @@ -0,0 +1,6 @@ +ARG ARCH=amd64 +FROM ${ARCH}/ubuntu:focal +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-ubuntu-hirsute-base.dockerfile b/contrib/ci/docker/00-ubuntu-hirsute-base.dockerfile new file mode 100644 index 000000000..b17dbf29e --- /dev/null +++ b/contrib/ci/docker/00-ubuntu-hirsute-base.dockerfile @@ -0,0 +1,6 @@ +ARG ARCH=amd64 +FROM ${ARCH}/ubuntu:hirsute +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-ubuntu-impish-base.dockerfile b/contrib/ci/docker/00-ubuntu-impish-base.dockerfile new file mode 100644 index 000000000..02ce35ef6 --- /dev/null +++ b/contrib/ci/docker/00-ubuntu-impish-base.dockerfile @@ -0,0 +1,6 @@ +ARG ARCH=amd64 +FROM ${ARCH}/ubuntu:impish +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-ubuntu-lts-base.dockerfile b/contrib/ci/docker/00-ubuntu-lts-base.dockerfile new file mode 100644 index 000000000..f3d190bff --- /dev/null +++ b/contrib/ci/docker/00-ubuntu-lts-base.dockerfile @@ -0,0 +1,6 @@ +ARG ARCH=amd64 +FROM ${ARCH}/ubuntu:focal +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-ubuntu-rolling-base.dockerfile b/contrib/ci/docker/00-ubuntu-rolling-base.dockerfile new file mode 100644 index 000000000..1d813f727 --- /dev/null +++ b/contrib/ci/docker/00-ubuntu-rolling-base.dockerfile @@ -0,0 +1,6 @@ +ARG ARCH=amd64 +FROM ${ARCH}/ubuntu:rolling +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/10-ubuntu-bionic.dockerfile b/contrib/ci/docker/10-ubuntu-bionic.dockerfile index 936eac28b..df395fce5 100644 --- a/contrib/ci/docker/10-ubuntu-bionic.dockerfile +++ b/contrib/ci/docker/10-ubuntu-bionic.dockerfile @@ -1,5 +1,5 @@ ARG ARCH=amd64 -FROM ${ARCH}/ubuntu:bionic +FROM registry.oxen.rocks/lokinet-ci-ubuntu-bionic-base/${ARCH} RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ diff --git a/contrib/ci/docker/10-ubuntu-focal.dockerfile b/contrib/ci/docker/10-ubuntu-focal.dockerfile index 7b0ab0f91..62310d8e9 100644 --- a/contrib/ci/docker/10-ubuntu-focal.dockerfile +++ b/contrib/ci/docker/10-ubuntu-focal.dockerfile @@ -1,5 +1,5 @@ ARG ARCH=amd64 -FROM ${ARCH}/ubuntu:focal +FROM registry.oxen.rocks/lokinet-ci-ubuntu-focal-base/${ARCH} RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ diff --git a/contrib/ci/docker/10-ubuntu-impish.dockerfile b/contrib/ci/docker/10-ubuntu-impish.dockerfile index 598e52f3e..d7e81cab0 100644 --- a/contrib/ci/docker/10-ubuntu-impish.dockerfile +++ b/contrib/ci/docker/10-ubuntu-impish.dockerfile @@ -1,5 +1,5 @@ ARG ARCH=amd64 -FROM ${ARCH}/ubuntu:impish +FROM registry.oxen.rocks/lokinet-ci-ubuntu-impish-base/${ARCH} RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ diff --git a/contrib/ci/docker/10-ubuntu-lts.dockerfile b/contrib/ci/docker/10-ubuntu-lts.dockerfile index 1a0aab36f..a2f2e7d79 100644 --- a/contrib/ci/docker/10-ubuntu-lts.dockerfile +++ b/contrib/ci/docker/10-ubuntu-lts.dockerfile @@ -1,5 +1,5 @@ ARG ARCH=amd64 -FROM ${ARCH}/ubuntu:latest +FROM registry.oxen.rocks/lokinet-ci-ubuntu-lts-base/${ARCH} RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ diff --git a/contrib/ci/docker/10-ubuntu-rolling.dockerfile b/contrib/ci/docker/10-ubuntu-rolling.dockerfile index 04777b6b7..0bcda4935 100644 --- a/contrib/ci/docker/10-ubuntu-rolling.dockerfile +++ b/contrib/ci/docker/10-ubuntu-rolling.dockerfile @@ -1,5 +1,5 @@ ARG ARCH=amd64 -FROM ${ARCH}/ubuntu:rolling +FROM registry.oxen.rocks/lokinet-ci-ubuntu-rolling-base/${ARCH} RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ diff --git a/contrib/ci/docker/20-deb-builder-bionic.dockerfile b/contrib/ci/docker/20-deb-builder-bionic.dockerfile new file mode 100644 index 000000000..8d1f92aef --- /dev/null +++ b/contrib/ci/docker/20-deb-builder-bionic.dockerfile @@ -0,0 +1,10 @@ +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-ubuntu-bionic-base/${ARCH} +RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + cmake \ + devscripts \ + equivs \ + g++ \ + git-buildpackage \ + openssh-client diff --git a/contrib/ci/docker/20-deb-builder-bullseye.dockerfile b/contrib/ci/docker/20-deb-builder-bullseye.dockerfile new file mode 100644 index 000000000..ac4da81b5 --- /dev/null +++ b/contrib/ci/docker/20-deb-builder-bullseye.dockerfile @@ -0,0 +1,10 @@ +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-debian-bullseye-base/${ARCH} +RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + cmake \ + devscripts \ + equivs \ + g++ \ + git-buildpackage \ + openssh-client diff --git a/contrib/ci/docker/20-deb-builder-buster.dockerfile b/contrib/ci/docker/20-deb-builder-buster.dockerfile new file mode 100644 index 000000000..5573fed90 --- /dev/null +++ b/contrib/ci/docker/20-deb-builder-buster.dockerfile @@ -0,0 +1,10 @@ +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-debian-buster-base/${ARCH} +RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + cmake \ + devscripts \ + equivs \ + g++ \ + git-buildpackage \ + openssh-client diff --git a/contrib/ci/docker/20-deb-builder-focal.dockerfile b/contrib/ci/docker/20-deb-builder-focal.dockerfile new file mode 100644 index 000000000..0831ac026 --- /dev/null +++ b/contrib/ci/docker/20-deb-builder-focal.dockerfile @@ -0,0 +1,10 @@ +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-ubuntu-focal-base/${ARCH} +RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + cmake \ + devscripts \ + equivs \ + g++ \ + git-buildpackage \ + openssh-client diff --git a/contrib/ci/docker/20-deb-builder-hirsute.dockerfile b/contrib/ci/docker/20-deb-builder-hirsute.dockerfile new file mode 100644 index 000000000..c7e1a6576 --- /dev/null +++ b/contrib/ci/docker/20-deb-builder-hirsute.dockerfile @@ -0,0 +1,10 @@ +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-ubuntu-hirsute-base/${ARCH} +RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + cmake \ + devscripts \ + equivs \ + g++ \ + git-buildpackage \ + openssh-client diff --git a/contrib/ci/docker/20-deb-builder-impish.dockerfile b/contrib/ci/docker/20-deb-builder-impish.dockerfile new file mode 100644 index 000000000..b8e002244 --- /dev/null +++ b/contrib/ci/docker/20-deb-builder-impish.dockerfile @@ -0,0 +1,10 @@ +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-ubuntu-impish-base/${ARCH} +RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + cmake \ + devscripts \ + equivs \ + g++ \ + git-buildpackage \ + openssh-client diff --git a/contrib/ci/docker/20-deb-builder-sid.dockerfile b/contrib/ci/docker/20-deb-builder-sid.dockerfile new file mode 100644 index 000000000..d90818747 --- /dev/null +++ b/contrib/ci/docker/20-deb-builder-sid.dockerfile @@ -0,0 +1,10 @@ +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-debian-sid-base/${ARCH} +RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + cmake \ + devscripts \ + equivs \ + g++ \ + git-buildpackage \ + openssh-client diff --git a/contrib/ci/docker/arm32v7/00-debian-bullseye-base.dockerfile b/contrib/ci/docker/arm32v7/00-debian-bullseye-base.dockerfile new file mode 120000 index 000000000..cfa13d1e1 --- /dev/null +++ b/contrib/ci/docker/arm32v7/00-debian-bullseye-base.dockerfile @@ -0,0 +1 @@ +../00-debian-bullseye-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/00-debian-sid-base.dockerfile b/contrib/ci/docker/arm32v7/00-debian-sid-base.dockerfile new file mode 120000 index 000000000..3fdec500d --- /dev/null +++ b/contrib/ci/docker/arm32v7/00-debian-sid-base.dockerfile @@ -0,0 +1 @@ +../00-debian-sid-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/00-ubuntu-bionic-base.dockerfile b/contrib/ci/docker/arm32v7/00-ubuntu-bionic-base.dockerfile new file mode 120000 index 000000000..3764c68b2 --- /dev/null +++ b/contrib/ci/docker/arm32v7/00-ubuntu-bionic-base.dockerfile @@ -0,0 +1 @@ +../00-ubuntu-bionic-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/00-ubuntu-focal-base.dockerfile b/contrib/ci/docker/arm32v7/00-ubuntu-focal-base.dockerfile new file mode 120000 index 000000000..23993a33b --- /dev/null +++ b/contrib/ci/docker/arm32v7/00-ubuntu-focal-base.dockerfile @@ -0,0 +1 @@ +../00-ubuntu-focal-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/00-ubuntu-hirsute-base.dockerfile b/contrib/ci/docker/arm32v7/00-ubuntu-hirsute-base.dockerfile new file mode 120000 index 000000000..825862d56 --- /dev/null +++ b/contrib/ci/docker/arm32v7/00-ubuntu-hirsute-base.dockerfile @@ -0,0 +1 @@ +../00-ubuntu-hirsute-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/00-ubuntu-impish-base.dockerfile b/contrib/ci/docker/arm32v7/00-ubuntu-impish-base.dockerfile new file mode 120000 index 000000000..b9dba0b96 --- /dev/null +++ b/contrib/ci/docker/arm32v7/00-ubuntu-impish-base.dockerfile @@ -0,0 +1 @@ +../00-ubuntu-impish-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/20-deb-builder-bionic.dockerfile b/contrib/ci/docker/arm32v7/20-deb-builder-bionic.dockerfile new file mode 120000 index 000000000..ffb9df8ed --- /dev/null +++ b/contrib/ci/docker/arm32v7/20-deb-builder-bionic.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-bionic.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/20-deb-builder-bullseye.dockerfile b/contrib/ci/docker/arm32v7/20-deb-builder-bullseye.dockerfile new file mode 120000 index 000000000..a7647cd32 --- /dev/null +++ b/contrib/ci/docker/arm32v7/20-deb-builder-bullseye.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-bullseye.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/20-deb-builder-buster.dockerfile b/contrib/ci/docker/arm32v7/20-deb-builder-buster.dockerfile new file mode 120000 index 000000000..301cbe821 --- /dev/null +++ b/contrib/ci/docker/arm32v7/20-deb-builder-buster.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-buster.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/20-deb-builder-focal.dockerfile b/contrib/ci/docker/arm32v7/20-deb-builder-focal.dockerfile new file mode 120000 index 000000000..1438e775c --- /dev/null +++ b/contrib/ci/docker/arm32v7/20-deb-builder-focal.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-focal.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/20-deb-builder-hirsute.dockerfile b/contrib/ci/docker/arm32v7/20-deb-builder-hirsute.dockerfile new file mode 120000 index 000000000..6750c95a0 --- /dev/null +++ b/contrib/ci/docker/arm32v7/20-deb-builder-hirsute.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-hirsute.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/20-deb-builder-impish.dockerfile b/contrib/ci/docker/arm32v7/20-deb-builder-impish.dockerfile new file mode 120000 index 000000000..b53e98286 --- /dev/null +++ b/contrib/ci/docker/arm32v7/20-deb-builder-impish.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-impish.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/20-deb-builder-sid.dockerfile b/contrib/ci/docker/arm32v7/20-deb-builder-sid.dockerfile new file mode 120000 index 000000000..f80b1be0a --- /dev/null +++ b/contrib/ci/docker/arm32v7/20-deb-builder-sid.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-sid.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/00-debian-bullseye-base.dockerfile b/contrib/ci/docker/arm64v8/00-debian-bullseye-base.dockerfile new file mode 120000 index 000000000..cfa13d1e1 --- /dev/null +++ b/contrib/ci/docker/arm64v8/00-debian-bullseye-base.dockerfile @@ -0,0 +1 @@ +../00-debian-bullseye-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/00-debian-buster-base.dockerfile b/contrib/ci/docker/arm64v8/00-debian-buster-base.dockerfile new file mode 120000 index 000000000..e4ab00108 --- /dev/null +++ b/contrib/ci/docker/arm64v8/00-debian-buster-base.dockerfile @@ -0,0 +1 @@ +../00-debian-buster-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/00-ubuntu-bionic-base.dockerfile b/contrib/ci/docker/arm64v8/00-ubuntu-bionic-base.dockerfile new file mode 120000 index 000000000..3764c68b2 --- /dev/null +++ b/contrib/ci/docker/arm64v8/00-ubuntu-bionic-base.dockerfile @@ -0,0 +1 @@ +../00-ubuntu-bionic-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/00-ubuntu-focal-base.dockerfile b/contrib/ci/docker/arm64v8/00-ubuntu-focal-base.dockerfile new file mode 120000 index 000000000..23993a33b --- /dev/null +++ b/contrib/ci/docker/arm64v8/00-ubuntu-focal-base.dockerfile @@ -0,0 +1 @@ +../00-ubuntu-focal-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/00-ubuntu-hirsute-base.dockerfile b/contrib/ci/docker/arm64v8/00-ubuntu-hirsute-base.dockerfile new file mode 120000 index 000000000..825862d56 --- /dev/null +++ b/contrib/ci/docker/arm64v8/00-ubuntu-hirsute-base.dockerfile @@ -0,0 +1 @@ +../00-ubuntu-hirsute-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/00-ubuntu-impish-base.dockerfile b/contrib/ci/docker/arm64v8/00-ubuntu-impish-base.dockerfile new file mode 120000 index 000000000..b9dba0b96 --- /dev/null +++ b/contrib/ci/docker/arm64v8/00-ubuntu-impish-base.dockerfile @@ -0,0 +1 @@ +../00-ubuntu-impish-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/20-deb-builder-bionic.dockerfile b/contrib/ci/docker/arm64v8/20-deb-builder-bionic.dockerfile new file mode 120000 index 000000000..ffb9df8ed --- /dev/null +++ b/contrib/ci/docker/arm64v8/20-deb-builder-bionic.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-bionic.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/20-deb-builder-bullseye.dockerfile b/contrib/ci/docker/arm64v8/20-deb-builder-bullseye.dockerfile new file mode 120000 index 000000000..a7647cd32 --- /dev/null +++ b/contrib/ci/docker/arm64v8/20-deb-builder-bullseye.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-bullseye.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/20-deb-builder-buster.dockerfile b/contrib/ci/docker/arm64v8/20-deb-builder-buster.dockerfile new file mode 120000 index 000000000..301cbe821 --- /dev/null +++ b/contrib/ci/docker/arm64v8/20-deb-builder-buster.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-buster.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/20-deb-builder-focal.dockerfile b/contrib/ci/docker/arm64v8/20-deb-builder-focal.dockerfile new file mode 120000 index 000000000..1438e775c --- /dev/null +++ b/contrib/ci/docker/arm64v8/20-deb-builder-focal.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-focal.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/20-deb-builder-hirsute.dockerfile b/contrib/ci/docker/arm64v8/20-deb-builder-hirsute.dockerfile new file mode 120000 index 000000000..6750c95a0 --- /dev/null +++ b/contrib/ci/docker/arm64v8/20-deb-builder-hirsute.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-hirsute.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/20-deb-builder-impish.dockerfile b/contrib/ci/docker/arm64v8/20-deb-builder-impish.dockerfile new file mode 120000 index 000000000..b53e98286 --- /dev/null +++ b/contrib/ci/docker/arm64v8/20-deb-builder-impish.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-impish.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/20-deb-builder-sid.dockerfile b/contrib/ci/docker/arm64v8/20-deb-builder-sid.dockerfile new file mode 120000 index 000000000..f80b1be0a --- /dev/null +++ b/contrib/ci/docker/arm64v8/20-deb-builder-sid.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-sid.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/00-debian-bullseye-base.dockerfile b/contrib/ci/docker/i386/00-debian-bullseye-base.dockerfile new file mode 120000 index 000000000..cfa13d1e1 --- /dev/null +++ b/contrib/ci/docker/i386/00-debian-bullseye-base.dockerfile @@ -0,0 +1 @@ +../00-debian-bullseye-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/00-debian-buster-base.dockerfile b/contrib/ci/docker/i386/00-debian-buster-base.dockerfile new file mode 120000 index 000000000..e4ab00108 --- /dev/null +++ b/contrib/ci/docker/i386/00-debian-buster-base.dockerfile @@ -0,0 +1 @@ +../00-debian-buster-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/00-debian-sid-base.dockerfile b/contrib/ci/docker/i386/00-debian-sid-base.dockerfile new file mode 120000 index 000000000..3fdec500d --- /dev/null +++ b/contrib/ci/docker/i386/00-debian-sid-base.dockerfile @@ -0,0 +1 @@ +../00-debian-sid-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/00-ubuntu-bionic-base.dockerfile b/contrib/ci/docker/i386/00-ubuntu-bionic-base.dockerfile new file mode 120000 index 000000000..3764c68b2 --- /dev/null +++ b/contrib/ci/docker/i386/00-ubuntu-bionic-base.dockerfile @@ -0,0 +1 @@ +../00-ubuntu-bionic-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/20-deb-builder-bionic.dockerfile b/contrib/ci/docker/i386/20-deb-builder-bionic.dockerfile new file mode 120000 index 000000000..ffb9df8ed --- /dev/null +++ b/contrib/ci/docker/i386/20-deb-builder-bionic.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-bionic.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/20-deb-builder-bullseye.dockerfile b/contrib/ci/docker/i386/20-deb-builder-bullseye.dockerfile new file mode 120000 index 000000000..a7647cd32 --- /dev/null +++ b/contrib/ci/docker/i386/20-deb-builder-bullseye.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-bullseye.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/20-deb-builder-buster.dockerfile b/contrib/ci/docker/i386/20-deb-builder-buster.dockerfile new file mode 120000 index 000000000..301cbe821 --- /dev/null +++ b/contrib/ci/docker/i386/20-deb-builder-buster.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-buster.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/20-deb-builder-sid.dockerfile b/contrib/ci/docker/i386/20-deb-builder-sid.dockerfile new file mode 120000 index 000000000..f80b1be0a --- /dev/null +++ b/contrib/ci/docker/i386/20-deb-builder-sid.dockerfile @@ -0,0 +1 @@ +../20-deb-builder-sid.dockerfile \ No newline at end of file From 1c4332ad6a87616169a7ba0516f70cc6e708f2b3 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 26 Oct 2021 03:24:44 -0300 Subject: [PATCH 07/53] Replace dockerfiles with a Python script --- .../docker/00-debian-bullseye-base.dockerfile | 6 - .../docker/00-debian-buster-base.dockerfile | 6 - .../ci/docker/00-debian-sid-base.dockerfile | 6 - .../docker/00-debian-stable-base.dockerfile | 6 - .../docker/00-debian-testing-base.dockerfile | 6 - .../docker/00-ubuntu-bionic-base.dockerfile | 6 - .../ci/docker/00-ubuntu-focal-base.dockerfile | 6 - .../docker/00-ubuntu-hirsute-base.dockerfile | 6 - .../docker/00-ubuntu-impish-base.dockerfile | 6 - .../ci/docker/00-ubuntu-lts-base.dockerfile | 6 - .../docker/00-ubuntu-rolling-base.dockerfile | 6 - .../ci/docker/10-debian-bullseye.dockerfile | 46 ---- contrib/ci/docker/10-debian-buster.dockerfile | 47 ---- contrib/ci/docker/10-debian-sid.dockerfile | 46 ---- contrib/ci/docker/10-debian-stable.dockerfile | 46 ---- contrib/ci/docker/10-ubuntu-bionic.dockerfile | 36 --- contrib/ci/docker/10-ubuntu-focal.dockerfile | 47 ---- contrib/ci/docker/10-ubuntu-impish.dockerfile | 47 ---- contrib/ci/docker/10-ubuntu-lts.dockerfile | 47 ---- .../ci/docker/10-ubuntu-rolling.dockerfile | 47 ---- .../docker/20-deb-builder-bionic.dockerfile | 10 - .../docker/20-deb-builder-bullseye.dockerfile | 10 - .../docker/20-deb-builder-buster.dockerfile | 10 - .../ci/docker/20-deb-builder-focal.dockerfile | 10 - .../docker/20-deb-builder-hirsute.dockerfile | 10 - .../docker/20-deb-builder-impish.dockerfile | 10 - .../ci/docker/20-deb-builder-sid.dockerfile | 10 - .../40-debian-bullseye-debhelper.dockerfile | 10 - .../docker/40-debian-sid-debhelper.dockerfile | 10 - .../40-ubuntu-focal-debhelper.dockerfile | 10 - .../40-ubuntu-impish-debhelper.dockerfile | 10 - contrib/ci/docker/50-android.dockerfile | 23 -- .../00-debian-bullseye-base.dockerfile | 1 - .../arm32v7/00-debian-buster-base.dockerfile | 1 - .../arm32v7/00-debian-sid-base.dockerfile | 1 - .../arm32v7/00-debian-stable-base.dockerfile | 1 - .../arm32v7/00-ubuntu-bionic-base.dockerfile | 1 - .../arm32v7/00-ubuntu-focal-base.dockerfile | 1 - .../arm32v7/00-ubuntu-hirsute-base.dockerfile | 1 - .../arm32v7/00-ubuntu-impish-base.dockerfile | 1 - .../arm32v7/10-debian-buster.dockerfile | 1 - .../arm32v7/10-debian-stable.dockerfile | 1 - .../arm32v7/20-deb-builder-bionic.dockerfile | 1 - .../20-deb-builder-bullseye.dockerfile | 1 - .../arm32v7/20-deb-builder-buster.dockerfile | 1 - .../arm32v7/20-deb-builder-focal.dockerfile | 1 - .../arm32v7/20-deb-builder-hirsute.dockerfile | 1 - .../arm32v7/20-deb-builder-impish.dockerfile | 1 - .../arm32v7/20-deb-builder-sid.dockerfile | 1 - .../00-debian-bullseye-base.dockerfile | 1 - .../arm64v8/00-debian-buster-base.dockerfile | 1 - .../arm64v8/00-debian-sid-base.dockerfile | 1 - .../arm64v8/00-ubuntu-bionic-base.dockerfile | 1 - .../arm64v8/00-ubuntu-focal-base.dockerfile | 1 - .../arm64v8/00-ubuntu-hirsute-base.dockerfile | 1 - .../arm64v8/00-ubuntu-impish-base.dockerfile | 1 - .../docker/arm64v8/10-debian-sid.dockerfile | 1 - .../arm64v8/20-deb-builder-bionic.dockerfile | 1 - .../20-deb-builder-bullseye.dockerfile | 1 - .../arm64v8/20-deb-builder-buster.dockerfile | 1 - .../arm64v8/20-deb-builder-focal.dockerfile | 1 - .../arm64v8/20-deb-builder-hirsute.dockerfile | 1 - .../arm64v8/20-deb-builder-impish.dockerfile | 1 - .../arm64v8/20-deb-builder-sid.dockerfile | 1 - .../40-debian-sid-debhelper.dockerfile | 1 - contrib/ci/docker/debian-sid-clang.dockerfile | 7 - .../ci/docker/debian-win32-cross.dockerfile | 24 -- contrib/ci/docker/flutter.dockerfile | 7 - .../i386/00-debian-bullseye-base.dockerfile | 1 - .../i386/00-debian-buster-base.dockerfile | 1 - .../docker/i386/00-debian-sid-base.dockerfile | 1 - .../i386/00-debian-stable-base.dockerfile | 1 - .../i386/00-ubuntu-bionic-base.dockerfile | 1 - .../docker/i386/10-debian-stable.dockerfile | 1 - .../i386/20-deb-builder-bionic.dockerfile | 1 - .../i386/20-deb-builder-bullseye.dockerfile | 1 - .../i386/20-deb-builder-buster.dockerfile | 1 - .../docker/i386/20-deb-builder-sid.dockerfile | 1 - contrib/ci/docker/lint.dockerfile | 7 - contrib/ci/docker/nodejs.dockerfile | 17 -- contrib/ci/docker/readme.md | 6 +- contrib/ci/docker/rebuild-docker-images.py | 251 ++++++++++++++++++ contrib/ci/docker/rebuild-docker-images.sh | 44 --- 83 files changed, 252 insertions(+), 762 deletions(-) delete mode 100644 contrib/ci/docker/00-debian-bullseye-base.dockerfile delete mode 100644 contrib/ci/docker/00-debian-buster-base.dockerfile delete mode 100644 contrib/ci/docker/00-debian-sid-base.dockerfile delete mode 100644 contrib/ci/docker/00-debian-stable-base.dockerfile delete mode 100644 contrib/ci/docker/00-debian-testing-base.dockerfile delete mode 100644 contrib/ci/docker/00-ubuntu-bionic-base.dockerfile delete mode 100644 contrib/ci/docker/00-ubuntu-focal-base.dockerfile delete mode 100644 contrib/ci/docker/00-ubuntu-hirsute-base.dockerfile delete mode 100644 contrib/ci/docker/00-ubuntu-impish-base.dockerfile delete mode 100644 contrib/ci/docker/00-ubuntu-lts-base.dockerfile delete mode 100644 contrib/ci/docker/00-ubuntu-rolling-base.dockerfile delete mode 100644 contrib/ci/docker/10-debian-bullseye.dockerfile delete mode 100644 contrib/ci/docker/10-debian-buster.dockerfile delete mode 100644 contrib/ci/docker/10-debian-sid.dockerfile delete mode 100644 contrib/ci/docker/10-debian-stable.dockerfile delete mode 100644 contrib/ci/docker/10-ubuntu-bionic.dockerfile delete mode 100644 contrib/ci/docker/10-ubuntu-focal.dockerfile delete mode 100644 contrib/ci/docker/10-ubuntu-impish.dockerfile delete mode 100644 contrib/ci/docker/10-ubuntu-lts.dockerfile delete mode 100644 contrib/ci/docker/10-ubuntu-rolling.dockerfile delete mode 100644 contrib/ci/docker/20-deb-builder-bionic.dockerfile delete mode 100644 contrib/ci/docker/20-deb-builder-bullseye.dockerfile delete mode 100644 contrib/ci/docker/20-deb-builder-buster.dockerfile delete mode 100644 contrib/ci/docker/20-deb-builder-focal.dockerfile delete mode 100644 contrib/ci/docker/20-deb-builder-hirsute.dockerfile delete mode 100644 contrib/ci/docker/20-deb-builder-impish.dockerfile delete mode 100644 contrib/ci/docker/20-deb-builder-sid.dockerfile delete mode 100644 contrib/ci/docker/40-debian-bullseye-debhelper.dockerfile delete mode 100644 contrib/ci/docker/40-debian-sid-debhelper.dockerfile delete mode 100644 contrib/ci/docker/40-ubuntu-focal-debhelper.dockerfile delete mode 100644 contrib/ci/docker/40-ubuntu-impish-debhelper.dockerfile delete mode 100644 contrib/ci/docker/50-android.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/00-debian-bullseye-base.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/00-debian-buster-base.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/00-debian-sid-base.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/00-debian-stable-base.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/00-ubuntu-bionic-base.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/00-ubuntu-focal-base.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/00-ubuntu-hirsute-base.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/00-ubuntu-impish-base.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/10-debian-buster.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/10-debian-stable.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/20-deb-builder-bionic.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/20-deb-builder-bullseye.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/20-deb-builder-buster.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/20-deb-builder-focal.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/20-deb-builder-hirsute.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/20-deb-builder-impish.dockerfile delete mode 120000 contrib/ci/docker/arm32v7/20-deb-builder-sid.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/00-debian-bullseye-base.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/00-debian-buster-base.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/00-debian-sid-base.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/00-ubuntu-bionic-base.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/00-ubuntu-focal-base.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/00-ubuntu-hirsute-base.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/00-ubuntu-impish-base.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/10-debian-sid.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/20-deb-builder-bionic.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/20-deb-builder-bullseye.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/20-deb-builder-buster.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/20-deb-builder-focal.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/20-deb-builder-hirsute.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/20-deb-builder-impish.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/20-deb-builder-sid.dockerfile delete mode 120000 contrib/ci/docker/arm64v8/40-debian-sid-debhelper.dockerfile delete mode 100644 contrib/ci/docker/debian-sid-clang.dockerfile delete mode 100644 contrib/ci/docker/debian-win32-cross.dockerfile delete mode 100644 contrib/ci/docker/flutter.dockerfile delete mode 120000 contrib/ci/docker/i386/00-debian-bullseye-base.dockerfile delete mode 120000 contrib/ci/docker/i386/00-debian-buster-base.dockerfile delete mode 120000 contrib/ci/docker/i386/00-debian-sid-base.dockerfile delete mode 120000 contrib/ci/docker/i386/00-debian-stable-base.dockerfile delete mode 120000 contrib/ci/docker/i386/00-ubuntu-bionic-base.dockerfile delete mode 120000 contrib/ci/docker/i386/10-debian-stable.dockerfile delete mode 120000 contrib/ci/docker/i386/20-deb-builder-bionic.dockerfile delete mode 120000 contrib/ci/docker/i386/20-deb-builder-bullseye.dockerfile delete mode 120000 contrib/ci/docker/i386/20-deb-builder-buster.dockerfile delete mode 120000 contrib/ci/docker/i386/20-deb-builder-sid.dockerfile delete mode 100644 contrib/ci/docker/lint.dockerfile delete mode 100644 contrib/ci/docker/nodejs.dockerfile create mode 100755 contrib/ci/docker/rebuild-docker-images.py delete mode 100755 contrib/ci/docker/rebuild-docker-images.sh diff --git a/contrib/ci/docker/00-debian-bullseye-base.dockerfile b/contrib/ci/docker/00-debian-bullseye-base.dockerfile deleted file mode 100644 index 4d691d074..000000000 --- a/contrib/ci/docker/00-debian-bullseye-base.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG ARCH=amd64 -FROM ${ARCH}/debian:bullseye -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-debian-buster-base.dockerfile b/contrib/ci/docker/00-debian-buster-base.dockerfile deleted file mode 100644 index 43c6ca4be..000000000 --- a/contrib/ci/docker/00-debian-buster-base.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG ARCH=amd64 -FROM ${ARCH}/debian:buster -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-debian-sid-base.dockerfile b/contrib/ci/docker/00-debian-sid-base.dockerfile deleted file mode 100644 index 9293b6e9e..000000000 --- a/contrib/ci/docker/00-debian-sid-base.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG ARCH=amd64 -FROM ${ARCH}/debian:sid -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-debian-stable-base.dockerfile b/contrib/ci/docker/00-debian-stable-base.dockerfile deleted file mode 100644 index 62eedbd67..000000000 --- a/contrib/ci/docker/00-debian-stable-base.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG ARCH=amd64 -FROM ${ARCH}/debian:stable -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-debian-testing-base.dockerfile b/contrib/ci/docker/00-debian-testing-base.dockerfile deleted file mode 100644 index 714e24542..000000000 --- a/contrib/ci/docker/00-debian-testing-base.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG ARCH=amd64 -FROM ${ARCH}/debian:testing -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-ubuntu-bionic-base.dockerfile b/contrib/ci/docker/00-ubuntu-bionic-base.dockerfile deleted file mode 100644 index 99ccc1bab..000000000 --- a/contrib/ci/docker/00-ubuntu-bionic-base.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG ARCH=amd64 -FROM ${ARCH}/ubuntu:bionic -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-ubuntu-focal-base.dockerfile b/contrib/ci/docker/00-ubuntu-focal-base.dockerfile deleted file mode 100644 index f3d190bff..000000000 --- a/contrib/ci/docker/00-ubuntu-focal-base.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG ARCH=amd64 -FROM ${ARCH}/ubuntu:focal -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-ubuntu-hirsute-base.dockerfile b/contrib/ci/docker/00-ubuntu-hirsute-base.dockerfile deleted file mode 100644 index b17dbf29e..000000000 --- a/contrib/ci/docker/00-ubuntu-hirsute-base.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG ARCH=amd64 -FROM ${ARCH}/ubuntu:hirsute -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-ubuntu-impish-base.dockerfile b/contrib/ci/docker/00-ubuntu-impish-base.dockerfile deleted file mode 100644 index 02ce35ef6..000000000 --- a/contrib/ci/docker/00-ubuntu-impish-base.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG ARCH=amd64 -FROM ${ARCH}/ubuntu:impish -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-ubuntu-lts-base.dockerfile b/contrib/ci/docker/00-ubuntu-lts-base.dockerfile deleted file mode 100644 index f3d190bff..000000000 --- a/contrib/ci/docker/00-ubuntu-lts-base.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG ARCH=amd64 -FROM ${ARCH}/ubuntu:focal -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-ubuntu-rolling-base.dockerfile b/contrib/ci/docker/00-ubuntu-rolling-base.dockerfile deleted file mode 100644 index 1d813f727..000000000 --- a/contrib/ci/docker/00-ubuntu-rolling-base.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG ARCH=amd64 -FROM ${ARCH}/ubuntu:rolling -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/10-debian-bullseye.dockerfile b/contrib/ci/docker/10-debian-bullseye.dockerfile deleted file mode 100644 index ed0106180..000000000 --- a/contrib/ci/docker/10-debian-bullseye.dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-debian-bullseye-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ - automake \ - ccache \ - cmake \ - eatmydata \ - g++ \ - gdb \ - git \ - libboost-program-options-dev \ - libboost-serialization-dev \ - libboost-thread-dev \ - libcurl4-openssl-dev \ - libevent-dev \ - libgtest-dev \ - libhidapi-dev \ - libjemalloc-dev \ - libminiupnpc-dev \ - libreadline-dev \ - libsodium-dev \ - libsqlite3-dev \ - libssl-dev \ - libsystemd-dev \ - libtool \ - libunbound-dev \ - libunwind8-dev \ - libusb-1.0.0-dev \ - libuv1-dev \ - libzmq3-dev \ - lsb-release \ - make \ - nettle-dev \ - ninja-build \ - openssh-client \ - patch \ - pkg-config \ - pybind11-dev \ - python3-dev \ - python3-pip \ - python3-pybind11 \ - python3-pytest \ - python3-setuptools \ - qttools5-dev diff --git a/contrib/ci/docker/10-debian-buster.dockerfile b/contrib/ci/docker/10-debian-buster.dockerfile deleted file mode 100644 index d653da838..000000000 --- a/contrib/ci/docker/10-debian-buster.dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-debian-buster-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ - automake \ - ccache \ - cmake \ - eatmydata \ - g++ \ - gdb \ - git \ - libboost-program-options-dev \ - libboost-serialization-dev \ - libboost-thread-dev \ - libcurl4-openssl-dev \ - libevent-dev \ - libgtest-dev \ - libhidapi-dev \ - libjemalloc-dev \ - libminiupnpc-dev \ - libreadline-dev \ - libsodium-dev \ - libsqlite3-dev \ - libssl-dev \ - libsystemd-dev \ - libtool \ - libunbound-dev \ - libunwind8-dev \ - libusb-1.0.0-dev \ - libuv1-dev \ - libzmq3-dev \ - lsb-release \ - make \ - nettle-dev \ - ninja-build \ - openssh-client \ - patch \ - pkg-config \ - pybind11-dev \ - python3-dev \ - python3-pip \ - python3-pybind11 \ - python3-pytest \ - python3-setuptools \ - qttools5-dev \ - xz-utils diff --git a/contrib/ci/docker/10-debian-sid.dockerfile b/contrib/ci/docker/10-debian-sid.dockerfile deleted file mode 100644 index 4c662a20c..000000000 --- a/contrib/ci/docker/10-debian-sid.dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-debian-sid-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ - automake \ - ccache \ - cmake \ - eatmydata \ - g++ \ - gdb \ - git \ - libboost-program-options-dev \ - libboost-serialization-dev \ - libboost-thread-dev \ - libcurl4-openssl-dev \ - libevent-dev \ - libgtest-dev \ - libhidapi-dev \ - libjemalloc-dev \ - libminiupnpc-dev \ - libreadline-dev \ - libsodium-dev \ - libsqlite3-dev \ - libssl-dev \ - libsystemd-dev \ - libtool \ - libunbound-dev \ - libunwind8-dev \ - libusb-1.0.0-dev \ - libuv1-dev \ - libzmq3-dev \ - lsb-release \ - make \ - nettle-dev \ - ninja-build \ - openssh-client \ - patch \ - pkg-config \ - pybind11-dev \ - python3-dev \ - python3-pip \ - python3-pybind11 \ - python3-pytest \ - python3-setuptools \ - qttools5-dev diff --git a/contrib/ci/docker/10-debian-stable.dockerfile b/contrib/ci/docker/10-debian-stable.dockerfile deleted file mode 100644 index adc4c6db7..000000000 --- a/contrib/ci/docker/10-debian-stable.dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-debian-stable-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ - automake \ - ccache \ - cmake \ - eatmydata \ - g++ \ - gdb \ - git \ - libboost-program-options-dev \ - libboost-serialization-dev \ - libboost-thread-dev \ - libcurl4-openssl-dev \ - libevent-dev \ - libgtest-dev \ - libhidapi-dev \ - libjemalloc-dev \ - libminiupnpc-dev \ - libreadline-dev \ - libsodium-dev \ - libsqlite3-dev \ - libssl-dev \ - libsystemd-dev \ - libtool \ - libunbound-dev \ - libunwind8-dev \ - libusb-1.0.0-dev \ - libuv1-dev \ - libzmq3-dev \ - lsb-release \ - make \ - nettle-dev \ - ninja-build \ - openssh-client \ - patch \ - pkg-config \ - pybind11-dev \ - python3-dev \ - python3-pip \ - python3-pybind11 \ - python3-pytest \ - python3-setuptools \ - qttools5-dev diff --git a/contrib/ci/docker/10-ubuntu-bionic.dockerfile b/contrib/ci/docker/10-ubuntu-bionic.dockerfile deleted file mode 100644 index df395fce5..000000000 --- a/contrib/ci/docker/10-ubuntu-bionic.dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-ubuntu-bionic-base/${ARCH} -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q --no-install-recommends install -y \ - autoconf \ - automake \ - ccache \ - cmake \ - eatmydata \ - file \ - g++-8 \ - gdb \ - git \ - gperf \ - libjemalloc-dev \ - libpgm-dev \ - libtool \ - libuv1-dev \ - libzmq3-dev \ - lsb-release \ - make \ - ninja-build \ - openssh-client \ - openssh-client \ - patch \ - pkg-config \ - pybind11-dev \ - python3-dev \ - python3-pip \ - python3-pybind11 \ - python3-pytest \ - python3-setuptools \ - qttools5-dev \ - && mkdir -p /usr/lib/x86_64-linux-gnu/pgm-5.2/include diff --git a/contrib/ci/docker/10-ubuntu-focal.dockerfile b/contrib/ci/docker/10-ubuntu-focal.dockerfile deleted file mode 100644 index 62310d8e9..000000000 --- a/contrib/ci/docker/10-ubuntu-focal.dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-ubuntu-focal-base/${ARCH} -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ - automake \ - ccache \ - cmake \ - eatmydata \ - g++ \ - gdb \ - git \ - libboost-program-options-dev \ - libboost-serialization-dev \ - libboost-thread-dev \ - libcurl4-openssl-dev \ - libevent-dev \ - libgtest-dev \ - libhidapi-dev \ - libjemalloc-dev \ - libminiupnpc-dev \ - libreadline-dev \ - libsodium-dev \ - libsqlite3-dev \ - libssl-dev \ - libsystemd-dev \ - libtool \ - libunbound-dev \ - libunwind8-dev \ - libusb-1.0.0-dev \ - libuv1-dev \ - libzmq3-dev \ - lsb-release \ - make \ - nettle-dev \ - ninja-build \ - openssh-client \ - patch \ - pkg-config \ - pybind11-dev \ - python3-dev \ - python3-pip \ - python3-pybind11 \ - python3-pytest \ - python3-setuptools \ - qttools5-dev diff --git a/contrib/ci/docker/10-ubuntu-impish.dockerfile b/contrib/ci/docker/10-ubuntu-impish.dockerfile deleted file mode 100644 index d7e81cab0..000000000 --- a/contrib/ci/docker/10-ubuntu-impish.dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-ubuntu-impish-base/${ARCH} -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ - automake \ - ccache \ - cmake \ - eatmydata \ - g++ \ - gdb \ - git \ - libboost-program-options-dev \ - libboost-serialization-dev \ - libboost-thread-dev \ - libcurl4-openssl-dev \ - libevent-dev \ - libgtest-dev \ - libhidapi-dev \ - libjemalloc-dev \ - libminiupnpc-dev \ - libreadline-dev \ - libsodium-dev \ - libsqlite3-dev \ - libssl-dev \ - libsystemd-dev \ - libtool \ - libunbound-dev \ - libunwind8-dev \ - libusb-1.0.0-dev \ - libuv1-dev \ - libzmq3-dev \ - lsb-release \ - make \ - nettle-dev \ - ninja-build \ - openssh-client \ - patch \ - pkg-config \ - pybind11-dev \ - python3-dev \ - python3-pip \ - python3-pybind11 \ - python3-pytest \ - python3-setuptools \ - qttools5-dev diff --git a/contrib/ci/docker/10-ubuntu-lts.dockerfile b/contrib/ci/docker/10-ubuntu-lts.dockerfile deleted file mode 100644 index a2f2e7d79..000000000 --- a/contrib/ci/docker/10-ubuntu-lts.dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-ubuntu-lts-base/${ARCH} -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ - automake \ - ccache \ - cmake \ - eatmydata \ - g++ \ - gdb \ - git \ - libboost-program-options-dev \ - libboost-serialization-dev \ - libboost-thread-dev \ - libcurl4-openssl-dev \ - libevent-dev \ - libgtest-dev \ - libhidapi-dev \ - libjemalloc-dev \ - libminiupnpc-dev \ - libreadline-dev \ - libsodium-dev \ - libsqlite3-dev \ - libssl-dev \ - libsystemd-dev \ - libtool \ - libunbound-dev \ - libunwind8-dev \ - libusb-1.0.0-dev \ - libuv1-dev \ - libzmq3-dev \ - lsb-release \ - make \ - nettle-dev \ - ninja-build \ - openssh-client \ - patch \ - pkg-config \ - pybind11-dev \ - python3-dev \ - python3-pip \ - python3-pybind11 \ - python3-pytest \ - python3-setuptools \ - qttools5-dev diff --git a/contrib/ci/docker/10-ubuntu-rolling.dockerfile b/contrib/ci/docker/10-ubuntu-rolling.dockerfile deleted file mode 100644 index 0bcda4935..000000000 --- a/contrib/ci/docker/10-ubuntu-rolling.dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-ubuntu-rolling-base/${ARCH} -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ - automake \ - ccache \ - cmake \ - eatmydata \ - g++ \ - gdb \ - git \ - libboost-program-options-dev \ - libboost-serialization-dev \ - libboost-thread-dev \ - libcurl4-openssl-dev \ - libevent-dev \ - libgtest-dev \ - libhidapi-dev \ - libjemalloc-dev \ - libminiupnpc-dev \ - libreadline-dev \ - libsodium-dev \ - libsqlite3-dev \ - libssl-dev \ - libsystemd-dev \ - libtool \ - libunbound-dev \ - libunwind8-dev \ - libusb-1.0.0-dev \ - libuv1-dev \ - libzmq3-dev \ - lsb-release \ - make \ - nettle-dev \ - ninja-build \ - openssh-client \ - patch \ - pkg-config \ - pybind11-dev \ - python3-dev \ - python3-pip \ - python3-pybind11 \ - python3-pytest \ - python3-setuptools \ - qttools5-dev diff --git a/contrib/ci/docker/20-deb-builder-bionic.dockerfile b/contrib/ci/docker/20-deb-builder-bionic.dockerfile deleted file mode 100644 index 8d1f92aef..000000000 --- a/contrib/ci/docker/20-deb-builder-bionic.dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-ubuntu-bionic-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ - ccache \ - cmake \ - devscripts \ - equivs \ - g++ \ - git-buildpackage \ - openssh-client diff --git a/contrib/ci/docker/20-deb-builder-bullseye.dockerfile b/contrib/ci/docker/20-deb-builder-bullseye.dockerfile deleted file mode 100644 index ac4da81b5..000000000 --- a/contrib/ci/docker/20-deb-builder-bullseye.dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-debian-bullseye-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ - ccache \ - cmake \ - devscripts \ - equivs \ - g++ \ - git-buildpackage \ - openssh-client diff --git a/contrib/ci/docker/20-deb-builder-buster.dockerfile b/contrib/ci/docker/20-deb-builder-buster.dockerfile deleted file mode 100644 index 5573fed90..000000000 --- a/contrib/ci/docker/20-deb-builder-buster.dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-debian-buster-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ - ccache \ - cmake \ - devscripts \ - equivs \ - g++ \ - git-buildpackage \ - openssh-client diff --git a/contrib/ci/docker/20-deb-builder-focal.dockerfile b/contrib/ci/docker/20-deb-builder-focal.dockerfile deleted file mode 100644 index 0831ac026..000000000 --- a/contrib/ci/docker/20-deb-builder-focal.dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-ubuntu-focal-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ - ccache \ - cmake \ - devscripts \ - equivs \ - g++ \ - git-buildpackage \ - openssh-client diff --git a/contrib/ci/docker/20-deb-builder-hirsute.dockerfile b/contrib/ci/docker/20-deb-builder-hirsute.dockerfile deleted file mode 100644 index c7e1a6576..000000000 --- a/contrib/ci/docker/20-deb-builder-hirsute.dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-ubuntu-hirsute-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ - ccache \ - cmake \ - devscripts \ - equivs \ - g++ \ - git-buildpackage \ - openssh-client diff --git a/contrib/ci/docker/20-deb-builder-impish.dockerfile b/contrib/ci/docker/20-deb-builder-impish.dockerfile deleted file mode 100644 index b8e002244..000000000 --- a/contrib/ci/docker/20-deb-builder-impish.dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-ubuntu-impish-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ - ccache \ - cmake \ - devscripts \ - equivs \ - g++ \ - git-buildpackage \ - openssh-client diff --git a/contrib/ci/docker/20-deb-builder-sid.dockerfile b/contrib/ci/docker/20-deb-builder-sid.dockerfile deleted file mode 100644 index d90818747..000000000 --- a/contrib/ci/docker/20-deb-builder-sid.dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-debian-sid-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ - ccache \ - cmake \ - devscripts \ - equivs \ - g++ \ - git-buildpackage \ - openssh-client diff --git a/contrib/ci/docker/40-debian-bullseye-debhelper.dockerfile b/contrib/ci/docker/40-debian-bullseye-debhelper.dockerfile deleted file mode 100644 index 05236d48c..000000000 --- a/contrib/ci/docker/40-debian-bullseye-debhelper.dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-debian-bullseye/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ - ccache \ - debhelper \ - devscripts \ - equivs \ - git \ - git-buildpackage \ - python3-dev diff --git a/contrib/ci/docker/40-debian-sid-debhelper.dockerfile b/contrib/ci/docker/40-debian-sid-debhelper.dockerfile deleted file mode 100644 index e05764508..000000000 --- a/contrib/ci/docker/40-debian-sid-debhelper.dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-debian-sid/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ - ccache \ - debhelper \ - devscripts \ - equivs \ - git \ - git-buildpackage \ - python3-dev diff --git a/contrib/ci/docker/40-ubuntu-focal-debhelper.dockerfile b/contrib/ci/docker/40-ubuntu-focal-debhelper.dockerfile deleted file mode 100644 index 288179538..000000000 --- a/contrib/ci/docker/40-ubuntu-focal-debhelper.dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-ubuntu-focal/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ - ccache \ - debhelper \ - devscripts \ - equivs \ - git \ - git-buildpackage \ - python3-dev diff --git a/contrib/ci/docker/40-ubuntu-impish-debhelper.dockerfile b/contrib/ci/docker/40-ubuntu-impish-debhelper.dockerfile deleted file mode 100644 index e73bb07ec..000000000 --- a/contrib/ci/docker/40-ubuntu-impish-debhelper.dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-ubuntu-impish/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ - ccache \ - debhelper \ - devscripts \ - equivs \ - git \ - git-buildpackage \ - python3-dev diff --git a/contrib/ci/docker/50-android.dockerfile b/contrib/ci/docker/50-android.dockerfile deleted file mode 100644 index 04163fc63..000000000 --- a/contrib/ci/docker/50-android.dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-debian-testing-base/${ARCH} -RUN /bin/bash -c 'sed -i "s/main/main contrib/g" /etc/apt/sources.list' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ - android-sdk \ - automake \ - ccache \ - cmake \ - curl \ - git \ - google-android-ndk-installer \ - libtool \ - make \ - openssh-client \ - patch \ - pkg-config \ - wget \ - xz-utils \ - zip \ - && git clone https://github.com/Shadowstyler/android-sdk-licenses.git /tmp/android-sdk-licenses \ - && cp -a /tmp/android-sdk-licenses/*-license /usr/lib/android-sdk/licenses \ - && rm -rf /tmp/android-sdk-licenses diff --git a/contrib/ci/docker/arm32v7/00-debian-bullseye-base.dockerfile b/contrib/ci/docker/arm32v7/00-debian-bullseye-base.dockerfile deleted file mode 120000 index cfa13d1e1..000000000 --- a/contrib/ci/docker/arm32v7/00-debian-bullseye-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-debian-bullseye-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/00-debian-buster-base.dockerfile b/contrib/ci/docker/arm32v7/00-debian-buster-base.dockerfile deleted file mode 120000 index e4ab00108..000000000 --- a/contrib/ci/docker/arm32v7/00-debian-buster-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-debian-buster-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/00-debian-sid-base.dockerfile b/contrib/ci/docker/arm32v7/00-debian-sid-base.dockerfile deleted file mode 120000 index 3fdec500d..000000000 --- a/contrib/ci/docker/arm32v7/00-debian-sid-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-debian-sid-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/00-debian-stable-base.dockerfile b/contrib/ci/docker/arm32v7/00-debian-stable-base.dockerfile deleted file mode 120000 index 2d62f76d4..000000000 --- a/contrib/ci/docker/arm32v7/00-debian-stable-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-debian-stable-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/00-ubuntu-bionic-base.dockerfile b/contrib/ci/docker/arm32v7/00-ubuntu-bionic-base.dockerfile deleted file mode 120000 index 3764c68b2..000000000 --- a/contrib/ci/docker/arm32v7/00-ubuntu-bionic-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-ubuntu-bionic-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/00-ubuntu-focal-base.dockerfile b/contrib/ci/docker/arm32v7/00-ubuntu-focal-base.dockerfile deleted file mode 120000 index 23993a33b..000000000 --- a/contrib/ci/docker/arm32v7/00-ubuntu-focal-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-ubuntu-focal-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/00-ubuntu-hirsute-base.dockerfile b/contrib/ci/docker/arm32v7/00-ubuntu-hirsute-base.dockerfile deleted file mode 120000 index 825862d56..000000000 --- a/contrib/ci/docker/arm32v7/00-ubuntu-hirsute-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-ubuntu-hirsute-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/00-ubuntu-impish-base.dockerfile b/contrib/ci/docker/arm32v7/00-ubuntu-impish-base.dockerfile deleted file mode 120000 index b9dba0b96..000000000 --- a/contrib/ci/docker/arm32v7/00-ubuntu-impish-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-ubuntu-impish-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/10-debian-buster.dockerfile b/contrib/ci/docker/arm32v7/10-debian-buster.dockerfile deleted file mode 120000 index 592305776..000000000 --- a/contrib/ci/docker/arm32v7/10-debian-buster.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../10-debian-buster.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/10-debian-stable.dockerfile b/contrib/ci/docker/arm32v7/10-debian-stable.dockerfile deleted file mode 120000 index a10f6b3c5..000000000 --- a/contrib/ci/docker/arm32v7/10-debian-stable.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../10-debian-stable.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/20-deb-builder-bionic.dockerfile b/contrib/ci/docker/arm32v7/20-deb-builder-bionic.dockerfile deleted file mode 120000 index ffb9df8ed..000000000 --- a/contrib/ci/docker/arm32v7/20-deb-builder-bionic.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-bionic.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/20-deb-builder-bullseye.dockerfile b/contrib/ci/docker/arm32v7/20-deb-builder-bullseye.dockerfile deleted file mode 120000 index a7647cd32..000000000 --- a/contrib/ci/docker/arm32v7/20-deb-builder-bullseye.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-bullseye.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/20-deb-builder-buster.dockerfile b/contrib/ci/docker/arm32v7/20-deb-builder-buster.dockerfile deleted file mode 120000 index 301cbe821..000000000 --- a/contrib/ci/docker/arm32v7/20-deb-builder-buster.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-buster.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/20-deb-builder-focal.dockerfile b/contrib/ci/docker/arm32v7/20-deb-builder-focal.dockerfile deleted file mode 120000 index 1438e775c..000000000 --- a/contrib/ci/docker/arm32v7/20-deb-builder-focal.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-focal.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/20-deb-builder-hirsute.dockerfile b/contrib/ci/docker/arm32v7/20-deb-builder-hirsute.dockerfile deleted file mode 120000 index 6750c95a0..000000000 --- a/contrib/ci/docker/arm32v7/20-deb-builder-hirsute.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-hirsute.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/20-deb-builder-impish.dockerfile b/contrib/ci/docker/arm32v7/20-deb-builder-impish.dockerfile deleted file mode 120000 index b53e98286..000000000 --- a/contrib/ci/docker/arm32v7/20-deb-builder-impish.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-impish.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/20-deb-builder-sid.dockerfile b/contrib/ci/docker/arm32v7/20-deb-builder-sid.dockerfile deleted file mode 120000 index f80b1be0a..000000000 --- a/contrib/ci/docker/arm32v7/20-deb-builder-sid.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-sid.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/00-debian-bullseye-base.dockerfile b/contrib/ci/docker/arm64v8/00-debian-bullseye-base.dockerfile deleted file mode 120000 index cfa13d1e1..000000000 --- a/contrib/ci/docker/arm64v8/00-debian-bullseye-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-debian-bullseye-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/00-debian-buster-base.dockerfile b/contrib/ci/docker/arm64v8/00-debian-buster-base.dockerfile deleted file mode 120000 index e4ab00108..000000000 --- a/contrib/ci/docker/arm64v8/00-debian-buster-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-debian-buster-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/00-debian-sid-base.dockerfile b/contrib/ci/docker/arm64v8/00-debian-sid-base.dockerfile deleted file mode 120000 index 3fdec500d..000000000 --- a/contrib/ci/docker/arm64v8/00-debian-sid-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-debian-sid-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/00-ubuntu-bionic-base.dockerfile b/contrib/ci/docker/arm64v8/00-ubuntu-bionic-base.dockerfile deleted file mode 120000 index 3764c68b2..000000000 --- a/contrib/ci/docker/arm64v8/00-ubuntu-bionic-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-ubuntu-bionic-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/00-ubuntu-focal-base.dockerfile b/contrib/ci/docker/arm64v8/00-ubuntu-focal-base.dockerfile deleted file mode 120000 index 23993a33b..000000000 --- a/contrib/ci/docker/arm64v8/00-ubuntu-focal-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-ubuntu-focal-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/00-ubuntu-hirsute-base.dockerfile b/contrib/ci/docker/arm64v8/00-ubuntu-hirsute-base.dockerfile deleted file mode 120000 index 825862d56..000000000 --- a/contrib/ci/docker/arm64v8/00-ubuntu-hirsute-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-ubuntu-hirsute-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/00-ubuntu-impish-base.dockerfile b/contrib/ci/docker/arm64v8/00-ubuntu-impish-base.dockerfile deleted file mode 120000 index b9dba0b96..000000000 --- a/contrib/ci/docker/arm64v8/00-ubuntu-impish-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-ubuntu-impish-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/10-debian-sid.dockerfile b/contrib/ci/docker/arm64v8/10-debian-sid.dockerfile deleted file mode 120000 index baf7ccaad..000000000 --- a/contrib/ci/docker/arm64v8/10-debian-sid.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../10-debian-sid.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/20-deb-builder-bionic.dockerfile b/contrib/ci/docker/arm64v8/20-deb-builder-bionic.dockerfile deleted file mode 120000 index ffb9df8ed..000000000 --- a/contrib/ci/docker/arm64v8/20-deb-builder-bionic.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-bionic.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/20-deb-builder-bullseye.dockerfile b/contrib/ci/docker/arm64v8/20-deb-builder-bullseye.dockerfile deleted file mode 120000 index a7647cd32..000000000 --- a/contrib/ci/docker/arm64v8/20-deb-builder-bullseye.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-bullseye.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/20-deb-builder-buster.dockerfile b/contrib/ci/docker/arm64v8/20-deb-builder-buster.dockerfile deleted file mode 120000 index 301cbe821..000000000 --- a/contrib/ci/docker/arm64v8/20-deb-builder-buster.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-buster.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/20-deb-builder-focal.dockerfile b/contrib/ci/docker/arm64v8/20-deb-builder-focal.dockerfile deleted file mode 120000 index 1438e775c..000000000 --- a/contrib/ci/docker/arm64v8/20-deb-builder-focal.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-focal.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/20-deb-builder-hirsute.dockerfile b/contrib/ci/docker/arm64v8/20-deb-builder-hirsute.dockerfile deleted file mode 120000 index 6750c95a0..000000000 --- a/contrib/ci/docker/arm64v8/20-deb-builder-hirsute.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-hirsute.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/20-deb-builder-impish.dockerfile b/contrib/ci/docker/arm64v8/20-deb-builder-impish.dockerfile deleted file mode 120000 index b53e98286..000000000 --- a/contrib/ci/docker/arm64v8/20-deb-builder-impish.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-impish.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/20-deb-builder-sid.dockerfile b/contrib/ci/docker/arm64v8/20-deb-builder-sid.dockerfile deleted file mode 120000 index f80b1be0a..000000000 --- a/contrib/ci/docker/arm64v8/20-deb-builder-sid.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-sid.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/40-debian-sid-debhelper.dockerfile b/contrib/ci/docker/arm64v8/40-debian-sid-debhelper.dockerfile deleted file mode 120000 index e7f04dc7a..000000000 --- a/contrib/ci/docker/arm64v8/40-debian-sid-debhelper.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../40-debian-sid-debhelper.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/debian-sid-clang.dockerfile b/contrib/ci/docker/debian-sid-clang.dockerfile deleted file mode 100644 index 6d21df7fe..000000000 --- a/contrib/ci/docker/debian-sid-clang.dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-debian-sid/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ - clang-13 \ - libc++-13-dev \ - libc++abi-13-dev \ - lld-13 diff --git a/contrib/ci/docker/debian-win32-cross.dockerfile b/contrib/ci/docker/debian-win32-cross.dockerfile deleted file mode 100644 index f2d852644..000000000 --- a/contrib/ci/docker/debian-win32-cross.dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-debian-testing-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ - autoconf \ - automake \ - build-essential \ - ccache \ - cmake \ - eatmydata \ - file \ - g++-mingw-w64-x86-64-posix \ - git \ - gperf \ - libtool \ - make \ - ninja-build \ - nsis \ - openssh-client \ - patch \ - pkg-config \ - qttools5-dev \ - zip \ - && update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix \ - && update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix diff --git a/contrib/ci/docker/flutter.dockerfile b/contrib/ci/docker/flutter.dockerfile deleted file mode 100644 index 17b39a980..000000000 --- a/contrib/ci/docker/flutter.dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-android/${ARCH} -RUN cd /opt \ - && curl https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_2.2.2-stable.tar.xz \ - | tar xJv \ - && ln -s /opt/flutter/bin/flutter /usr/local/bin/ \ - && flutter precache diff --git a/contrib/ci/docker/i386/00-debian-bullseye-base.dockerfile b/contrib/ci/docker/i386/00-debian-bullseye-base.dockerfile deleted file mode 120000 index cfa13d1e1..000000000 --- a/contrib/ci/docker/i386/00-debian-bullseye-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-debian-bullseye-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/00-debian-buster-base.dockerfile b/contrib/ci/docker/i386/00-debian-buster-base.dockerfile deleted file mode 120000 index e4ab00108..000000000 --- a/contrib/ci/docker/i386/00-debian-buster-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-debian-buster-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/00-debian-sid-base.dockerfile b/contrib/ci/docker/i386/00-debian-sid-base.dockerfile deleted file mode 120000 index 3fdec500d..000000000 --- a/contrib/ci/docker/i386/00-debian-sid-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-debian-sid-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/00-debian-stable-base.dockerfile b/contrib/ci/docker/i386/00-debian-stable-base.dockerfile deleted file mode 120000 index 2d62f76d4..000000000 --- a/contrib/ci/docker/i386/00-debian-stable-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-debian-stable-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/00-ubuntu-bionic-base.dockerfile b/contrib/ci/docker/i386/00-ubuntu-bionic-base.dockerfile deleted file mode 120000 index 3764c68b2..000000000 --- a/contrib/ci/docker/i386/00-ubuntu-bionic-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../00-ubuntu-bionic-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/10-debian-stable.dockerfile b/contrib/ci/docker/i386/10-debian-stable.dockerfile deleted file mode 120000 index a10f6b3c5..000000000 --- a/contrib/ci/docker/i386/10-debian-stable.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../10-debian-stable.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/20-deb-builder-bionic.dockerfile b/contrib/ci/docker/i386/20-deb-builder-bionic.dockerfile deleted file mode 120000 index ffb9df8ed..000000000 --- a/contrib/ci/docker/i386/20-deb-builder-bionic.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-bionic.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/20-deb-builder-bullseye.dockerfile b/contrib/ci/docker/i386/20-deb-builder-bullseye.dockerfile deleted file mode 120000 index a7647cd32..000000000 --- a/contrib/ci/docker/i386/20-deb-builder-bullseye.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-bullseye.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/20-deb-builder-buster.dockerfile b/contrib/ci/docker/i386/20-deb-builder-buster.dockerfile deleted file mode 120000 index 301cbe821..000000000 --- a/contrib/ci/docker/i386/20-deb-builder-buster.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-buster.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/20-deb-builder-sid.dockerfile b/contrib/ci/docker/i386/20-deb-builder-sid.dockerfile deleted file mode 120000 index f80b1be0a..000000000 --- a/contrib/ci/docker/i386/20-deb-builder-sid.dockerfile +++ /dev/null @@ -1 +0,0 @@ -../20-deb-builder-sid.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/lint.dockerfile b/contrib/ci/docker/lint.dockerfile deleted file mode 100644 index d9636827d..000000000 --- a/contrib/ci/docker/lint.dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -ARG ARCH=amd64 -FROM registry.oxen.rocks/lokinet-ci-debian-sid-base/${ARCH} -RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ - clang-format-11 \ - eatmydata \ - git \ - jsonnet diff --git a/contrib/ci/docker/nodejs.dockerfile b/contrib/ci/docker/nodejs.dockerfile deleted file mode 100644 index 625251173..000000000 --- a/contrib/ci/docker/nodejs.dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM node:14.16.1 -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ - ccache \ - cmake \ - eatmydata \ - g++ \ - gdb \ - git \ - make \ - ninja-build \ - openssh-client \ - patch \ - pkg-config \ - wine diff --git a/contrib/ci/docker/readme.md b/contrib/ci/docker/readme.md index 5cd271f2d..210c17655 100644 --- a/contrib/ci/docker/readme.md +++ b/contrib/ci/docker/readme.md @@ -3,11 +3,7 @@ To rebuild all ci images and push them to the oxen registry server do: $ docker login registry.oxen.rocks - $ ./rebuild-docker-images.sh + $ ./rebuild-docker-images.py If you aren't part of the Oxen team, you'll likely need to set up your own registry and change registry.oxen.rocks to your own domain name in order to do anything useful with this. - -The docker images will be `registry.oxen.rocks/lokinet-ci-*`for each \*.dockerfile in this -directory, with the leading numeric `NN-` removed, if present (so that you can ensure proper -ordering using two-digit numeric prefixes). diff --git a/contrib/ci/docker/rebuild-docker-images.py b/contrib/ci/docker/rebuild-docker-images.py new file mode 100755 index 000000000..8d8aa846a --- /dev/null +++ b/contrib/ci/docker/rebuild-docker-images.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 + +import subprocess +import tempfile + +no_cache = True # Whether to build with --no-cache + +registry_base = 'registry.oxen.rocks/lokinet-ci-' + +distros = [*(['debian', x] for x in ('sid', 'stable', 'testing', 'bullseye', 'buster')), + *(['ubuntu', x] for x in ('rolling', 'lts', 'impish', 'hirsute', 'focal', 'bionic'))] + +manifests = {} # "image:latest": ["image/amd64", "image/arm64v8", ...] + + +def arches(distro): + a = ['amd64', 'arm64v8', 'arm32v7'] + if distro[0] == 'debian' or distro == ['ubuntu', 'bionic']: + a.append('i386') # i386 builds don't work on later ubuntu + return a + + +def build_tag(tag_base, arch, contents): + with tempfile.NamedTemporaryFile() as dockerfile: + dockerfile.write(contents.encode()) + dockerfile.flush() + + tag = '{}/{}'.format(tag_base, arch) + print("\033[32;1mrebuilding \033[35;1m{}\033[0m".format(tag)) + subprocess.run(['docker', 'build', '--pull', '-f', dockerfile.name, '-t', tag, + *(('--no-cache',) if no_cache else ()), '.'], + check=True) + subprocess.run(['docker', 'push', tag], check=True) + + latest = tag_base + ':latest' + if latest in manifests: + manifests[latest].append(tag) + else: + manifests[latest] = [tag] + + +def distro_build(distro, arch): + prefix = '{r}{distro[0]}-{distro[1]}'.format(r=registry_base, distro=distro) + fmtargs = dict(arch=arch, distro=distro, prefix=prefix) + + # (distro)-(codename)-base: Base image from upstream: we sync the repos, but do nothing else. + build_tag(prefix + '-base', arch, """ +FROM {arch}/{distro[0]}:{codename} +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y +""".format(**fmtargs, codename='latest' if distro == ['ubuntu', 'lts'] else distro[1])) + + # (distro)-(codename)-builder: Deb builder image used for building debs; we add the basic tools + # we use to build debs, not including things that should come from the dependencies in the + # debian/control file. + build_tag(prefix + '-builder', arch, """ +FROM {prefix}-base/{arch} +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + devscripts \ + equivs \ + g++ \ + git \ + git-buildpackage \ + openssh-client +""".format(**fmtargs)) + + # (distro)-(codename): Basic image we use for most builds. This takes the -builder and adds + # most dependencies found in our packages. + build_tag(prefix, arch, """ +FROM {prefix}-builder/{arch} +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + automake \ + ccache \ + cmake \ + eatmydata \ + g++ \ + gdb \ + git \ + libboost-program-options-dev \ + libboost-serialization-dev \ + libboost-thread-dev \ + libcurl4-openssl-dev \ + libevent-dev \ + libgtest-dev \ + libhidapi-dev \ + libjemalloc-dev \ + libminiupnpc-dev \ + libreadline-dev \ + libsodium-dev \ + libsqlite3-dev \ + libssl-dev \ + libsystemd-dev \ + libtool \ + libunbound-dev \ + libunwind8-dev \ + libusb-1.0.0-dev \ + libuv1-dev \ + libzmq3-dev \ + lsb-release \ + make \ + nettle-dev \ + ninja-build \ + openssh-client \ + patch \ + pkg-config \ + pybind11-dev \ + python3-dev \ + python3-pip \ + python3-pybind11 \ + python3-pytest \ + python3-setuptools \ + qttools5-dev +""".format(**fmtargs)) + + +for d in distros: + for a in arches(distros): + distro_build(d, a) + + +# Other images: + +# lint is a tiny build with just formatting checking tools + +build_tag(registry_base + 'lint', 'amd64', """ +FROM {r}debian-stable-base +RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + clang-format-11 \ + eatmydata \ + git \ + jsonnet +""".format(r=registry_base)) + +# nodejs +build_tag(registry_base + 'nodejs', 'amd64', """ +FROM node:14.16.1 +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + ccache \ + cmake \ + eatmydata \ + g++ \ + gdb \ + git \ + make \ + ninja-build \ + openssh-client \ + patch \ + pkg-config \ + wine +""") + + +# Android builds on debian-stable-base and adds a ton of android crap: +build_tag(registry_base + 'android', 'amd64', """ +FROM {r}debian-stable-base +RUN /bin/bash -c 'sed -i "s/main/main contrib/g" /etc/apt/sources.list' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + android-sdk \ + automake \ + ccache \ + cmake \ + curl \ + git \ + google-android-ndk-installer \ + libtool \ + make \ + openssh-client \ + patch \ + pkg-config \ + wget \ + xz-utils \ + zip \ + && git clone https://github.com/Shadowstyler/android-sdk-licenses.git /tmp/android-sdk-licenses \ + && cp -a /tmp/android-sdk-licenses/*-license /usr/lib/android-sdk/licenses \ + && rm -rf /tmp/android-sdk-licenses +""".format(r=registry_base)) + + +# Flutter image takes android and adds even more crap: +build_tag(registry_base + 'flutter', 'amd64', """ +FROM {r}android +RUN cd /opt \ + && curl https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_2.2.2-stable.tar.xz \ + | tar xJv \ + && ln -s /opt/flutter/bin/flutter /usr/local/bin/ \ + && flutter precache +""".format(r=registry_base)) + + +# debian-sid-clang adds clang + libc++ to debian-sid (amd64 only) +build_tag(registry_base + 'debian-sid-clang', 'amd64', """ +FROM {r}debian-sid +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + clang-13 \ + libc++-13-dev \ + libc++abi-13-dev \ + lld-13 +""".format(r=registry_base)) + + +# debian-win32-cross is debian-testing-base + stuff for compiling windows binaries +build_tag(registry_base + 'debian-win32-cross', 'amd64', """ +FROM {r}debian-testing-base +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + autoconf \ + automake \ + build-essential \ + ccache \ + cmake \ + eatmydata \ + file \ + g++-mingw-w64-x86-64-posix \ + git \ + gperf \ + libtool \ + make \ + ninja-build \ + nsis \ + openssh-client \ + patch \ + pkg-config \ + qttools5-dev \ + zip \ + && update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix \ + && update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix +""".format(r=registry_base)) + + +for latest, tags in manifests.items(): + print("\033[32;1mpushing new manifest for \033[33;1m{}[\033[35;1m{}\033[33;1m]\033[0m".format( + latest, ', '.join(tags))) + + subprocess.run(['manifest', 'rm', latest], stderr=DEVNULL, check=False) + subprocess.run(['manifest', 'create', latest, *tags], check=True) + subprocess.run(['manifest', 'push', latest], check=True) diff --git a/contrib/ci/docker/rebuild-docker-images.sh b/contrib/ci/docker/rebuild-docker-images.sh deleted file mode 100755 index bded2f2f4..000000000 --- a/contrib/ci/docker/rebuild-docker-images.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -set -o errexit - -trap 'echo -e "\n\n\n\e[31;1mAn error occurred!\e[1m\n\n"' ERR - -registry=registry.oxen.rocks - -if [[ $# -eq 0 ]]; then - files=(*.dockerfile i386/*.dockerfile arm64v8/*.dockerfile arm32v7/*.dockerfile) -else - files=("$@") -fi - -declare -A manifests - -for file in "${files[@]}"; do - if [[ "$file" == */* ]]; then - arch="${file%%/*}" - name="${file#*/}" - else - arch="amd64" - name="$file" - fi - - name="${name#[0-9][0-9]-}" # s/^\d\d-// - name="${name%.dockerfile}" # s/\.dockerfile$// - namearch=$registry/lokinet-ci-$name/$arch - latest=$registry/lokinet-ci-$name:latest - echo -e "\e[32;1mrebuilding \e[35;1m$namearch\e[0m" - docker build --pull -f $file -t $namearch --build-arg ARCH=$arch $DOCKER_BUILD_OPTS . - docker push $namearch - - manifests[$latest]="${manifests[$latest]} $namearch" -done - -for latest in "${!manifests[@]}"; do - echo -e "\e[32;1mpushing new manifest for \e[33;1m$latest[\e[35;1m${manifests[$latest]} \e[33;1m]\e[0m" - docker manifest rm $latest 2>/dev/null || true - docker manifest create $latest ${manifests[$latest]} - docker manifest push $latest -done - -echo -e "\n\n\n\e[32;1mAll done!\e[1m\n\n" From 9b7ec7dd50a31b9c6f2e280be77d33f6f337236e Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 26 Oct 2021 10:56:27 -0300 Subject: [PATCH 08/53] Add --no-cache option; fix manifest generation --- contrib/ci/docker/rebuild-docker-images.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/contrib/ci/docker/rebuild-docker-images.py b/contrib/ci/docker/rebuild-docker-images.py index 8d8aa846a..2934b501b 100755 --- a/contrib/ci/docker/rebuild-docker-images.py +++ b/contrib/ci/docker/rebuild-docker-images.py @@ -2,8 +2,12 @@ import subprocess import tempfile +import optparse -no_cache = True # Whether to build with --no-cache +parser = optparse.OptionParser() +parser.add_option("--no-cache", action="store_true", + help="Run `docker build` with the `--no-cache` option to ignore existing images") +(options, args) = parser.parse_args() registry_base = 'registry.oxen.rocks/lokinet-ci-' @@ -28,7 +32,7 @@ def build_tag(tag_base, arch, contents): tag = '{}/{}'.format(tag_base, arch) print("\033[32;1mrebuilding \033[35;1m{}\033[0m".format(tag)) subprocess.run(['docker', 'build', '--pull', '-f', dockerfile.name, '-t', tag, - *(('--no-cache',) if no_cache else ()), '.'], + *(('--no-cache',) if options.no_cache else ()), '.'], check=True) subprocess.run(['docker', 'push', tag], check=True) @@ -246,6 +250,6 @@ for latest, tags in manifests.items(): print("\033[32;1mpushing new manifest for \033[33;1m{}[\033[35;1m{}\033[33;1m]\033[0m".format( latest, ', '.join(tags))) - subprocess.run(['manifest', 'rm', latest], stderr=DEVNULL, check=False) - subprocess.run(['manifest', 'create', latest, *tags], check=True) - subprocess.run(['manifest', 'push', latest], check=True) + subprocess.run(['docker', 'manifest', 'rm', latest], stderr=subprocess.DEVNULL, check=False) + subprocess.run(['docker', 'manifest', 'create', latest, *tags], check=True) + subprocess.run(['docker', 'manifest', 'push', latest], check=True) From 2064ca049dd565aad380fb26abf26739ec03e2d8 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 26 Oct 2021 14:05:35 -0300 Subject: [PATCH 09/53] Parallelize builds --- contrib/ci/docker/rebuild-docker-images.py | 188 +++++++++++---------- 1 file changed, 100 insertions(+), 88 deletions(-) diff --git a/contrib/ci/docker/rebuild-docker-images.py b/contrib/ci/docker/rebuild-docker-images.py index 2934b501b..59a549aa3 100755 --- a/contrib/ci/docker/rebuild-docker-images.py +++ b/contrib/ci/docker/rebuild-docker-images.py @@ -3,58 +3,93 @@ import subprocess import tempfile import optparse +import sys +from concurrent.futures import ThreadPoolExecutor parser = optparse.OptionParser() parser.add_option("--no-cache", action="store_true", help="Run `docker build` with the `--no-cache` option to ignore existing images") +parser.add_option("--parallel", "-j", type="int", default=1, + help="Run up to this many builds in parallel") (options, args) = parser.parse_args() registry_base = 'registry.oxen.rocks/lokinet-ci-' -distros = [*(['debian', x] for x in ('sid', 'stable', 'testing', 'bullseye', 'buster')), - *(['ubuntu', x] for x in ('rolling', 'lts', 'impish', 'hirsute', 'focal', 'bionic'))] +distros = [*(('debian', x) for x in ('sid', 'stable', 'testing', 'bullseye', 'buster')), + *(('ubuntu', x) for x in ('rolling', 'lts', 'impish', 'hirsute', 'focal', 'bionic'))] manifests = {} # "image:latest": ["image/amd64", "image/arm64v8", ...] def arches(distro): a = ['amd64', 'arm64v8', 'arm32v7'] - if distro[0] == 'debian' or distro == ['ubuntu', 'bionic']: + if distro[0] == 'debian' or distro == ('ubuntu', 'bionic'): a.append('i386') # i386 builds don't work on later ubuntu return a +failure = False + + +def run_or_report(*args): + try: + subprocess.run( + args, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf8') + except subprocess.CalledProcessError as e: + with tempfile.NamedTemporaryFile(suffix=".log", delete=False) as log: + log.write(e.output.encode()) + global failure + failure = True + print(""" +\033[31;1mAn error occured ({}) running: + {} +See {} for the command log.\033[0m +""".format(e, ' '.join(args), log.name), file=sys.stderr) + raise e + + def build_tag(tag_base, arch, contents): + if failure: + raise ChildProcessError() with tempfile.NamedTemporaryFile() as dockerfile: dockerfile.write(contents.encode()) dockerfile.flush() tag = '{}/{}'.format(tag_base, arch) - print("\033[32;1mrebuilding \033[35;1m{}\033[0m".format(tag)) - subprocess.run(['docker', 'build', '--pull', '-f', dockerfile.name, '-t', tag, - *(('--no-cache',) if options.no_cache else ()), '.'], - check=True) - subprocess.run(['docker', 'push', tag], check=True) + print("\033[33;1mrebuilding \033[35;1m{}\033[0m".format(tag)) + run_or_report('docker', 'build', '--pull', '-f', dockerfile.name, '-t', tag, + *(('--no-cache',) if options.no_cache else ()), '.') + print("\033[33;1mpushing \033[35;1m{}\033[0m".format(tag)) + run_or_report('docker', 'push', tag) + print("\033[32;1mFinished \033[35;1m{}\033[0m".format(tag)) latest = tag_base + ':latest' + global manifests if latest in manifests: manifests[latest].append(tag) else: manifests[latest] = [tag] +def base_distro_build(distro, arch): + tag = '{r}{distro[0]}-{distro[1]}-base'.format(r=registry_base, distro=distro) + codename = 'latest' if distro == ('ubuntu', 'lts') else distro[1] + build_tag(tag, arch, """ +FROM {}/{}:{} +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y + """.format(arch, distro[0], codename)) + + def distro_build(distro, arch): prefix = '{r}{distro[0]}-{distro[1]}'.format(r=registry_base, distro=distro) fmtargs = dict(arch=arch, distro=distro, prefix=prefix) # (distro)-(codename)-base: Base image from upstream: we sync the repos, but do nothing else. - build_tag(prefix + '-base', arch, """ -FROM {arch}/{distro[0]}:{codename} -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y -""".format(**fmtargs, codename='latest' if distro == ['ubuntu', 'lts'] else distro[1])) + if (distro, arch) != (('debian', 'stable'), 'amd64'): # debian-stable-base/amd64 already built + base_distro_build(distro, arch) # (distro)-(codename)-builder: Deb builder image used for building debs; we add the basic tools # we use to build debs, not including things that should come from the dependencies in the @@ -124,48 +159,11 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ """.format(**fmtargs)) -for d in distros: - for a in arches(distros): - distro_build(d, a) - - -# Other images: - -# lint is a tiny build with just formatting checking tools - -build_tag(registry_base + 'lint', 'amd64', """ -FROM {r}debian-stable-base -RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ - clang-format-11 \ - eatmydata \ - git \ - jsonnet -""".format(r=registry_base)) - -# nodejs -build_tag(registry_base + 'nodejs', 'amd64', """ -FROM node:14.16.1 -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ - ccache \ - cmake \ - eatmydata \ - g++ \ - gdb \ - git \ - make \ - ninja-build \ - openssh-client \ - patch \ - pkg-config \ - wine -""") - - -# Android builds on debian-stable-base and adds a ton of android crap: -build_tag(registry_base + 'android', 'amd64', """ +# Android and flutter builds on top of debian-stable-base and adds a ton of android crap; we +# schedule this job as soon as the debian-sid-base/amd64 build finishes, because they easily take +# the longest and are by far the biggest images. +def android_builds(): + build_tag(registry_base + 'android', 'amd64', """ FROM {r}debian-stable-base RUN /bin/bash -c 'sed -i "s/main/main contrib/g" /etc/apt/sources.list' RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ @@ -191,9 +189,7 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ && rm -rf /tmp/android-sdk-licenses """.format(r=registry_base)) - -# Flutter image takes android and adds even more crap: -build_tag(registry_base + 'flutter', 'amd64', """ + build_tag(registry_base + 'flutter', 'amd64', """ FROM {r}android RUN cd /opt \ && curl https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_2.2.2-stable.tar.xz \ @@ -203,47 +199,63 @@ RUN cd /opt \ """.format(r=registry_base)) -# debian-sid-clang adds clang + libc++ to debian-sid (amd64 only) -build_tag(registry_base + 'debian-sid-clang', 'amd64', """ -FROM {r}debian-sid -RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ - && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ - clang-13 \ - libc++-13-dev \ - libc++abi-13-dev \ - lld-13 +# lint is a tiny build (on top of debian-stable-base) with just formatting checking tools +def lint_build(): + build_tag(registry_base + 'lint', 'amd64', """ +FROM {r}debian-stable-base +RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + clang-format-11 \ + eatmydata \ + git \ + jsonnet """.format(r=registry_base)) -# debian-win32-cross is debian-testing-base + stuff for compiling windows binaries -build_tag(registry_base + 'debian-win32-cross', 'amd64', """ -FROM {r}debian-testing-base +def nodejs_build(): + build_tag(registry_base + 'nodejs', 'amd64', """ +FROM node:14.16.1 +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ - autoconf \ - automake \ - build-essential \ ccache \ cmake \ eatmydata \ - file \ - g++-mingw-w64-x86-64-posix \ + g++ \ + gdb \ git \ - gperf \ - libtool \ make \ ninja-build \ - nsis \ openssh-client \ patch \ pkg-config \ - qttools5-dev \ - zip \ - && update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix \ - && update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix -""".format(r=registry_base)) + wine +""") + + +# Start debian-stable-base/amd64 on its own, because other builds depend on it and we want to get +# those (especially android/flutter) fired off as soon as possible (because it's slow and huge). +base_distro_build(['debian', 'stable'], 'amd64') + +executor = ThreadPoolExecutor(max_workers=max(options.parallel, 1)) + +jobs = [executor.submit(b) for b in (android_builds, lint_build, nodejs_build)] + +for d in distros: + for a in arches(distros): + jobs.append(executor.submit(distro_build, d, a)) +while len(jobs): + j = jobs.pop(0) + try: + j.result() + except (ChildProcessError, subprocess.CalledProcessError): + for k in jobs: + k.cancel() + + +if failure: + print("Error(s) occured, aborting!", file=sys.stderr) + sys.exit(1) for latest, tags in manifests.items(): @@ -251,5 +263,5 @@ for latest, tags in manifests.items(): latest, ', '.join(tags))) subprocess.run(['docker', 'manifest', 'rm', latest], stderr=subprocess.DEVNULL, check=False) - subprocess.run(['docker', 'manifest', 'create', latest, *tags], check=True) - subprocess.run(['docker', 'manifest', 'push', latest], check=True) + run_or_report('docker', 'manifest', 'create', latest, *tags) + run_or_report('docker', 'manifest', 'push', latest) From 09e97eef77a2def2218fc7a067a17eb9fae6f11b Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 26 Oct 2021 14:57:58 -0300 Subject: [PATCH 10/53] Make it snazzy Shows statuses that "fill in" when run interactively. --- contrib/ci/docker/rebuild-docker-images.py | 79 +++++++++++++++++----- 1 file changed, 63 insertions(+), 16 deletions(-) diff --git a/contrib/ci/docker/rebuild-docker-images.py b/contrib/ci/docker/rebuild-docker-images.py index 59a549aa3..2d6a88bc8 100755 --- a/contrib/ci/docker/rebuild-docker-images.py +++ b/contrib/ci/docker/rebuild-docker-images.py @@ -5,6 +5,7 @@ import tempfile import optparse import sys from concurrent.futures import ThreadPoolExecutor +import threading parser = optparse.OptionParser() parser.add_option("--no-cache", action="store_true", @@ -19,6 +20,7 @@ distros = [*(('debian', x) for x in ('sid', 'stable', 'testing', 'bullseye', 'bu *(('ubuntu', x) for x in ('rolling', 'lts', 'impish', 'hirsute', 'focal', 'bionic'))] manifests = {} # "image:latest": ["image/amd64", "image/arm64v8", ...] +manifestlock = threading.Lock() def arches(distro): @@ -30,45 +32,65 @@ def arches(distro): failure = False +lineno = 0 +linelock = threading.Lock() -def run_or_report(*args): + +def print_line(myline, value): + linelock.acquire() + global lineno + if myline != lineno and sys.__stdout__.isatty(): + jump = lineno - myline + print("\033[{jump}A\r\033[K{value}\033[{jump}B\r".format(jump=jump, value=value), end='') + sys.stdout.flush() + else: + print(value) + lineno += 1 + linelock.release() + + +def run_or_report(*args, myline): try: subprocess.run( args, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf8') except subprocess.CalledProcessError as e: with tempfile.NamedTemporaryFile(suffix=".log", delete=False) as log: + log.write("Error running {}: {}\n\nOutput:\n\n".format(' '.join(args), e).encode()) log.write(e.output.encode()) global failure failure = True - print(""" -\033[31;1mAn error occured ({}) running: - {} -See {} for the command log.\033[0m -""".format(e, ' '.join(args), log.name), file=sys.stderr) + print_line(myline, "\033[31;1mError! See {} for details", log.name) raise e def build_tag(tag_base, arch, contents): if failure: raise ChildProcessError() + + linelock.acquire() + myline = lineno + linelock.release() + with tempfile.NamedTemporaryFile() as dockerfile: dockerfile.write(contents.encode()) dockerfile.flush() tag = '{}/{}'.format(tag_base, arch) - print("\033[33;1mrebuilding \033[35;1m{}\033[0m".format(tag)) + print_line(myline, "\033[33;1mRebuilding \033[35;1m{}\033[0m".format(tag)) run_or_report('docker', 'build', '--pull', '-f', dockerfile.name, '-t', tag, - *(('--no-cache',) if options.no_cache else ()), '.') - print("\033[33;1mpushing \033[35;1m{}\033[0m".format(tag)) - run_or_report('docker', 'push', tag) - print("\033[32;1mFinished \033[35;1m{}\033[0m".format(tag)) + *(('--no-cache',) if options.no_cache else ()), '.', myline=myline) + print_line(myline, "\033[33;1mPushing \033[35;1m{}\033[0m".format(tag)) + run_or_report('docker', 'push', tag, myline=myline) + print_line(myline, "\033[32;1mFinished build \033[35;1m{}\033[0m".format(tag)) latest = tag_base + ':latest' global manifests + manifestlock.acquire() if latest in manifests: manifests[latest].append(tag) else: manifests[latest] = [tag] + manifestlock.release() def base_distro_build(distro, arch): @@ -258,10 +280,35 @@ if failure: sys.exit(1) -for latest, tags in manifests.items(): - print("\033[32;1mpushing new manifest for \033[33;1m{}[\033[35;1m{}\033[33;1m]\033[0m".format( - latest, ', '.join(tags))) +print("\n\n\033[32;1mAll builds finished successfully; pushing manifests...\033[0m\n") + + +def push_manifest(latest, tags): + if failure: + raise ChildProcessError() + + linelock.acquire() + myline = lineno + linelock.release() subprocess.run(['docker', 'manifest', 'rm', latest], stderr=subprocess.DEVNULL, check=False) - run_or_report('docker', 'manifest', 'create', latest, *tags) - run_or_report('docker', 'manifest', 'push', latest) + print_line(myline, "\033[33;1mCreating manifest \033[35;1m{}\033[0m".format(latest)) + run_or_report('docker', 'manifest', 'create', latest, *tags, myline=myline) + print_line(myline, "\033[33;1mPushing manifest \033[35;1m{}\033[0m".format(latest)) + run_or_report('docker', 'manifest', 'push', latest, myline=myline) + print_line(myline, "\033[32;1mFinished manifest \033[35;1m{}\033[0m".format(latest)) + + +for latest, tags in manifests.items(): + jobs.append(executor.submit(push_manifest, latest, tags)) + +while len(jobs): + j = jobs.pop(0) + try: + j.result() + except (ChildProcessError, subprocess.CalledProcessError): + for k in jobs: + k.cancel() + + +print("\n\n\033[32;1mAll done!\n") From 0572713533374fca36cab588d631b66967dc11cb Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 26 Oct 2021 15:10:29 -0300 Subject: [PATCH 11/53] Fix i386 builds --- contrib/ci/docker/rebuild-docker-images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/ci/docker/rebuild-docker-images.py b/contrib/ci/docker/rebuild-docker-images.py index 2d6a88bc8..149d70a49 100755 --- a/contrib/ci/docker/rebuild-docker-images.py +++ b/contrib/ci/docker/rebuild-docker-images.py @@ -264,7 +264,7 @@ executor = ThreadPoolExecutor(max_workers=max(options.parallel, 1)) jobs = [executor.submit(b) for b in (android_builds, lint_build, nodejs_build)] for d in distros: - for a in arches(distros): + for a in arches(d): jobs.append(executor.submit(distro_build, d, a)) while len(jobs): j = jobs.pop(0) From 0b4c56dff513010e3cc0988f7b2c7db6657dd5f0 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 26 Oct 2021 17:38:17 -0300 Subject: [PATCH 12/53] Add hacks; add --distro to rebuild just one Bionic is broken and needs a hack to create some dumb pgm include directory. Add --distro cli argument so that you can rebuild just one distro. --- contrib/ci/docker/rebuild-docker-images.py | 46 ++++++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/contrib/ci/docker/rebuild-docker-images.py b/contrib/ci/docker/rebuild-docker-images.py index 149d70a49..847265f52 100755 --- a/contrib/ci/docker/rebuild-docker-images.py +++ b/contrib/ci/docker/rebuild-docker-images.py @@ -12,6 +12,9 @@ parser.add_option("--no-cache", action="store_true", help="Run `docker build` with the `--no-cache` option to ignore existing images") parser.add_option("--parallel", "-j", type="int", default=1, help="Run up to this many builds in parallel") +parser.add_option("--distro", type="string", default="", + help="Build only this distro; should be DISTRO-CODE or DISTRO-CODE/ARCH, " + "e.g. debian-sid/amd64") (options, args) = parser.parse_args() registry_base = 'registry.oxen.rocks/lokinet-ci-' @@ -19,17 +22,37 @@ registry_base = 'registry.oxen.rocks/lokinet-ci-' distros = [*(('debian', x) for x in ('sid', 'stable', 'testing', 'bullseye', 'buster')), *(('ubuntu', x) for x in ('rolling', 'lts', 'impish', 'hirsute', 'focal', 'bionic'))] +if options.distro: + d = options.distro.split('-') + if len(d) != 2 or d[0] not in ('debian', 'ubuntu') or not d[1]: + print("Bad --distro value '{}'".format(options.distro), file=sys.stderr) + sys.exit(1) + distros = [(d[0], d[1].split('/')[0])] + + manifests = {} # "image:latest": ["image/amd64", "image/arm64v8", ...] manifestlock = threading.Lock() def arches(distro): + if options.distro and '/' in options.distro: + arch = options.distro.split('/') + if arch not in ('amd64', 'i386', 'arm64v8', 'arm32v7'): + print("Bad --distro value '{}'".format(options.distro), file=sys.stderr) + sys.exit(1) + return [arch] + a = ['amd64', 'arm64v8', 'arm32v7'] if distro[0] == 'debian' or distro == ('ubuntu', 'bionic'): a.append('i386') # i386 builds don't work on later ubuntu return a +hacks = { + registry_base + 'ubuntu-bionic-builder': '&& mkdir -p /usr/lib/x86_64-linux-gnu/pgm-5.2/include' +} + + failure = False lineno = 0 @@ -101,8 +124,9 @@ FROM {}/{}:{} RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ - && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y - """.format(arch, distro[0], codename)) + && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y \ + {hacks} +""".format(arch, distro[0], codename, hacks=hacks.get(tag, ''))) def distro_build(distro, arch): @@ -127,8 +151,9 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ g++ \ git \ git-buildpackage \ - openssh-client -""".format(**fmtargs)) + openssh-client \ + {hacks} +""".format(**fmtargs, hacks=hacks.get(prefix + '-builder', ''))) # (distro)-(codename): Basic image we use for most builds. This takes the -builder and adds # most dependencies found in our packages. @@ -177,8 +202,9 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ python3-pybind11 \ python3-pytest \ python3-setuptools \ - qttools5-dev -""".format(**fmtargs)) + qttools5-dev \ + {hacks} +""".format(**fmtargs, hacks=hacks.get(prefix, ''))) # Android and flutter builds on top of debian-stable-base and adds a ton of android crap; we @@ -257,11 +283,15 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ # Start debian-stable-base/amd64 on its own, because other builds depend on it and we want to get # those (especially android/flutter) fired off as soon as possible (because it's slow and huge). -base_distro_build(['debian', 'stable'], 'amd64') +if ('debian', 'stable') in distros: + base_distro_build(['debian', 'stable'], 'amd64') executor = ThreadPoolExecutor(max_workers=max(options.parallel, 1)) -jobs = [executor.submit(b) for b in (android_builds, lint_build, nodejs_build)] +if options.distro: + jobs = [] +else: + jobs = [executor.submit(b) for b in (android_builds, lint_build, nodejs_build)] for d in distros: for a in arches(d): From cd49031bd2e92510de85f45bfbdd38fe406b38b6 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 28 Oct 2021 13:49:20 -0300 Subject: [PATCH 13/53] Fix race condition in interactive output --- contrib/ci/docker/rebuild-docker-images.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/contrib/ci/docker/rebuild-docker-images.py b/contrib/ci/docker/rebuild-docker-images.py index 847265f52..32aeb9d7d 100755 --- a/contrib/ci/docker/rebuild-docker-images.py +++ b/contrib/ci/docker/rebuild-docker-images.py @@ -62,13 +62,12 @@ linelock = threading.Lock() def print_line(myline, value): linelock.acquire() global lineno - if myline != lineno and sys.__stdout__.isatty(): + if sys.__stdout__.isatty(): jump = lineno - myline print("\033[{jump}A\r\033[K{value}\033[{jump}B\r".format(jump=jump, value=value), end='') sys.stdout.flush() else: print(value) - lineno += 1 linelock.release() @@ -91,7 +90,10 @@ def build_tag(tag_base, arch, contents): raise ChildProcessError() linelock.acquire() + global lineno myline = lineno + lineno += 1 + print() linelock.release() with tempfile.NamedTemporaryFile() as dockerfile: @@ -318,7 +320,10 @@ def push_manifest(latest, tags): raise ChildProcessError() linelock.acquire() + global lineno myline = lineno + lineno += 1 + print() linelock.release() subprocess.run(['docker', 'manifest', 'rm', latest], stderr=subprocess.DEVNULL, check=False) From e6622c1d4681eae44e32b4c9d5bca7252081e85b Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 28 Oct 2021 13:49:37 -0300 Subject: [PATCH 14/53] Install g++-8 on bionic --- contrib/ci/docker/rebuild-docker-images.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/ci/docker/rebuild-docker-images.py b/contrib/ci/docker/rebuild-docker-images.py index 32aeb9d7d..916fa2ee2 100755 --- a/contrib/ci/docker/rebuild-docker-images.py +++ b/contrib/ci/docker/rebuild-docker-images.py @@ -49,7 +49,8 @@ def arches(distro): hacks = { - registry_base + 'ubuntu-bionic-builder': '&& mkdir -p /usr/lib/x86_64-linux-gnu/pgm-5.2/include' + registry_base + 'ubuntu-bionic-builder': """g++-8 \ + && mkdir -p /usr/lib/x86_64-linux-gnu/pgm-5.2/include""", } From fcba709fcb663277b6b8578b58133ab0f444eaa1 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Fri, 29 Oct 2021 10:45:02 -0400 Subject: [PATCH 15/53] do not requeue nodes for testing from failing queue if we do not have them marked as failing anymore --- llarp/consensus/reachability_testing.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llarp/consensus/reachability_testing.cpp b/llarp/consensus/reachability_testing.cpp index 99190bc4f..36281d4f7 100644 --- a/llarp/consensus/reachability_testing.cpp +++ b/llarp/consensus/reachability_testing.cpp @@ -124,7 +124,8 @@ namespace llarp::consensus auto& [pk, retest_time, failures] = failing_queue.top(); if (retest_time > now) break; - result.emplace_back(pk, failures); + if (failing.count(pk)) + result.emplace_back(pk, failures); failing_queue.pop(); } return result; From 0ec50e6624d664f25110a0a02066c2317c62d8d4 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 29 Oct 2021 11:52:54 -0300 Subject: [PATCH 16/53] Fix comment typo --- llarp/consensus/reachability_testing.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llarp/consensus/reachability_testing.cpp b/llarp/consensus/reachability_testing.cpp index 99190bc4f..8a667492d 100644 --- a/llarp/consensus/reachability_testing.cpp +++ b/llarp/consensus/reachability_testing.cpp @@ -109,7 +109,7 @@ namespace llarp::consensus std::shuffle(testing_queue.begin(), testing_queue.end(), rng); - // Recurse with the rebuild list, but don't let it try rebuilding again + // Recurse with the rebuilt list, but don't let it try rebuilding again return next_random(router, now, false); } From 22d4b88edc1f3dfaacac58f7b9173d59e257ed6a Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 9 Nov 2021 11:20:53 -0500 Subject: [PATCH 17/53] make PumpLL idempotent to reduce cpu use a bit --- llarp/router/router.cpp | 19 +++++++++++-------- llarp/router/router.hpp | 2 ++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index e7eae19a1..2537ded55 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -69,6 +69,16 @@ namespace llarp _running.store(false); _lastTick = llarp::time_now_ms(); m_NextExploreAt = Clock_t::now(); + m_Pump = _loop->make_waker([this]() { + llarp::LogTrace("Router::PumpLL() start"); + if (_stopping.load()) + return; + paths.PumpDownstream(); + paths.PumpUpstream(); + _outboundMessageHandler.Tick(); + _linkManager.PumpLinks(); + llarp::LogTrace("Router::PumpLL() end"); + }); } Router::~Router() @@ -245,14 +255,7 @@ namespace llarp void Router::PumpLL() { - llarp::LogTrace("Router::PumpLL() start"); - if (_stopping.load()) - return; - paths.PumpDownstream(); - paths.PumpUpstream(); - _outboundMessageHandler.Tick(); - _linkManager.PumpLinks(); - llarp::LogTrace("Router::PumpLL() end"); + m_Pump->Trigger(); } bool diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index 946bc8594..36bbbd2a4 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -76,6 +76,8 @@ namespace llarp path::BuildLimiter m_PathBuildLimiter; + std::shared_ptr m_Pump; + path::BuildLimiter& pathBuildLimiter() override { From 8744c939448529f1724fd2c43f48ba8445e5d721 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 9 Nov 2021 12:21:07 -0500 Subject: [PATCH 18/53] prevent 100% cpu usage on service nodes call_soon wakes up the mainloop and is often reentrant, we dont want to ever way up the event loop but we also want to always defer the call so we always use the workers --- llarp/router/router.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 2537ded55..da607fb27 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -1467,10 +1467,7 @@ namespace llarp void Router::QueueWork(std::function func) { - if (m_isServiceNode) - _loop->call_soon(std::move(func)); - else - m_lmq->job(std::move(func)); + m_lmq->job(std::move(func)); } void From bb86996acfacc27599a4294dcf937ea34dbe38c0 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 9 Nov 2021 12:44:35 -0500 Subject: [PATCH 19/53] make event loop pump function non idempotent --- llarp/router/router.cpp | 28 ++++++++++++++++------------ llarp/router/router.hpp | 3 +++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index da607fb27..cfb43522d 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -69,16 +69,7 @@ namespace llarp _running.store(false); _lastTick = llarp::time_now_ms(); m_NextExploreAt = Clock_t::now(); - m_Pump = _loop->make_waker([this]() { - llarp::LogTrace("Router::PumpLL() start"); - if (_stopping.load()) - return; - paths.PumpDownstream(); - paths.PumpUpstream(); - _outboundMessageHandler.Tick(); - _linkManager.PumpLinks(); - llarp::LogTrace("Router::PumpLL() end"); - }); + m_Pump = _loop->make_waker([this]() { PumpLLNonIdempotent(); }); } Router::~Router() @@ -86,6 +77,19 @@ namespace llarp llarp_dht_context_free(_dht); } + void + Router::PumpLLNonIdempotent() + { + llarp::LogTrace("Router::PumpLL() start"); + if (_stopping.load()) + return; + paths.PumpDownstream(); + paths.PumpUpstream(); + _outboundMessageHandler.Tick(); + _linkManager.PumpLinks(); + llarp::LogTrace("Router::PumpLL() end"); + } + util::StatusObject Router::ExtractStatus() const { @@ -1241,9 +1245,9 @@ namespace llarp #ifdef _WIN32 // windows uses proactor event loop so we need to constantly pump - _loop->add_ticker([this] { PumpLL(); }); + _loop->add_ticker([this] { PumpLLNonIdempotent(); }); #else - _loop->set_pump_function([this] { PumpLL(); }); + _loop->set_pump_function([this] { PumpLLNonIdempotent(); }); #endif _loop->call_every(ROUTER_TICK_INTERVAL, weak_from_this(), [this] { Tick(); }); _running.store(true); diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index 36bbbd2a4..f444bb3f1 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -286,6 +286,9 @@ namespace llarp void PumpLL() override; + void + PumpLLNonIdempotent(); + const oxenmq::address DefaultRPCBindAddr = oxenmq::address::tcp("127.0.0.1", 1190); bool enableRPCServer = false; oxenmq::address rpcBindAddr = DefaultRPCBindAddr; From 8a9025e234fb61ed2ff1571f2459d2c04f22429b Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 9 Nov 2021 13:02:47 -0500 Subject: [PATCH 20/53] only pump path context in idempotent pumpll --- llarp/router/router.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index cfb43522d..1fd0e44f6 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -69,7 +69,11 @@ namespace llarp _running.store(false); _lastTick = llarp::time_now_ms(); m_NextExploreAt = Clock_t::now(); - m_Pump = _loop->make_waker([this]() { PumpLLNonIdempotent(); }); + m_Pump = _loop->make_waker([this]() { + paths.PumpDownstream(); + paths.PumpUpstream(); + PumpLLNonIdempotent(); + }); } Router::~Router() @@ -83,8 +87,6 @@ namespace llarp llarp::LogTrace("Router::PumpLL() start"); if (_stopping.load()) return; - paths.PumpDownstream(); - paths.PumpUpstream(); _outboundMessageHandler.Tick(); _linkManager.PumpLinks(); llarp::LogTrace("Router::PumpLL() end"); From faf95cbd0af86f14552e3f98e893e24a625108cf Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 9 Nov 2021 19:39:56 -0400 Subject: [PATCH 21/53] Allow a nullptr callback for event loop PumpLL --- llarp/ev/ev_libuv.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/llarp/ev/ev_libuv.cpp b/llarp/ev/ev_libuv.cpp index 8f38750c1..04b447987 100644 --- a/llarp/ev/ev_libuv.cpp +++ b/llarp/ev/ev_libuv.cpp @@ -111,13 +111,14 @@ namespace llarp::uv { llarp::LogTrace("ticking event loop."); FlushLogic(); - PumpLL(); + if (PumpLL) + PumpLL(); auto& log = llarp::LogContext::Instance(); if (log.logStream) log.logStream->Tick(time_now()); } - Loop::Loop(size_t queue_size) : llarp::EventLoop{}, PumpLL{[] {}}, m_LogicCalls{queue_size} + Loop::Loop(size_t queue_size) : llarp::EventLoop{}, PumpLL{nullptr}, m_LogicCalls{queue_size} { if (!(m_Impl = uvw::Loop::create())) throw std::runtime_error{"Failed to construct libuv loop"}; From 633431be66e91c4cffc8ffdf92ac1c87682e03a5 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 9 Nov 2021 19:46:09 -0400 Subject: [PATCH 22/53] Make outbound message queue PumpLL again if it doesn't send all --- llarp/router/outbound_message_handler.cpp | 51 +++++++++++------------ llarp/router/outbound_message_handler.hpp | 16 +++---- llarp/router/router.cpp | 2 +- 3 files changed, 33 insertions(+), 36 deletions(-) diff --git a/llarp/router/outbound_message_handler.cpp b/llarp/router/outbound_message_handler.cpp index ac8fda64f..22fcd959f 100644 --- a/llarp/router/outbound_message_handler.cpp +++ b/llarp/router/outbound_message_handler.cpp @@ -1,9 +1,7 @@ #include "outbound_message_handler.hpp" #include -#include "i_outbound_session_maker.hpp" -#include "i_rc_lookup_handler.hpp" -#include +#include "router.hpp" #include #include #include @@ -26,7 +24,8 @@ namespace llarp const RouterID& remote, const ILinkMessage& msg, SendStatusHandler callback) { // if the destination is invalid, callback with failure and return - if (not _linkManager->SessionIsClient(remote) and not _lookupHandler->SessionIsAllowed(remote)) + if (not _router->linkManager().SessionIsClient(remote) + and not _router->rcLookupHandler().SessionIsAllowed(remote)) { DoCallback(callback, SendStatus::InvalidRouter); return true; @@ -47,7 +46,7 @@ namespace llarp std::copy_n(buf.base, buf.sz, message.first.data()); // if we have a session to the destination, queue the message and return - if (_linkManager->HasSessionTo(remote)) + if (_router->linkManager().HasSessionTo(remote)) { QueueOutboundMessage(remote, std::move(message), msg.pathid, priority); return true; @@ -87,7 +86,8 @@ namespace llarp m_Killer.TryAccess([this]() { recentlyRemovedPaths.Decay(); ProcessOutboundQueue(); - SendRoundRobin(); + if (/*bool more = */ SendRoundRobin()) + _router->PumpLL(); }); } @@ -127,13 +127,9 @@ namespace llarp } void - OutboundMessageHandler::Init( - ILinkManager* linkManager, I_RCLookupHandler* lookupHandler, EventLoop_ptr loop) + OutboundMessageHandler::Init(AbstractRouter* router) { - _linkManager = linkManager; - _lookupHandler = lookupHandler; - _loop = std::move(loop); - + _router = router; outboundMessageQueues.emplace(zeroID, MessageQueue()); } @@ -168,14 +164,14 @@ namespace llarp OutboundMessageHandler::DoCallback(SendStatusHandler callback, SendStatus status) { if (callback) - _loop->call([f = std::move(callback), status] { f(status); }); + _router->loop()->call([f = std::move(callback), status] { f(status); }); } void OutboundMessageHandler::QueueSessionCreation(const RouterID& remote) { auto fn = util::memFn(&OutboundMessageHandler::OnSessionResult, this); - _linkManager->GetSessionMaker()->CreateSessionTo(remote, fn); + _router->linkManager().GetSessionMaker()->CreateSessionTo(remote, fn); } bool @@ -199,7 +195,7 @@ namespace llarp const llarp_buffer_t buf(msg.first); auto callback = msg.second; m_queueStats.sent++; - return _linkManager->SendTo(remote, buf, [=](ILinkSession::DeliveryStatus status) { + return _router->linkManager().SendTo(remote, buf, [=](ILinkSession::DeliveryStatus status) { if (status == ILinkSession::DeliveryStatus::eDeliverySuccess) DoCallback(callback, SendStatus::Success); else @@ -212,7 +208,7 @@ namespace llarp bool OutboundMessageHandler::SendIfSession(const RouterID& remote, const Message& msg) { - if (_linkManager->HasSessionTo(remote)) + if (_router->linkManager().HasSessionTo(remote)) { return Send(remote, msg); } @@ -282,7 +278,7 @@ namespace llarp } } - void + bool OutboundMessageHandler::SendRoundRobin() { m_queueStats.numTicks++; @@ -296,7 +292,6 @@ namespace llarp routing_mq.pop(); } - size_t empty_count = 0; size_t num_queues = roundRobinOrder.size(); // if any paths have been removed since last tick, remove any stale @@ -317,16 +312,16 @@ namespace llarp removedSomePaths = false; num_queues = roundRobinOrder.size(); - size_t sent_count = 0; - if (num_queues == 0) // if no queues, return + if (num_queues == 0) { - return; + return false; } // send messages for each pathid in roundRobinOrder, stopping when // either every path's queue is empty or a set maximum amount of // messages have been sent. - while (sent_count < MAX_OUTBOUND_MESSAGES_PER_TICK) + size_t consecutive_empty = 0; + for (size_t sent_count = 0; sent_count < MAX_OUTBOUND_MESSAGES_PER_TICK;) { PathID_t pathid = std::move(roundRobinOrder.front()); roundRobinOrder.pop(); @@ -339,24 +334,26 @@ namespace llarp Send(entry.router, entry.message); message_queue.pop(); - empty_count = 0; - sent_count++; + consecutive_empty = 0; + consecutive_empty++; } else { - empty_count++; + consecutive_empty++; } roundRobinOrder.push(std::move(pathid)); // if num_queues empty queues in a row, all queues empty. - if (empty_count == num_queues) + if (consecutive_empty == num_queues) { break; } } - m_queueStats.perTickMax = std::max((uint32_t)sent_count, m_queueStats.perTickMax); + m_queueStats.perTickMax = std::max((uint32_t)consecutive_empty, m_queueStats.perTickMax); + + return consecutive_empty != num_queues; } void diff --git a/llarp/router/outbound_message_handler.hpp b/llarp/router/outbound_message_handler.hpp index be4d151eb..e0402d633 100644 --- a/llarp/router/outbound_message_handler.hpp +++ b/llarp/router/outbound_message_handler.hpp @@ -17,8 +17,7 @@ struct llarp_buffer_t; namespace llarp { - struct ILinkManager; - struct I_RCLookupHandler; + struct AbstractRouter; enum class SessionResult; struct OutboundMessageHandler final : public IOutboundMessageHandler @@ -72,7 +71,7 @@ namespace llarp ExtractStatus() const override; void - Init(ILinkManager* linkManager, I_RCLookupHandler* lookupHandler, EventLoop_ptr loop); + Init(AbstractRouter* router); private: using Message = std::pair, SendStatusHandler>; @@ -160,14 +159,17 @@ namespace llarp ProcessOutboundQueue(); /* - * Sends all routing messages that have been queued, indicated by pathid 0 when queued. + * Sends routing messages that have been queued, indicated by pathid 0 when queued. * * Sends messages from path queues until all are empty or a set cap has been reached. * This will send one message from each queue in a round-robin fashion such that they * all have roughly equal access to bandwidth. A notion of priority may be introduced * at a later time, but for now only routing messages get priority. + * + * Returns true if there is more to send (i.e. we hit the limit before emptying all path + * queues), false if all queues were drained. */ - void + bool SendRoundRobin(); /* Invoked when an outbound session establish attempt has concluded. @@ -193,9 +195,7 @@ namespace llarp std::queue roundRobinOrder; - ILinkManager* _linkManager; - I_RCLookupHandler* _lookupHandler; - EventLoop_ptr _loop; + AbstractRouter* _router; util::ContentionKiller m_Killer; diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 1fd0e44f6..2739c85d4 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -661,7 +661,7 @@ namespace llarp LogInfo("Loaded ", bootstrapRCList.size(), " bootstrap routers"); // Init components after relevant config settings loaded - _outboundMessageHandler.Init(&_linkManager, &_rcLookupHandler, _loop); + _outboundMessageHandler.Init(this); _outboundSessionMaker.Init( this, &_linkManager, From cf5c4e72842bef2139d3d709cd1703d24cc81d83 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 9 Nov 2021 19:47:29 -0400 Subject: [PATCH 23/53] outbound_message_handler: skip instead of aborting on removed paths --- llarp/router/outbound_message_handler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llarp/router/outbound_message_handler.cpp b/llarp/router/outbound_message_handler.cpp index 22fcd959f..2aaa6a30e 100644 --- a/llarp/router/outbound_message_handler.cpp +++ b/llarp/router/outbound_message_handler.cpp @@ -254,7 +254,7 @@ namespace llarp // so check here if the pathid was recently removed. if (recentlyRemovedPaths.Contains(entry.pathid)) { - return; + continue; } auto [queue_itr, is_new] = outboundMessageQueues.emplace(entry.pathid, MessageQueue()); From 41807f1763a7722e17d99444f6efb15e9aa79b92 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 9 Nov 2021 19:48:41 -0400 Subject: [PATCH 24/53] transit hop: drop instead of flushing when full If full happens we are going to trigger a bunch of flushes which probably isn't very useful, so drop instead. --- llarp/path/transit_hop.cpp | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/llarp/path/transit_hop.cpp b/llarp/path/transit_hop.cpp index 09cd6059c..2e09a6f74 100644 --- a/llarp/path/transit_hop.cpp +++ b/llarp/path/transit_hop.cpp @@ -142,14 +142,6 @@ namespace llarp void TransitHop::UpstreamWork(TrafficQueue_ptr msgs, AbstractRouter* r) { - auto flushIt = [self = shared_from_this(), r]() { - std::vector msgs; - while (auto maybe = self->m_UpstreamGather.tryPopFront()) - { - msgs.push_back(*maybe); - } - self->HandleAllUpstream(std::move(msgs), r); - }; for (auto& ev : *msgs) { const llarp_buffer_t buf(ev.first); @@ -158,14 +150,19 @@ namespace llarp msg.pathid = info.txID; msg.Y = ev.second ^ nonceXOR; msg.X = buf; - if (m_UpstreamGather.full()) + if (m_UpstreamGather.tryPushBack(msg) != thread::QueueReturn::Success) + break; + } + + // Flush it: + r->loop()->call([self = shared_from_this(), r] { + std::vector msgs; + while (auto maybe = self->m_UpstreamGather.tryPopFront()) { - r->loop()->call(flushIt); + msgs.push_back(*maybe); } - if (m_UpstreamGather.enabled()) - m_UpstreamGather.pushBack(msg); - } - r->loop()->call(flushIt); + self->HandleAllUpstream(std::move(msgs), r); + }); } void From bfc6d35b33c97f140aaf914c6beb35263cd28982 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 9 Nov 2021 21:03:36 -0400 Subject: [PATCH 25/53] Simplifications & C++17 - Modernize some iterator loops - Simplify a couple places by using `if (init; ...)` - Replace various std::binds with lambdas --- llarp/crypto/encrypted_frame.hpp | 2 +- llarp/dht/explorenetworkjob.cpp | 3 +- llarp/handlers/exit.cpp | 87 ++++++++++-------------------- llarp/handlers/exit.hpp | 8 ++- llarp/hook/shell.cpp | 2 +- llarp/messages/relay_status.cpp | 6 +-- llarp/router/rc_lookup_handler.cpp | 5 +- 7 files changed, 39 insertions(+), 74 deletions(-) diff --git a/llarp/crypto/encrypted_frame.hpp b/llarp/crypto/encrypted_frame.hpp index d10d799a3..7731d74e5 100644 --- a/llarp/crypto/encrypted_frame.hpp +++ b/llarp/crypto/encrypted_frame.hpp @@ -80,7 +80,7 @@ namespace llarp AsyncDecrypt(const EncryptedFrame& frame, User_ptr u, WorkerFunction_t worker) { target = frame; - worker(std::bind(&AsyncFrameDecrypter::Decrypt, this, std::move(u))); + worker([this, u = std::move(u)]() mutable { Decrypt(std::move(u)); }); } }; } // namespace llarp diff --git a/llarp/dht/explorenetworkjob.cpp b/llarp/dht/explorenetworkjob.cpp index 379f2d4b4..55c35a33c 100644 --- a/llarp/dht/explorenetworkjob.cpp +++ b/llarp/dht/explorenetworkjob.cpp @@ -30,14 +30,13 @@ namespace llarp llarp::LogDebug("got ", valuesFound.size(), " routers from exploration"); auto router = parent->GetRouter(); - using std::placeholders::_1; for (const auto& pk : valuesFound) { // lookup router if (router and router->nodedb()->Has(pk)) continue; parent->LookupRouter( - pk, std::bind(&AbstractRouter::HandleDHTLookupForExplore, router, pk, _1)); + pk, [router, pk](const auto& res) { router->HandleDHTLookupForExplore(pk, res); }); } } } // namespace dht diff --git a/llarp/handlers/exit.cpp b/llarp/handlers/exit.cpp index f775bcd3e..7c3d28a21 100644 --- a/llarp/handlers/exit.cpp +++ b/llarp/handlers/exit.cpp @@ -92,8 +92,7 @@ namespace llarp } return std::nullopt; } - else - return std::nullopt; + return std::nullopt; } const EventLoop_ptr& @@ -112,16 +111,13 @@ namespace llarp return false; if (auto* rid = std::get_if(&*maybeAddr)) { - auto range = m_ActiveExits.equal_range(PubKey{*rid}); - auto itr = range.first; - while (itr != range.second) + for (auto [itr, end] = m_ActiveExits.equal_range(PubKey{*rid}); itr != end; ++itr) { if (not itr->second->LooksDead(Now())) { if (itr->second->QueueInboundTraffic(ManagedBuffer{payload}, type)) return true; } - ++itr; } if (not m_Router->PathToRouterAllowed(*rid)) @@ -136,8 +132,7 @@ namespace llarp } return true; } - else - return false; + return false; } bool @@ -357,13 +352,9 @@ namespace llarp ExitEndpoint::VisitEndpointsFor( const PubKey& pk, std::function visit) const { - auto range = m_ActiveExits.equal_range(pk); - auto itr = range.first; - while (itr != range.second) + for (auto [itr, end] = m_ActiveExits.equal_range(pk); itr != end; ++itr) { - if (visit(itr->second.get())) - ++itr; - else + if (not visit(itr->second.get())) return true; } return false; @@ -422,25 +413,17 @@ namespace llarp " as we have no working endpoints"); } }); + for (auto& [pubkey, endpoint] : m_ActiveExits) { - auto itr = m_ActiveExits.begin(); - while (itr != m_ActiveExits.end()) + if (!endpoint->Flush()) { - if (!itr->second->Flush()) - { - LogWarn("exit session with ", itr->first, " dropped packets"); - } - ++itr; + LogWarn("exit session with ", pubkey, " dropped packets"); } } + for (auto& [id, session] : m_SNodeSessions) { - auto itr = m_SNodeSessions.begin(); - while (itr != m_SNodeSessions.end()) - { - itr->second->FlushUpstream(); - itr->second->FlushDownstream(); - ++itr; - } + session->FlushUpstream(); + session->FlushDownstream(); } m_Router->PumpLL(); } @@ -558,15 +541,13 @@ namespace llarp // find oldest activity ip address huint128_t found = {0}; llarp_time_t min = std::numeric_limits::max(); - auto itr = m_IPActivity.begin(); - while (itr != m_IPActivity.end()) + for (const auto& [addr, time] : m_IPActivity) { - if (itr->second < min) + if (time < min) { - found.h = itr->first.h; - min = itr->second; + found.h = addr.h; + min = time; } - ++itr; } // kick old ident off exit // TODO: DoS @@ -620,9 +601,9 @@ namespace llarp ExitEndpoint::AllRemoteEndpoints() const { std::unordered_set remote; - for (auto itr = m_Paths.begin(); itr != m_Paths.end(); ++itr) + for (const auto& [path, pubkey] : m_Paths) { - remote.insert(RouterID{itr->second}); + remote.insert(RouterID{pubkey}); } return remote; } @@ -640,9 +621,7 @@ namespace llarp huint128_t ip = m_KeyToIP[pk]; m_KeyToIP.erase(pk); m_IPToKey.erase(ip); - auto range = m_ActiveExits.equal_range(pk); - auto exit_itr = range.first; - while (exit_itr != range.second) + for (auto [exit_itr, end] = m_ActiveExits.equal_range(pk); exit_itr != end;) exit_itr = m_ActiveExits.erase(exit_itr); } @@ -677,19 +656,14 @@ namespace llarp { exit::Endpoint* endpoint = nullptr; PubKey pk; - { - auto itr = m_Paths.find(path); - if (itr == m_Paths.end()) - return nullptr; + if (auto itr = m_Paths.find(path); itr != m_Paths.end()) pk = itr->second; - } + else + return nullptr; + if (auto itr = m_ActiveExits.find(pk); itr != m_ActiveExits.end()) { - auto itr = m_ActiveExits.find(pk); - if (itr != m_ActiveExits.end()) - { - if (itr->second->PubKey() == pk) - endpoint = itr->second.get(); - } + if (itr->second->PubKey() == pk) + endpoint = itr->second.get(); } return endpoint; } @@ -698,8 +672,7 @@ namespace llarp ExitEndpoint::UpdateEndpointPath(const PubKey& remote, const PathID_t& next) { // check if already mapped - auto itr = m_Paths.find(next); - if (itr != m_Paths.end()) + if (auto itr = m_Paths.find(next); itr != m_Paths.end()) return false; m_Paths.emplace(next, remote); return true; @@ -780,7 +753,7 @@ namespace llarp { auto session = std::make_shared( other, - std::bind(&ExitEndpoint::QueueSNodePacket, this, std::placeholders::_1, ip), + [this, ip](const auto& buf) { return QueueSNodePacket(buf, ip); }, GetRouter(), 2, 1, @@ -837,18 +810,14 @@ namespace llarp void ExitEndpoint::RemoveExit(const exit::Endpoint* ep) { - auto range = m_ActiveExits.equal_range(ep->PubKey()); - auto itr = range.first; - while (itr != range.second) + for (auto [itr, end] = m_ActiveExits.equal_range(ep->PubKey()); itr != end; ++itr) { if (itr->second->GetCurrentPath() == ep->GetCurrentPath()) { - itr = m_ActiveExits.erase(itr); + m_ActiveExits.erase(itr); // now ep is gone af return; } - - ++itr; } } diff --git a/llarp/handlers/exit.hpp b/llarp/handlers/exit.hpp index cfea855e9..d5053b8d2 100644 --- a/llarp/handlers/exit.hpp +++ b/llarp/handlers/exit.hpp @@ -103,12 +103,10 @@ namespace llarp void CalculateTrafficStats(Stats& stats) { - auto itr = m_ActiveExits.begin(); - while (itr != m_ActiveExits.end()) + for (auto& [pubkey, endpoint] : m_ActiveExits) { - stats[itr->first].first += itr->second->TxRate(); - stats[itr->first].second += itr->second->RxRate(); - ++itr; + stats[pubkey].first += endpoint->TxRate(); + stats[pubkey].second += endpoint->RxRate(); } } diff --git a/llarp/hook/shell.cpp b/llarp/hook/shell.cpp index 9de5e2aee..2a740ea14 100644 --- a/llarp/hook/shell.cpp +++ b/llarp/hook/shell.cpp @@ -136,7 +136,7 @@ namespace llarp { auto job = std::make_shared(shared_from_this(), std::move(params)); - m_ThreadPool.addJob(std::bind(&ExecShellHookJob::Exec, job)); + m_ThreadPool.addJob([job = std::move(job)] { job->Exec(); }); } Backend_ptr diff --git a/llarp/messages/relay_status.cpp b/llarp/messages/relay_status.cpp index 77c3664a5..9ea58a55b 100644 --- a/llarp/messages/relay_status.cpp +++ b/llarp/messages/relay_status.cpp @@ -31,12 +31,12 @@ namespace llarp uint64_t _status, HopHandler_ptr _hop, AbstractRouter* _router, - const PathID_t& pathid) + PathID_t pathid) : frames{std::move(_frames)} , status{_status} , hop{std::move(_hop)} , router{_router} - , pathid{pathid} + , pathid{std::move(pathid)} {} ~LRSM_AsyncHandler() = default; @@ -51,7 +51,7 @@ namespace llarp void queue_handle() { - auto func = std::bind(&llarp::LRSM_AsyncHandler::handle, shared_from_this()); + auto func = [self = shared_from_this()] { self->handle(); }; router->QueueWork(func); } }; diff --git a/llarp/router/rc_lookup_handler.cpp b/llarp/router/rc_lookup_handler.cpp index 7a6255cd3..48150090f 100644 --- a/llarp/router/rc_lookup_handler.cpp +++ b/llarp/router/rc_lookup_handler.cpp @@ -94,8 +94,7 @@ namespace llarp if (shouldDoLookup) { - auto fn = - std::bind(&RCLookupHandler::HandleDHTLookupResult, this, router, std::placeholders::_1); + auto fn = [this, router](const auto& res) { HandleDHTLookupResult(router, res); }; // if we are a client try using the hidden service endpoints if (!isServiceNode) @@ -232,7 +231,7 @@ namespace llarp if (!SessionIsAllowed(newrc.pubkey)) return false; - auto func = std::bind(&RCLookupHandler::CheckRC, this, newrc); + auto func = [this, newrc] { CheckRC(newrc); }; _work(func); // update dht if required From 9113a6b36b1ae92d72be74310243d4eee5d7fa1f Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 11 Nov 2021 10:17:48 -0400 Subject: [PATCH 26/53] Triggered pumping This redoes how/when we pump so that we should only be calling the idempotent PumpLL() when we actually have (or may have) something to pump. --- llarp/exit/session.cpp | 7 +++--- llarp/iwp/session.cpp | 26 +++++++++++++++------ llarp/iwp/session.hpp | 7 ++++-- llarp/link/server.cpp | 19 ++++++++++----- llarp/link/server.hpp | 18 ++++++++------ llarp/link/session.hpp | 2 +- llarp/router/i_outbound_message_handler.hpp | 2 +- llarp/router/outbound_message_handler.cpp | 2 +- llarp/router/outbound_message_handler.hpp | 10 ++++---- llarp/router/router.cpp | 11 ++++----- 10 files changed, 63 insertions(+), 41 deletions(-) diff --git a/llarp/exit/session.cpp b/llarp/exit/session.cpp index f6c2d5136..485ca768f 100644 --- a/llarp/exit/session.cpp +++ b/llarp/exit/session.cpp @@ -288,9 +288,8 @@ namespace llarp auto path = PickEstablishedPath(llarp::path::ePathRoleExit); if (path) { - for (auto& item : m_Upstream) + for (auto& [i, queue] : m_Upstream) { - auto& queue = item.second; while (queue.size()) { auto& msg = queue.front(); @@ -305,8 +304,8 @@ namespace llarp if (m_Upstream.size()) llarp::LogWarn("no path for exit session"); // discard upstream - for (auto& item : m_Upstream) - item.second.clear(); + for (auto& [i, queue] : m_Upstream) + queue.clear(); m_Upstream.clear(); if (numHops == 1) { diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp index 0b5b3bec5..ae0af1444 100644 --- a/llarp/iwp/session.cpp +++ b/llarp/iwp/session.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace llarp { @@ -136,6 +137,7 @@ namespace llarp Session::EncryptAndSend(ILinkSession::Packet_t data) { m_EncryptNext.emplace_back(std::move(data)); + TriggerPump(); if (!IsEstablished()) { EncryptWorker(std::move(m_EncryptNext)); @@ -190,6 +192,7 @@ namespace llarp const auto bufsz = buf.size(); auto& msg = m_TXMsgs.emplace(msgid, OutboundMessage{msgid, std::move(buf), now, completed}) .first->second; + TriggerPump(); EncryptAndSend(msg.XMIT()); if (bufsz > FragmentSize) { @@ -225,6 +228,12 @@ namespace llarp } } + void + Session::TriggerPump() + { + m_Parent->Router()->PumpLL(); + } + void Session::Pump() { @@ -233,18 +242,18 @@ namespace llarp { if (ShouldPing()) SendKeepAlive(); - for (auto& item : m_RXMsgs) + for (auto& [id, msg] : m_RXMsgs) { - if (item.second.ShouldSendACKS(now)) + if (msg.ShouldSendACKS(now)) { - item.second.SendACKS(util::memFn(&Session::EncryptAndSend, this), now); + msg.SendACKS(util::memFn(&Session::EncryptAndSend, this), now); } } - for (auto& item : m_TXMsgs) + for (auto& [id, msg] : m_TXMsgs) { - if (item.second.ShouldFlush(now)) + if (msg.ShouldFlush(now)) { - item.second.FlushUnAcked(util::memFn(&Session::EncryptAndSend, this), now); + msg.FlushUnAcked(util::memFn(&Session::EncryptAndSend, this), now); } } } @@ -613,6 +622,7 @@ namespace llarp Session::HandleSessionData(Packet_t pkt) { m_DecryptNext.emplace_back(std::move(pkt)); + TriggerPump(); } void @@ -679,7 +689,7 @@ namespace llarp } } SendMACK(); - Pump(); + TriggerPump(); } void @@ -774,6 +784,8 @@ namespace llarp { itr = m_RXMsgs.emplace(rxid, InboundMessage{rxid, sz, ShortHash{pos}, m_Parent->Now()}) .first; + TriggerPump(); + sz = std::min(sz, uint16_t{FragmentSize}); if ((data.size() - XMITOverhead) == sz) { diff --git a/llarp/iwp/session.hpp b/llarp/iwp/session.hpp index 7a96c4389..d37a5464a 100644 --- a/llarp/iwp/session.hpp +++ b/llarp/iwp/session.hpp @@ -48,8 +48,11 @@ namespace llarp /// inbound session Session(LinkLayer* parent, const SockAddr& from); - ~Session() = default; + // Signal the event loop that a pump is needed (idempotent) + void + TriggerPump(); + // Does the actual pump void Pump() override; @@ -191,7 +194,7 @@ namespace llarp /// maps rxid to time recieved std::unordered_map m_ReplayFilter; /// rx messages to send in next round of multiacks - std::priority_queue, std::greater> m_SendMACKs; + std::priority_queue, std::greater<>> m_SendMACKs; using CryptoQueue_t = std::vector; diff --git a/llarp/link/server.cpp b/llarp/link/server.cpp index 23658dfe1..7166ab59f 100644 --- a/llarp/link/server.cpp +++ b/llarp/link/server.cpp @@ -7,6 +7,7 @@ #include #include #include +#include static constexpr auto LINK_LAYER_TICK_INTERVAL = 100ms; @@ -38,7 +39,11 @@ namespace llarp , m_SecretKey(keyManager->transportKey) {} - ILinkLayer::~ILinkLayer() = default; + llarp_time_t + ILinkLayer::Now() const + { + return m_Router->loop()->time_now(); + } bool ILinkLayer::HasSessionTo(const RouterID& id) @@ -124,10 +129,10 @@ namespace llarp } bool - ILinkLayer::Configure(EventLoop_ptr loop, const std::string& ifname, int af, uint16_t port) + ILinkLayer::Configure(AbstractRouter* router, const std::string& ifname, int af, uint16_t port) { - m_Loop = std::move(loop); - m_udp = m_Loop->make_udp( + m_Router = router; + m_udp = m_Router->loop()->make_udp( [this]([[maybe_unused]] UDPHandle& udp, const SockAddr& from, llarp_buffer_t buf) { ILinkSession::Packet_t pkt; pkt.resize(buf.sz); @@ -163,7 +168,6 @@ namespace llarp if (not m_udp->listen(m_ourAddr)) return false; - m_Loop->add_ticker([this] { Pump(); }); return true; } @@ -247,6 +251,7 @@ namespace llarp } m_AuthedLinks.emplace(pk, itr->second); itr = m_Pending.erase(itr); + m_Router->PumpLL(); return true; } return false; @@ -345,7 +350,8 @@ namespace llarp { // Tie the lifetime of this repeater to this arbitrary shared_ptr: m_repeater_keepalive = std::make_shared(0); - m_Loop->call_every(LINK_LAYER_TICK_INTERVAL, m_repeater_keepalive, [this] { Tick(Now()); }); + m_Router->loop()->call_every( + LINK_LAYER_TICK_INTERVAL, m_repeater_keepalive, [this] { Tick(Now()); }); return true; } @@ -487,6 +493,7 @@ namespace llarp if (m_Pending.count(address)) return false; m_Pending.emplace(address, s); + m_Router->PumpLL(); return true; } diff --git a/llarp/link/server.hpp b/llarp/link/server.hpp index e58e4da56..803120454 100644 --- a/llarp/link/server.hpp +++ b/llarp/link/server.hpp @@ -85,14 +85,11 @@ namespace llarp SessionClosedHandler closed, PumpDoneHandler pumpDone, WorkerFunc_t doWork); - virtual ~ILinkLayer(); + virtual ~ILinkLayer() = default; /// get current time via event loop llarp_time_t - Now() const - { - return m_Loop->time_now(); - } + Now() const; bool HasSessionTo(const RouterID& pk); @@ -108,7 +105,7 @@ namespace llarp SendTo_LL(const SockAddr& to, const llarp_buffer_t& pkt); virtual bool - Configure(EventLoop_ptr loop, const std::string& ifname, int af, uint16_t port); + Configure(AbstractRouter* loop, const std::string& ifname, int af, uint16_t port); virtual std::shared_ptr NewOutboundSession(const RouterContact& rc, const AddressInfo& ai) = 0; @@ -225,6 +222,13 @@ namespace llarp std::optional GetUDPFD() const; + // Gets a pointer to the router owning us. + AbstractRouter* + Router() const + { + return m_Router; + } + private: const SecretKey& m_RouterEncSecret; @@ -239,7 +243,7 @@ namespace llarp bool PutSession(const std::shared_ptr& s); - EventLoop_ptr m_Loop; + AbstractRouter* m_Router; SockAddr m_ourAddr; std::shared_ptr m_udp; SecretKey m_SecretKey; diff --git a/llarp/link/session.hpp b/llarp/link/session.hpp index 60430458f..11f326b52 100644 --- a/llarp/link/session.hpp +++ b/llarp/link/session.hpp @@ -42,7 +42,7 @@ namespace llarp virtual void OnLinkEstablished(ILinkLayer*){}; - /// called every event loop tick + /// called during pumping virtual void Pump() = 0; diff --git a/llarp/router/i_outbound_message_handler.hpp b/llarp/router/i_outbound_message_handler.hpp index 0ab96c434..ed9e542a5 100644 --- a/llarp/router/i_outbound_message_handler.hpp +++ b/llarp/router/i_outbound_message_handler.hpp @@ -35,7 +35,7 @@ namespace llarp QueueMessage(const RouterID& remote, const ILinkMessage& msg, SendStatusHandler callback) = 0; virtual void - Tick() = 0; + Pump() = 0; virtual void RemovePath(const PathID_t& pathid) = 0; diff --git a/llarp/router/outbound_message_handler.cpp b/llarp/router/outbound_message_handler.cpp index 2aaa6a30e..7e5a0cab4 100644 --- a/llarp/router/outbound_message_handler.cpp +++ b/llarp/router/outbound_message_handler.cpp @@ -81,7 +81,7 @@ namespace llarp } void - OutboundMessageHandler::Tick() + OutboundMessageHandler::Pump() { m_Killer.TryAccess([this]() { recentlyRemovedPaths.Decay(); diff --git a/llarp/router/outbound_message_handler.hpp b/llarp/router/outbound_message_handler.hpp index e0402d633..e648ad1fa 100644 --- a/llarp/router/outbound_message_handler.hpp +++ b/llarp/router/outbound_message_handler.hpp @@ -34,9 +34,9 @@ namespace llarp * router, one is created. * * If there is a session to the destination router, the message is placed on the shared - * outbound message queue to be processed on Tick(). + * outbound message queue to be processed on Pump(). * - * When this class' Tick() is called, that queue is emptied and the messages there + * When this class' Pump() is called, that queue is emptied and the messages there * are placed in their paths' respective individual queues. * * Returns false if encoding the message into a buffer fails, true otherwise. @@ -47,7 +47,7 @@ namespace llarp QueueMessage(const RouterID& remote, const ILinkMessage& msg, SendStatusHandler callback) override EXCLUDES(_mutex); - /* Called once per event loop tick. + /* Called when pumping output queues, typically scheduled via a call to Router::PumpLL(). * * Processes messages on the shared message queue into their paths' respective * individual queues. @@ -59,7 +59,7 @@ namespace llarp * Sends messages from path queues until all are empty or a set cap has been reached. */ void - Tick() override; + Pump() override; /* Called from outside this class to inform it that a path has died / expired * and its queue should be discarded. @@ -145,7 +145,7 @@ namespace llarp * If the queue is full, the message is dropped and the message's status * callback is invoked with a congestion status. * - * When this class' Tick() is called, that queue is emptied and the messages there + * When this class' Pump() is called, that queue is emptied and the messages there * are placed in their paths' respective individual queues. */ bool diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 2739c85d4..51283428b 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -87,7 +87,7 @@ namespace llarp llarp::LogTrace("Router::PumpLL() start"); if (_stopping.load()) return; - _outboundMessageHandler.Tick(); + _outboundMessageHandler.Pump(); _linkManager.PumpLinks(); llarp::LogTrace("Router::PumpLL() end"); } @@ -106,10 +106,7 @@ namespace llarp {"links", _linkManager.ExtractStatus()}, {"outboundMessages", _outboundMessageHandler.ExtractStatus()}}; } - else - { - return util::StatusObject{{"running", false}}; - } + return util::StatusObject{{"running", false}}; } util::StatusObject @@ -716,7 +713,7 @@ namespace llarp const std::string& key = serverConfig.interface; int af = serverConfig.addressFamily; uint16_t port = serverConfig.port; - if (!server->Configure(loop(), key, af, port)) + if (!server->Configure(this, key, af, port)) { throw std::runtime_error(stringify("failed to bind inbound link on ", key, " port ", port)); } @@ -1521,7 +1518,7 @@ namespace llarp for (const auto af : {AF_INET, AF_INET6}) { - if (not link->Configure(loop(), "*", af, m_OutboundPort)) + if (not link->Configure(this, "*", af, m_OutboundPort)) continue; #if defined(ANDROID) From b03f3c2d790514662cceb0be497c673b515ff17c Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 11 Nov 2021 10:54:51 -0400 Subject: [PATCH 27/53] Remove another unwanted unconditional PumpLL --- llarp/handlers/exit.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/llarp/handlers/exit.cpp b/llarp/handlers/exit.cpp index 7c3d28a21..8c88c3753 100644 --- a/llarp/handlers/exit.cpp +++ b/llarp/handlers/exit.cpp @@ -425,7 +425,6 @@ namespace llarp session->FlushUpstream(); session->FlushDownstream(); } - m_Router->PumpLL(); } bool From 362fda5a56538aa84178bfe8842dc54641b90fb8 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 11 Nov 2021 13:41:36 -0400 Subject: [PATCH 28/53] Combine empty + pop into single call --- llarp/iwp/session.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp index ae0af1444..2e5d05c08 100644 --- a/llarp/iwp/session.cpp +++ b/llarp/iwp/session.cpp @@ -654,10 +654,9 @@ namespace llarp void Session::HandlePlaintext() { - while (not m_PlaintextRecv.empty()) + while (auto maybe_queue = m_PlaintextRecv.tryPopFront()) { - auto queue = m_PlaintextRecv.popFront(); - for (auto& result : queue) + for (auto& result : *maybe_queue) { LogTrace("Command ", int(result[PacketOverhead + 1]), " from ", m_RemoteAddr); switch (result[PacketOverhead + 1]) From aa1dc8345984586ad9f61eb953bfcc4dac30e6b1 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 11 Nov 2021 19:32:54 -0400 Subject: [PATCH 29/53] Simplify --- llarp/iwp/linklayer.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/llarp/iwp/linklayer.cpp b/llarp/iwp/linklayer.cpp index d3c10c218..6a0745c36 100644 --- a/llarp/iwp/linklayer.cpp +++ b/llarp/iwp/linklayer.cpp @@ -58,14 +58,15 @@ namespace llarp::iwp if (itr == m_AuthedAddrs.end()) { Lock_t lock{m_PendingMutex}; - if (m_Pending.count(from) == 0) + auto it = m_Pending.find(from); + if (it == m_Pending.end()) { if (not m_Inbound) return; isNewSession = true; - m_Pending.insert({from, std::make_shared(this, from)}); + it = m_Pending.emplace(from, std::make_shared(this, from)).first; } - session = m_Pending.find(from)->second; + session = it->second; } else { @@ -78,7 +79,7 @@ namespace llarp::iwp if (not success and isNewSession) { LogWarn("Brand new session failed; removing from pending sessions list"); - m_Pending.erase(m_Pending.find(from)); + m_Pending.erase(from); } } } From 74215fc44c583aa0116dfb0c5115338a009cb5b1 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 11 Nov 2021 17:46:53 -0400 Subject: [PATCH 30/53] Fix link layer delivery race condition (fix random ping delays) We trigger a pump immediately, but this is racey because we add to our plaintext data in a worker thread; if the worker thread runs after the pump then it ends up leaving plaintext to be handled, but there's no wakeup until the next one. This was the cause of seeing a random +1s and bunching added to ping responses sometimes: it wasn't until the *next* ping goes through the network that the plaintext queue gets processed, at which point it flushes the old one and often the new one together. The fix here gets rid of the map of sessions needing wakeups and instead adds an atomic flag to all of them to let us figure out which ones need to be flushed. --- llarp/iwp/linklayer.cpp | 26 ++++++++++---------------- llarp/iwp/linklayer.hpp | 5 +---- llarp/iwp/session.cpp | 8 ++++++-- llarp/iwp/session.hpp | 3 ++- llarp/link/server.cpp | 4 +--- llarp/link/session.hpp | 3 +++ llarp/router/router.cpp | 1 + 7 files changed, 24 insertions(+), 26 deletions(-) diff --git a/llarp/iwp/linklayer.cpp b/llarp/iwp/linklayer.cpp index 6a0745c36..7615a28d2 100644 --- a/llarp/iwp/linklayer.cpp +++ b/llarp/iwp/linklayer.cpp @@ -23,9 +23,7 @@ namespace llarp::iwp : ILinkLayer( keyManager, getrc, h, sign, before, est, reneg, timeout, closed, pumpDone, worker) , m_Wakeup{ev->make_waker([this]() { HandleWakeupPlaintext(); })} - , m_PlaintextRecv{1024} , m_Inbound{allowInbound} - {} const char* @@ -81,6 +79,7 @@ namespace llarp::iwp LogWarn("Brand new session failed; removing from pending sessions list"); m_Pending.erase(from); } + WakeupPlaintext(); } } @@ -107,13 +106,6 @@ namespace llarp::iwp return std::make_shared(this, rc, ai); } - void - LinkLayer::AddWakeup(std::weak_ptr session) - { - if (auto ptr = session.lock()) - m_PlaintextRecv[ptr->GetRemoteEndpoint()] = session; - } - void LinkLayer::WakeupPlaintext() { @@ -123,13 +115,15 @@ namespace llarp::iwp void LinkLayer::HandleWakeupPlaintext() { - for (const auto& [addr, session] : m_PlaintextRecv) - { - auto ptr = session.lock(); - if (ptr) - ptr->HandlePlaintext(); - } - m_PlaintextRecv.clear(); + // Copy bare pointers out first because HandlePlaintext can end up removing themselves from the + // structures. + m_WakingUp.clear(); // Reused to minimize allocations. + for (const auto& [router_id, session] : m_AuthedLinks) + m_WakingUp.push_back(session.get()); + for (const auto& [addr, session] : m_Pending) + m_WakingUp.push_back(session.get()); + for (auto* session : m_WakingUp) + session->HandlePlaintext(); PumpDone(); } diff --git a/llarp/iwp/linklayer.hpp b/llarp/iwp/linklayer.hpp index 0ebb4d8aa..52bfd34e6 100644 --- a/llarp/iwp/linklayer.hpp +++ b/llarp/iwp/linklayer.hpp @@ -53,9 +53,6 @@ namespace llarp::iwp void WakeupPlaintext(); - void - AddWakeup(std::weak_ptr peer); - std::string PrintableName() const; @@ -64,8 +61,8 @@ namespace llarp::iwp HandleWakeupPlaintext(); const std::shared_ptr m_Wakeup; - std::unordered_map> m_PlaintextRecv; std::unordered_map m_AuthedAddrs; + std::vector m_WakingUp; const bool m_Inbound; }; diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp index 2e5d05c08..09be64f86 100644 --- a/llarp/iwp/session.cpp +++ b/llarp/iwp/session.cpp @@ -40,6 +40,7 @@ namespace llarp , m_PlaintextRecv{PlaintextQueueSize} { token.Zero(); + m_PlaintextEmpty.test_and_set(); GotLIM = util::memFn(&Session::GotOutboundLIM, this); CryptoManager::instance()->shorthash(m_SessionKey, llarp_buffer_t(rc.pubkey)); } @@ -53,6 +54,7 @@ namespace llarp , m_PlaintextRecv{PlaintextQueueSize} { token.Randomize(); + m_PlaintextEmpty.test_and_set(); GotLIM = util::memFn(&Session::GotInboundLIM, this); const PubKey pk = m_Parent->GetOurRC().pubkey; CryptoManager::instance()->shorthash(m_SessionKey, llarp_buffer_t(pk)); @@ -267,7 +269,6 @@ namespace llarp if (not m_DecryptNext.empty()) { - m_Parent->AddWakeup(weak_from_this()); m_Parent->QueueWork([self, data = m_DecryptNext] { self->DecryptWorker(data); }); m_DecryptNext.clear(); } @@ -648,12 +649,15 @@ namespace llarp ++itr; } m_PlaintextRecv.tryPushBack(std::move(msgs)); + m_PlaintextEmpty.clear(); m_Parent->WakeupPlaintext(); } void Session::HandlePlaintext() { + if (m_PlaintextEmpty.test_and_set()) + return; while (auto maybe_queue = m_PlaintextRecv.tryPopFront()) { for (auto& result : *maybe_queue) @@ -688,7 +692,7 @@ namespace llarp } } SendMACK(); - TriggerPump(); + m_Parent->WakeupPlaintext(); } void diff --git a/llarp/iwp/session.hpp b/llarp/iwp/session.hpp index d37a5464a..460063005 100644 --- a/llarp/iwp/session.hpp +++ b/llarp/iwp/session.hpp @@ -132,7 +132,7 @@ namespace llarp return m_Inbound; } void - HandlePlaintext(); + HandlePlaintext() override; private: enum class State @@ -201,6 +201,7 @@ namespace llarp CryptoQueue_t m_EncryptNext; CryptoQueue_t m_DecryptNext; + std::atomic_flag m_PlaintextEmpty; llarp::thread::Queue m_PlaintextRecv; void diff --git a/llarp/link/server.cpp b/llarp/link/server.cpp index 7166ab59f..553186a73 100644 --- a/llarp/link/server.cpp +++ b/llarp/link/server.cpp @@ -408,8 +408,7 @@ namespace llarp Lock_t l(m_AuthedLinksMutex); RouterID r = remote; llarp::LogInfo("Closing all to ", r); - auto [itr, end] = m_AuthedLinks.equal_range(r); - while (itr != end) + for (auto [itr, end] = m_AuthedLinks.equal_range(r); itr != end;) { itr->second->Close(); m_RecentlyClosed.emplace(itr->second->GetRemoteEndpoint(), now + CloseGraceWindow); @@ -493,7 +492,6 @@ namespace llarp if (m_Pending.count(address)) return false; m_Pending.emplace(address, s); - m_Router->PumpLL(); return true; } diff --git a/llarp/link/session.hpp b/llarp/link/session.hpp index 11f326b52..ab8b5d4f5 100644 --- a/llarp/link/session.hpp +++ b/llarp/link/session.hpp @@ -130,5 +130,8 @@ namespace llarp virtual util::StatusObject ExtractStatus() const = 0; + + virtual void + HandlePlaintext() = 0; }; } // namespace llarp diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 51283428b..50e8f1a57 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -1248,6 +1248,7 @@ namespace llarp #else _loop->set_pump_function([this] { PumpLLNonIdempotent(); }); #endif + //_loop->call_every(10ms, weak_from_this(), [this] { PumpLLNonIdempotent(); }); _loop->call_every(ROUTER_TICK_INTERVAL, weak_from_this(), [this] { Tick(); }); _running.store(true); _startedAt = Now(); From 031ea7aa373114acf9aeafb97b2ca97c0e712805 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 12 Nov 2021 09:25:14 -0400 Subject: [PATCH 31/53] Get rid of external event loop direct wakeups If something needs to wake up the event loop it should be using an async, as we are now with PumpLL(); but we had various code triggering a wakeup, expecting that PumpLL gets called on every wakeup, which isn't true anymore. --- llarp/ev/ev.hpp | 17 +++++++---------- llarp/ev/ev_libuv.cpp | 10 +--------- llarp/ev/ev_libuv.hpp | 11 +++-------- llarp/path/ihophandler.cpp | 4 ++-- llarp/path/path.cpp | 4 ++-- llarp/path/transit_hop.cpp | 5 ++--- llarp/router/router.cpp | 3 --- llarp/service/endpoint.cpp | 6 +++--- 8 files changed, 20 insertions(+), 40 deletions(-) diff --git a/llarp/ev/ev.hpp b/llarp/ev/ev.hpp index 7e3d38740..f39d3041f 100644 --- a/llarp/ev/ev.hpp +++ b/llarp/ev/ev.hpp @@ -81,12 +81,6 @@ namespace llarp return llarp::time_now_ms(); } - // Triggers an event loop wakeup; use when something has been done that requires the event loop - // to wake up (e.g. adding to queues). This is called implicitly by call() and call_soon(). - // Idempotent and thread-safe. - virtual void - wakeup() = 0; - // Calls a function/lambda/etc. If invoked from within the event loop itself this calls the // given lambda immediately; otherwise it passes it to `call_soon()` to be queued to run at the // next event loop iteration. @@ -196,10 +190,6 @@ namespace llarp virtual std::shared_ptr make_udp(UDPReceiveFunc on_recv) = 0; - /// set the function that is called once per cycle the flush all the queues - virtual void - set_pump_function(std::function pumpll) = 0; - /// Make a thread-safe event loop waker (an "async" in libuv terminology) on this event loop; /// you can call `->Trigger()` on the returned shared pointer to fire the callback at the next /// available event loop iteration. (Multiple Trigger calls invoked before the call is actually @@ -227,6 +217,13 @@ namespace llarp { return nullptr; } + + protected: + // Triggers an event loop wakeup; use when something has been done that requires the event loop + // to wake up (e.g. adding to queues). This is called implicitly by call() and call_soon(). + // Idempotent and thread-safe. + virtual void + wakeup() = 0; }; using EventLoop_ptr = std::shared_ptr; diff --git a/llarp/ev/ev_libuv.cpp b/llarp/ev/ev_libuv.cpp index 04b447987..b491f42b0 100644 --- a/llarp/ev/ev_libuv.cpp +++ b/llarp/ev/ev_libuv.cpp @@ -111,14 +111,12 @@ namespace llarp::uv { llarp::LogTrace("ticking event loop."); FlushLogic(); - if (PumpLL) - PumpLL(); auto& log = llarp::LogContext::Instance(); if (log.logStream) log.logStream->Tick(time_now()); } - Loop::Loop(size_t queue_size) : llarp::EventLoop{}, PumpLL{nullptr}, m_LogicCalls{queue_size} + Loop::Loop(size_t queue_size) : llarp::EventLoop{}, m_LogicCalls{queue_size} { if (!(m_Impl = uvw::Loop::create())) throw std::runtime_error{"Failed to construct libuv loop"}; @@ -162,12 +160,6 @@ namespace llarp::uv m_WakeUp->send(); } - void - Loop::set_pump_function(std::function pump) - { - PumpLL = std::move(pump); - } - std::shared_ptr Loop::make_udp(UDPReceiveFunc on_recv) { diff --git a/llarp/ev/ev_libuv.hpp b/llarp/ev/ev_libuv.hpp index e03b52c93..86dacac79 100644 --- a/llarp/ev/ev_libuv.hpp +++ b/llarp/ev/ev_libuv.hpp @@ -31,9 +31,6 @@ namespace llarp::uv bool running() const override; - void - wakeup() override; - void call_later(llarp_time_t delay_ms, std::function callback) override; @@ -54,9 +51,6 @@ namespace llarp::uv void call_soon(std::function f) override; - void - set_pump_function(std::function pumpll) override; - std::shared_ptr make_waker(std::function callback) override; @@ -69,8 +63,6 @@ namespace llarp::uv void FlushLogic(); - std::function PumpLL; - std::shared_ptr MaybeGetUVWLoop() override; @@ -95,6 +87,9 @@ namespace llarp::uv std::unordered_map> m_Polls; std::optional m_EventLoopThreadID; + + void + wakeup() override; }; } // namespace llarp::uv diff --git a/llarp/path/ihophandler.cpp b/llarp/path/ihophandler.cpp index 1b185e3cb..589d4e4cd 100644 --- a/llarp/path/ihophandler.cpp +++ b/llarp/path/ihophandler.cpp @@ -16,7 +16,7 @@ namespace llarp pkt.first.resize(X.sz); std::copy_n(X.base, X.sz, pkt.first.begin()); pkt.second = Y; - r->loop()->wakeup(); + r->PumpLL(); return true; } @@ -31,7 +31,7 @@ namespace llarp pkt.first.resize(X.sz); std::copy_n(X.base, X.sz, pkt.first.begin()); pkt.second = Y; - r->loop()->wakeup(); + r->PumpLL(); return true; } diff --git a/llarp/path/path.cpp b/llarp/path/path.cpp index 9692233f1..0181a838b 100644 --- a/llarp/path/path.cpp +++ b/llarp/path/path.cpp @@ -488,7 +488,7 @@ namespace llarp LogDebug("failed to send upstream to ", Upstream()); } } - r->linkManager().PumpLinks(); + r->PumpLL(); } void @@ -600,7 +600,7 @@ namespace llarp m_RXRate += buf.sz; if (HandleRoutingMessage(buf, r)) { - r->loop()->wakeup(); + r->PumpLL(); m_LastRecvMessage = r->Now(); } } diff --git a/llarp/path/transit_hop.cpp b/llarp/path/transit_hop.cpp index 2e09a6f74..cb2abcd2e 100644 --- a/llarp/path/transit_hop.cpp +++ b/llarp/path/transit_hop.cpp @@ -185,7 +185,6 @@ namespace llarp other->FlushDownstream(r); } m_FlushOthers.clear(); - r->loop()->wakeup(); } else { @@ -200,8 +199,8 @@ namespace llarp info.upstream); r->SendToOrQueue(info.upstream, msg); } - r->linkManager().PumpLinks(); } + r->PumpLL(); } void @@ -218,7 +217,7 @@ namespace llarp info.downstream); r->SendToOrQueue(info.downstream, msg); } - r->linkManager().PumpLinks(); + r->PumpLL(); } void diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 50e8f1a57..0cd9f00aa 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -1245,10 +1245,7 @@ namespace llarp #ifdef _WIN32 // windows uses proactor event loop so we need to constantly pump _loop->add_ticker([this] { PumpLLNonIdempotent(); }); -#else - _loop->set_pump_function([this] { PumpLLNonIdempotent(); }); #endif - //_loop->call_every(10ms, weak_from_this(), [this] { PumpLLNonIdempotent(); }); _loop->call_every(ROUTER_TICK_INTERVAL, weak_from_this(), [this] { Tick(); }); _running.store(true); _startedAt = Now(); diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index 943dfe4e4..139325f49 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -1587,7 +1587,7 @@ namespace llarp if (*ptr == m_Identity.pub.Addr()) { ConvoTagTX(tag); - Loop()->wakeup(); + m_state->m_Router->PumpLL(); if (not HandleInboundPacket(tag, pkt, t, 0)) return false; ConvoTagRX(tag); @@ -1596,7 +1596,7 @@ namespace llarp } if (not SendToOrQueue(*maybe, pkt, t)) return false; - Loop()->wakeup(); + m_state->m_Router->PumpLL(); return true; } LogDebug("SendToOrQueue failed: no endpoint for convo tag ", tag); @@ -1673,7 +1673,7 @@ namespace llarp } UpstreamFlush(router); - router->linkManager().PumpLinks(); + router->PumpLL(); } std::optional From 78cc466bf2b8f50270dcfee7d5bad868ee0d1e39 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 12 Nov 2021 09:51:39 -0400 Subject: [PATCH 32/53] Rename PumpLL -> TriggerPump And rename the actual pump implementation back to PumpLL. --- llarp/iwp/session.cpp | 2 +- llarp/link/server.cpp | 2 +- llarp/path/ihophandler.cpp | 4 ++-- llarp/path/path.cpp | 4 ++-- llarp/path/transit_hop.cpp | 4 ++-- llarp/router/abstractrouter.hpp | 4 ++-- llarp/router/outbound_message_handler.cpp | 2 +- llarp/router/outbound_message_handler.hpp | 2 +- llarp/router/router.cpp | 12 ++++++------ llarp/router/router.hpp | 4 ++-- llarp/service/endpoint.cpp | 6 +++--- 11 files changed, 23 insertions(+), 23 deletions(-) diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp index 09be64f86..3806d8696 100644 --- a/llarp/iwp/session.cpp +++ b/llarp/iwp/session.cpp @@ -233,7 +233,7 @@ namespace llarp void Session::TriggerPump() { - m_Parent->Router()->PumpLL(); + m_Parent->Router()->TriggerPump(); } void diff --git a/llarp/link/server.cpp b/llarp/link/server.cpp index 553186a73..083bd71ea 100644 --- a/llarp/link/server.cpp +++ b/llarp/link/server.cpp @@ -251,7 +251,7 @@ namespace llarp } m_AuthedLinks.emplace(pk, itr->second); itr = m_Pending.erase(itr); - m_Router->PumpLL(); + m_Router->TriggerPump(); return true; } return false; diff --git a/llarp/path/ihophandler.cpp b/llarp/path/ihophandler.cpp index 589d4e4cd..e55e0d850 100644 --- a/llarp/path/ihophandler.cpp +++ b/llarp/path/ihophandler.cpp @@ -16,7 +16,7 @@ namespace llarp pkt.first.resize(X.sz); std::copy_n(X.base, X.sz, pkt.first.begin()); pkt.second = Y; - r->PumpLL(); + r->TriggerPump(); return true; } @@ -31,7 +31,7 @@ namespace llarp pkt.first.resize(X.sz); std::copy_n(X.base, X.sz, pkt.first.begin()); pkt.second = Y; - r->PumpLL(); + r->TriggerPump(); return true; } diff --git a/llarp/path/path.cpp b/llarp/path/path.cpp index 0181a838b..36491671c 100644 --- a/llarp/path/path.cpp +++ b/llarp/path/path.cpp @@ -488,7 +488,7 @@ namespace llarp LogDebug("failed to send upstream to ", Upstream()); } } - r->PumpLL(); + r->TriggerPump(); } void @@ -600,7 +600,7 @@ namespace llarp m_RXRate += buf.sz; if (HandleRoutingMessage(buf, r)) { - r->PumpLL(); + r->TriggerPump(); m_LastRecvMessage = r->Now(); } } diff --git a/llarp/path/transit_hop.cpp b/llarp/path/transit_hop.cpp index cb2abcd2e..a04c3537e 100644 --- a/llarp/path/transit_hop.cpp +++ b/llarp/path/transit_hop.cpp @@ -200,7 +200,7 @@ namespace llarp r->SendToOrQueue(info.upstream, msg); } } - r->PumpLL(); + r->TriggerPump(); } void @@ -217,7 +217,7 @@ namespace llarp info.downstream); r->SendToOrQueue(info.downstream, msg); } - r->PumpLL(); + r->TriggerPump(); } void diff --git a/llarp/router/abstractrouter.hpp b/llarp/router/abstractrouter.hpp index c249ef420..3d0d1e747 100644 --- a/llarp/router/abstractrouter.hpp +++ b/llarp/router/abstractrouter.hpp @@ -210,9 +210,9 @@ namespace llarp virtual void Die() = 0; - /// pump low level links + /// Trigger a pump of low level links. Idempotent. virtual void - PumpLL() = 0; + TriggerPump() = 0; virtual bool IsBootstrapNode(RouterID r) const = 0; diff --git a/llarp/router/outbound_message_handler.cpp b/llarp/router/outbound_message_handler.cpp index 7e5a0cab4..2a8639d09 100644 --- a/llarp/router/outbound_message_handler.cpp +++ b/llarp/router/outbound_message_handler.cpp @@ -87,7 +87,7 @@ namespace llarp recentlyRemovedPaths.Decay(); ProcessOutboundQueue(); if (/*bool more = */ SendRoundRobin()) - _router->PumpLL(); + _router->TriggerPump(); }); } diff --git a/llarp/router/outbound_message_handler.hpp b/llarp/router/outbound_message_handler.hpp index e648ad1fa..3edb6d917 100644 --- a/llarp/router/outbound_message_handler.hpp +++ b/llarp/router/outbound_message_handler.hpp @@ -47,7 +47,7 @@ namespace llarp QueueMessage(const RouterID& remote, const ILinkMessage& msg, SendStatusHandler callback) override EXCLUDES(_mutex); - /* Called when pumping output queues, typically scheduled via a call to Router::PumpLL(). + /* Called when pumping output queues, typically scheduled via a call to Router::TriggerPump(). * * Processes messages on the shared message queue into their paths' respective * individual queues. diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 0cd9f00aa..cad2d0ae5 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -72,7 +72,7 @@ namespace llarp m_Pump = _loop->make_waker([this]() { paths.PumpDownstream(); paths.PumpUpstream(); - PumpLLNonIdempotent(); + PumpLL(); }); } @@ -82,7 +82,7 @@ namespace llarp } void - Router::PumpLLNonIdempotent() + Router::PumpLL() { llarp::LogTrace("Router::PumpLL() start"); if (_stopping.load()) @@ -256,7 +256,7 @@ namespace llarp } void - Router::PumpLL() + Router::TriggerPump() { m_Pump->Trigger(); } @@ -707,7 +707,7 @@ namespace llarp util::memFn(&AbstractRouter::CheckRenegotiateValid, this), util::memFn(&Router::ConnectionTimedOut, this), util::memFn(&AbstractRouter::SessionClosed, this), - util::memFn(&AbstractRouter::PumpLL, this), + util::memFn(&AbstractRouter::TriggerPump, this), util::memFn(&AbstractRouter::QueueWork, this)); const std::string& key = serverConfig.interface; @@ -1244,7 +1244,7 @@ namespace llarp #ifdef _WIN32 // windows uses proactor event loop so we need to constantly pump - _loop->add_ticker([this] { PumpLLNonIdempotent(); }); + _loop->add_ticker([this] { PumpLL(); }); #endif _loop->call_every(ROUTER_TICK_INTERVAL, weak_from_this(), [this] { Tick(); }); _running.store(true); @@ -1508,7 +1508,7 @@ namespace llarp util::memFn(&AbstractRouter::CheckRenegotiateValid, this), util::memFn(&Router::ConnectionTimedOut, this), util::memFn(&AbstractRouter::SessionClosed, this), - util::memFn(&AbstractRouter::PumpLL, this), + util::memFn(&AbstractRouter::TriggerPump, this), util::memFn(&AbstractRouter::QueueWork, this)); if (!link) diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index f444bb3f1..0e56e4920 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -284,10 +284,10 @@ namespace llarp RoutePoker m_RoutePoker; void - PumpLL() override; + TriggerPump() override; void - PumpLLNonIdempotent(); + PumpLL(); const oxenmq::address DefaultRPCBindAddr = oxenmq::address::tcp("127.0.0.1", 1190); bool enableRPCServer = false; diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index 139325f49..0b4faac33 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -1587,7 +1587,7 @@ namespace llarp if (*ptr == m_Identity.pub.Addr()) { ConvoTagTX(tag); - m_state->m_Router->PumpLL(); + m_state->m_Router->TriggerPump(); if (not HandleInboundPacket(tag, pkt, t, 0)) return false; ConvoTagRX(tag); @@ -1596,7 +1596,7 @@ namespace llarp } if (not SendToOrQueue(*maybe, pkt, t)) return false; - m_state->m_Router->PumpLL(); + m_state->m_Router->TriggerPump(); return true; } LogDebug("SendToOrQueue failed: no endpoint for convo tag ", tag); @@ -1673,7 +1673,7 @@ namespace llarp } UpstreamFlush(router); - router->PumpLL(); + router->TriggerPump(); } std::optional From e47d3098acd7040d46f95bec483cfee7988dd5a9 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 12 Nov 2021 10:09:16 -0400 Subject: [PATCH 33/53] Increase plaintext queue size from 32 to 512 In testing we were sometimes hitting a full queue error, which makes sense because the queue size seems really small. Increase it to 512. --- llarp/iwp/session.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp index 3806d8696..b3999895e 100644 --- a/llarp/iwp/session.cpp +++ b/llarp/iwp/session.cpp @@ -27,7 +27,7 @@ namespace llarp return pkt; } - constexpr size_t PlaintextQueueSize = 32; + constexpr size_t PlaintextQueueSize = 512; Session::Session(LinkLayer* p, const RouterContact& rc, const AddressInfo& ai) : m_State{State::Initial} From 99b12940ada9249cb0f6488c7bff49ba96082d1a Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 12 Nov 2021 15:33:54 -0400 Subject: [PATCH 34/53] Cleanups --- llarp/handlers/tun.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/llarp/handlers/tun.cpp b/llarp/handlers/tun.cpp index 8150ba154..97ef79f06 100644 --- a/llarp/handlers/tun.cpp +++ b/llarp/handlers/tun.cpp @@ -1091,14 +1091,12 @@ namespace llarp service::Address addr{}; for (const auto& [range, exitAddr] : exitEntries) { - if (range.BogonRange() and range.Contains(dst)) + if ( + // we permit this because it matches our rules and we allow bogons: + (range.BogonRange() and range.Contains(dst)) + // allow because the destination is not a bogon and the mapped range is not a bogon + or not IsBogon(dst)) { - // we permit this because it matches our rules and we allow bogons - addr = exitAddr; - } - else if (not IsBogon(dst)) - { - // allow because the destination is not a bogon and the mapped range is not a bogon addr = exitAddr; } // we do not permit bogons when they don't explicitly match a permitted bogon range @@ -1162,7 +1160,7 @@ namespace llarp if (not maybe) { var::visit( - [&](auto&& addr) { + [this](auto&& addr) { LogWarn(Name(), " failed to ensure path to ", addr, " no convo tag found"); }, to); @@ -1174,7 +1172,7 @@ namespace llarp else { var::visit( - [&](auto&& addr) { + [this](auto&& addr) { LogWarn(Name(), " failed to send to ", addr, ", SendToOrQueue failed"); }, to); From 9844d358e6ea647c1df368c0452a26956f7874e8 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 12 Nov 2021 15:34:03 -0400 Subject: [PATCH 35/53] Fix packet delay creep - Make the main PumpLL also pump hidden services, rather than using separate wakers in each TunEndpoint. It seems there is some interactions that just one or the other is not enough. - Eliminate TunEndpoint send queue -- it isn't needed as we can just send directly. --- llarp/handlers/tun.cpp | 284 +++++++++++++++++-------------------- llarp/handlers/tun.hpp | 20 +-- llarp/router/router.cpp | 1 + llarp/service/context.cpp | 8 ++ llarp/service/context.hpp | 4 + llarp/service/endpoint.cpp | 17 +-- llarp/service/endpoint.hpp | 4 + 7 files changed, 163 insertions(+), 175 deletions(-) diff --git a/llarp/handlers/tun.cpp b/llarp/handlers/tun.cpp index 97ef79f06..5c49ca216 100644 --- a/llarp/handlers/tun.cpp +++ b/llarp/handlers/tun.cpp @@ -86,13 +86,7 @@ namespace llarp TunEndpoint::TunEndpoint(AbstractRouter* r, service::Context* parent) : service::Endpoint(r, parent) - , m_UserToNetworkPktQueue("endpoint_sendq", r->loop(), r->loop()) { - m_PacketSendWaker = r->loop()->make_waker([this]() { FlushWrite(); }); - m_MessageSendWaker = r->loop()->make_waker([this]() { - FlushSend(); - Pump(Now()); - }); m_PacketRouter = std::make_unique( [this](net::IPPacket pkt) { HandleGotUserPacket(std::move(pkt)); }); #if defined(ANDROID) || defined(__APPLE__) @@ -362,15 +356,7 @@ namespace llarp } void - TunEndpoint::Flush() - { - FlushSend(); - Pump(Now()); - FlushWrite(); - } - - void - TunEndpoint::FlushWrite() + TunEndpoint::Pump(llarp_time_t now) { // flush network to user while (not m_NetworkToUserPktQueue.empty()) @@ -378,6 +364,8 @@ namespace llarp m_NetIf->WritePacket(m_NetworkToUserPktQueue.top().pkt); m_NetworkToUserPktQueue.pop(); } + + service::Endpoint::Pump(now); } static bool @@ -952,7 +940,6 @@ namespace llarp LogInfo(Name(), " has ipv6 address ", m_OurIPv6); } #endif - Router()->loop()->add_ticker([this] { Flush(); }); // Attempt to register DNS on the interface systemd_resolved_set_dns( @@ -1037,149 +1024,151 @@ namespace llarp } void - TunEndpoint::FlushSend() + TunEndpoint::HandleGotUserPacket(net::IPPacket pkt) { - m_UserToNetworkPktQueue.Process([&](net::IPPacket& pkt) { - huint128_t dst, src; - if (pkt.IsV4()) - { - dst = pkt.dst4to6(); - src = pkt.src4to6(); - } - else - { - dst = pkt.dstv6(); - src = pkt.srcv6(); - } - // this is for ipv6 slaac on ipv6 exits - /* - constexpr huint128_t ipv6_multicast_all_nodes = - huint128_t{uint128_t{0xff01'0000'0000'0000UL, 1UL}}; - constexpr huint128_t ipv6_multicast_all_routers = - huint128_t{uint128_t{0xff01'0000'0000'0000UL, 2UL}}; - if (dst == ipv6_multicast_all_nodes and m_state->m_ExitEnabled) + huint128_t dst, src; + if (pkt.IsV4()) + { + dst = pkt.dst4to6(); + src = pkt.src4to6(); + } + else + { + dst = pkt.dstv6(); + src = pkt.srcv6(); + } + // this is for ipv6 slaac on ipv6 exits + /* + constexpr huint128_t ipv6_multicast_all_nodes = + huint128_t{uint128_t{0xff01'0000'0000'0000UL, 1UL}}; + constexpr huint128_t ipv6_multicast_all_routers = + huint128_t{uint128_t{0xff01'0000'0000'0000UL, 2UL}}; + if (dst == ipv6_multicast_all_nodes and m_state->m_ExitEnabled) + { + // send ipv6 multicast + for (const auto& [ip, addr] : m_IPToAddr) { - // send ipv6 multicast - for (const auto& [ip, addr] : m_IPToAddr) - { - (void)ip; - SendToOrQueue( - service::Address{addr.as_array()}, pkt.ConstBuffer(), service::ProtocolType::Exit); - } - return; + (void)ip; + SendToOrQueue( + service::Address{addr.as_array()}, pkt.ConstBuffer(), service::ProtocolType::Exit); } + return; + } - */ - if (m_state->m_ExitEnabled) - { - dst = net::ExpandV4(net::TruncateV6(dst)); - } - auto itr = m_IPToAddr.find(dst); - if (itr == m_IPToAddr.end()) + */ + if (m_state->m_ExitEnabled) + { + dst = net::ExpandV4(net::TruncateV6(dst)); + } + auto itr = m_IPToAddr.find(dst); + if (itr == m_IPToAddr.end()) + { + // find all ranges that match the destination ip + const auto exitEntries = m_ExitMap.FindAllEntries(dst); + if (exitEntries.empty()) { - // find all ranges that match the destination ip - const auto exitEntries = m_ExitMap.FindAllEntries(dst); - if (exitEntries.empty()) + // send icmp unreachable as we dont have any exits for this ip + if (const auto icmp = pkt.MakeICMPUnreachable()) { - // send icmp unreachable as we dont have any exits for this ip - if (const auto icmp = pkt.MakeICMPUnreachable()) - { - HandleWriteIPPacket(icmp->ConstBuffer(), dst, src, 0); - } - return; - } - service::Address addr{}; - for (const auto& [range, exitAddr] : exitEntries) - { - if ( - // we permit this because it matches our rules and we allow bogons: - (range.BogonRange() and range.Contains(dst)) - // allow because the destination is not a bogon and the mapped range is not a bogon - or not IsBogon(dst)) - { - addr = exitAddr; - } - // we do not permit bogons when they don't explicitly match a permitted bogon range + /// ? + HandleWriteIPPacket(icmp->ConstBuffer(), dst, src, 0); } - if (addr.IsZero()) // drop becase no exit was found that matches our rules - return; - pkt.ZeroSourceAddress(); - MarkAddressOutbound(addr); - EnsurePathToService( - addr, - [pkt](service::Address addr, service::OutboundContext* ctx) { - if (ctx) - { - ctx->SendPacketToRemote(pkt.ConstBuffer(), service::ProtocolType::Exit); - return; - } - LogWarn("cannot ensure path to exit ", addr, " so we drop some packets"); - }, - PathAlignmentTimeout()); return; } - std::variant to; - service::ProtocolType type; - if (m_SNodes.at(itr->second)) - { - to = RouterID{itr->second.as_array()}; - type = service::ProtocolType::TrafficV4; - } - else - { - to = service::Address{itr->second.as_array()}; - type = m_state->m_ExitEnabled and src != m_OurIP ? service::ProtocolType::Exit - : pkt.ServiceProtocol(); - } - - // prepare packet for insertion into network - // this includes clearing IP addresses, recalculating checksums, etc - // this does not happen for exits because the point is they don't rewrite addresses - if (type != service::ProtocolType::Exit) - { - if (pkt.IsV4()) - pkt.UpdateIPv4Address({0}, {0}); - else - pkt.UpdateIPv6Address({0}, {0}); - } - // try sending it on an existing convotag - // this succeds for inbound convos, probably. - if (auto maybe = GetBestConvoTagFor(to)) + service::Address addr{}; + for (const auto& [range, exitAddr] : exitEntries) { - if (SendToOrQueue(*maybe, pkt.ConstBuffer(), type)) + if ( + // we permit this because it matches our rules and we allow bogons: + (range.BogonRange() and range.Contains(dst)) + // allow because the destination is not a bogon and the mapped range is not a bogon + or not IsBogon(dst)) { - MarkIPActive(dst); - return; + addr = exitAddr; } + // we do not permit bogons when they don't explicitly match a permitted bogon range } - // try establishing a path to this guy - // will fail if it's an inbound convo - EnsurePathTo( - to, - [pkt, type, dst, to, this](auto maybe) { - if (not maybe) - { - var::visit( - [this](auto&& addr) { - LogWarn(Name(), " failed to ensure path to ", addr, " no convo tag found"); - }, - to); - } - if (SendToOrQueue(*maybe, pkt.ConstBuffer(), type)) - { - MarkIPActive(dst); - } - else + if (addr.IsZero()) // drop becase no exit was found that matches our rules + return; + pkt.ZeroSourceAddress(); + MarkAddressOutbound(addr); + EnsurePathToService( + addr, + [pkt, this](service::Address addr, service::OutboundContext* ctx) { + if (ctx) { - var::visit( - [this](auto&& addr) { - LogWarn(Name(), " failed to send to ", addr, ", SendToOrQueue failed"); - }, - to); + ctx->SendPacketToRemote(pkt.ConstBuffer(), service::ProtocolType::Exit); + Router()->TriggerPump(); + return; } + LogWarn("cannot ensure path to exit ", addr, " so we drop some packets"); }, PathAlignmentTimeout()); - }); + return; + } + std::variant to; + service::ProtocolType type; + if (m_SNodes.at(itr->second)) + { + to = RouterID{itr->second.as_array()}; + type = service::ProtocolType::TrafficV4; + } + else + { + to = service::Address{itr->second.as_array()}; + type = m_state->m_ExitEnabled and src != m_OurIP ? service::ProtocolType::Exit + : pkt.ServiceProtocol(); + } + + // prepare packet for insertion into network + // this includes clearing IP addresses, recalculating checksums, etc + // this does not happen for exits because the point is they don't rewrite addresses + if (type != service::ProtocolType::Exit) + { + if (pkt.IsV4()) + pkt.UpdateIPv4Address({0}, {0}); + else + pkt.UpdateIPv6Address({0}, {0}); + } + // try sending it on an existing convotag + // this succeds for inbound convos, probably. + if (auto maybe = GetBestConvoTagFor(to)) + { + if (SendToOrQueue(*maybe, pkt.ConstBuffer(), type)) + { + MarkIPActive(dst); + Router()->TriggerPump(); + return; + } + } + // try establishing a path to this guy + // will fail if it's an inbound convo + EnsurePathTo( + to, + [pkt, type, dst, to, this](auto maybe) { + if (not maybe) + { + var::visit( + [this](auto&& addr) { + LogWarn(Name(), " failed to ensure path to ", addr, " no convo tag found"); + }, + to); + } + if (SendToOrQueue(*maybe, pkt.ConstBuffer(), type)) + { + MarkIPActive(dst); + Router()->TriggerPump(); + } + else + { + var::visit( + [this](auto&& addr) { + LogWarn(Name(), " failed to send to ", addr, ", SendToOrQueue failed"); + }, + to); + } + }, + PathAlignmentTimeout()); } bool @@ -1333,8 +1322,8 @@ namespace llarp pkt.UpdateIPv6Address(src, dst); } m_NetworkToUserPktQueue.push(std::move(write)); - // wake up packet flushing event so we ensure that all packets are written to user - m_PacketSendWaker->Trigger(); + // wake up so we ensure that all packets are written to user + Router()->TriggerPump(); return true; } @@ -1439,13 +1428,6 @@ namespace llarp m_IPActivity[ip] = std::numeric_limits::max(); } - void - TunEndpoint::HandleGotUserPacket(net::IPPacket pkt) - { - m_UserToNetworkPktQueue.Emplace(std::move(pkt)); - m_MessageSendWaker->Trigger(); - } - TunEndpoint::~TunEndpoint() = default; } // namespace handlers diff --git a/llarp/handlers/tun.hpp b/llarp/handlers/tun.hpp index 6ea44b794..434a3314b 100644 --- a/llarp/handlers/tun.hpp +++ b/llarp/handlers/tun.hpp @@ -172,10 +172,6 @@ namespace llarp huint128_t ObtainIPForAddr(std::variant addr) override; - /// flush network traffic - void - Flush(); - void ResetInternalState() override; @@ -187,9 +183,6 @@ namespace llarp net::IPPacket::CompareOrder, net::IPPacket::GetNow>; - /// queue for sending packets over the network from us - PacketQueue_t m_UserToNetworkPktQueue; - struct WritePacket { uint64_t seqno; @@ -204,6 +197,10 @@ namespace llarp /// queue for sending packets to user from network std::priority_queue m_NetworkToUserPktQueue; + + void + Pump(llarp_time_t now) override; + /// return true if we have a remote loki address for this ip address bool HasRemoteForIP(huint128_t ipv4) const; @@ -216,10 +213,6 @@ namespace llarp void MarkIPActiveForever(huint128_t ip); - /// flush ip packets - virtual void - FlushSend(); - /// flush writing ip packets to interface void FlushWrite(); @@ -292,11 +285,6 @@ namespace llarp std::set m_OwnedRanges; /// how long to wait for path alignment llarp_time_t m_PathAlignmentTimeout; - /// idempotent wakeup for writing packets to user - std::shared_ptr m_PacketSendWaker; - - /// idempotent wakeup for writing messages to network - std::shared_ptr m_MessageSendWaker; /// a file to load / store the ephemeral address map to std::optional m_PersistAddrMapFile; diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index cad2d0ae5..e7ca84ad8 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -89,6 +89,7 @@ namespace llarp return; _outboundMessageHandler.Pump(); _linkManager.PumpLinks(); + _hiddenServiceContext.Pump(); llarp::LogTrace("Router::PumpLL() end"); } diff --git a/llarp/service/context.cpp b/llarp/service/context.cpp index 3f75ca828..a376b14b5 100644 --- a/llarp/service/context.cpp +++ b/llarp/service/context.cpp @@ -80,6 +80,14 @@ namespace llarp } } + void + Context::Pump() + { + auto now = time_now_ms(); + for (auto& [name, endpoint] : m_Endpoints) + endpoint->Pump(now); + } + bool Context::RemoveEndpoint(const std::string& name) { diff --git a/llarp/service/context.hpp b/llarp/service/context.hpp index e7bbdc7dc..a3ef35f3f 100644 --- a/llarp/service/context.hpp +++ b/llarp/service/context.hpp @@ -36,6 +36,10 @@ namespace llarp void ForEachService(std::function visit) const; + /// Pumps the hidden service endpoints, called during Router::PumpLL + void + Pump(); + /// add endpoint via config void AddEndpoint(const Config& conf, bool autostart = false); diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index 0b4faac33..40537b27d 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -1596,7 +1596,6 @@ namespace llarp } if (not SendToOrQueue(*maybe, pkt, t)) return false; - m_state->m_Router->TriggerPump(); return true; } LogDebug("SendToOrQueue failed: no endpoint for convo tag ", tag); @@ -1610,12 +1609,14 @@ namespace llarp auto pkt = std::make_shared(); if (!pkt->Load(buf)) return false; - EnsurePathToSNode(addr, [=](RouterID, exit::BaseSession_ptr s, ConvoTag) { - if (s) - { - s->SendPacketToRemote(pkt->ConstBuffer(), t); - } - }); + EnsurePathToSNode( + addr, [this, t, pkt = std::move(pkt)](RouterID, exit::BaseSession_ptr s, ConvoTag) { + if (s) + { + s->SendPacketToRemote(pkt->ConstBuffer(), t); + Router()->TriggerPump(); + } + }); return true; } @@ -1673,7 +1674,6 @@ namespace llarp } UpstreamFlush(router); - router->TriggerPump(); } std::optional @@ -1885,6 +1885,7 @@ namespace llarp return; } self->m_SendQueue.tryPushBack(SendEvent_t{transfer, p}); + self->Router()->TriggerPump(); }); return true; } diff --git a/llarp/service/endpoint.hpp b/llarp/service/endpoint.hpp index cfb5a2ef9..56f255f69 100644 --- a/llarp/service/endpoint.hpp +++ b/llarp/service/endpoint.hpp @@ -244,6 +244,10 @@ namespace llarp std::string service, std::function)> resultHandler) override; + /// called when something needs a pump to trigger the pump idempotently + void + TriggerPump(); + /// called on event loop pump virtual void Pump(llarp_time_t now); From f9395cd5dda6c9512e9873a6bf9fa0106114a8dd Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 14 Nov 2021 10:45:18 -0400 Subject: [PATCH 36/53] Cleanup: avoid pointless copy of `this` pointer --- llarp/lokinet_shared.cpp | 6 +++--- llarp/router_version.cpp | 8 ++++---- llarp/service/endpoint.cpp | 15 +++++++-------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/llarp/lokinet_shared.cpp b/llarp/lokinet_shared.cpp index 6b285e20f..7a4891032 100644 --- a/llarp/lokinet_shared.cpp +++ b/llarp/lokinet_shared.cpp @@ -153,18 +153,18 @@ struct lokinet_srv_lookup_private auto lock = ctx->acquire(); if (ctx->impl and ctx->impl->IsUp()) { - ctx->impl->CallSafe([host, service, &promise, ctx, self = this]() { + ctx->impl->CallSafe([host, service, &promise, ctx, this]() { auto ep = ctx->endpoint(); if (ep == nullptr) { promise.set_value(ENOTSUP); return; } - ep->LookupServiceAsync(host, service, [self, &promise, host](auto results) { + ep->LookupServiceAsync(host, service, [this, &promise, host](auto results) { for (const auto& result : results) { if (auto maybe = SRVFromData(result, host)) - self->results.emplace_back(*maybe); + this->results.emplace_back(*maybe); } promise.set_value(0); }); diff --git a/llarp/router_version.cpp b/llarp/router_version.cpp index 33c2760f3..c15a7c2a9 100644 --- a/llarp/router_version.cpp +++ b/llarp/router_version.cpp @@ -56,7 +56,7 @@ namespace llarp Clear(); size_t idx = 0; if (not bencode_read_list( - [self = this, &idx](llarp_buffer_t* buffer, bool has) { + [this, &idx](llarp_buffer_t* buffer, bool has) { if (has) { uint64_t i; @@ -65,14 +65,14 @@ namespace llarp uint64_t val = -1; if (not bencode_read_integer(buffer, &val)) return false; - self->m_ProtoVersion = val; + m_ProtoVersion = val; } else if (bencode_read_integer(buffer, &i)) { // prevent overflow (note that idx includes version too) - if (idx > self->m_Version.max_size()) + if (idx > m_Version.max_size()) return false; - self->m_Version[idx - 1] = i; + m_Version[idx - 1] = i; } else return false; diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index 40537b27d..a83b23932 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -1877,15 +1877,14 @@ namespace llarp f.S = m->seqno; f.F = p->intro.pathID; transfer->P = replyIntro.pathID; - auto self = this; - Router()->QueueWork([transfer, p, m, K, self]() { - if (not transfer->T.EncryptAndSign(*m, K, self->m_Identity)) + Router()->QueueWork([transfer, p, m, K, this]() { + if (not transfer->T.EncryptAndSign(*m, K, m_Identity)) { LogError("failed to encrypt and sign for sessionn T=", transfer->T.T); return; } - self->m_SendQueue.tryPushBack(SendEvent_t{transfer, p}); - self->Router()->TriggerPump(); + m_SendQueue.tryPushBack(SendEvent_t{transfer, p}); + Router()->TriggerPump(); }); return true; } @@ -1927,10 +1926,10 @@ namespace llarp traffic[remote].emplace_back(data, t); EnsurePathToService( remote, - [self = this](Address addr, OutboundContext* ctx) { + [this](Address addr, OutboundContext* ctx) { if (ctx) { - for (auto& pending : self->m_state->m_PendingTraffic[addr]) + for (auto& pending : m_state->m_PendingTraffic[addr]) { ctx->AsyncEncryptAndSendTo(pending.Buffer(), pending.protocol); } @@ -1939,7 +1938,7 @@ namespace llarp { LogWarn("no path made to ", addr); } - self->m_state->m_PendingTraffic.erase(addr); + m_state->m_PendingTraffic.erase(addr); }, PathAlignmentTimeout()); return true; From 0924dd8862a7b1bde3a2d9d37703e74290cd3ff2 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 14 Nov 2021 11:03:20 -0400 Subject: [PATCH 37/53] Remove dead declaration --- llarp/handlers/tun.hpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/llarp/handlers/tun.hpp b/llarp/handlers/tun.hpp index 434a3314b..a5173c17d 100644 --- a/llarp/handlers/tun.hpp +++ b/llarp/handlers/tun.hpp @@ -112,10 +112,6 @@ namespace llarp HandleWriteIPPacket( const llarp_buffer_t& buf, huint128_t src, huint128_t dst, uint64_t seqno); - /// queue outbound packet to the world - bool - QueueOutboundTraffic(llarp::net::IPPacket&& pkt); - /// we got a packet from the user void HandleGotUserPacket(llarp::net::IPPacket pkt); From b3a8160c8974cb58554aaff31b7b2fceb7608867 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 14 Nov 2021 11:03:33 -0400 Subject: [PATCH 38/53] Pump hidden context sooner This has to happen before pumping links because it is putting things on links. --- llarp/router/router.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index e7ca84ad8..948f3fb1c 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -87,9 +87,9 @@ namespace llarp llarp::LogTrace("Router::PumpLL() start"); if (_stopping.load()) return; + _hiddenServiceContext.Pump(); _outboundMessageHandler.Pump(); _linkManager.PumpLinks(); - _hiddenServiceContext.Pump(); llarp::LogTrace("Router::PumpLL() end"); } From 8cdde74020461267101bf038c7e20b3503948bbc Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 14 Nov 2021 11:51:44 -0400 Subject: [PATCH 39/53] Simplify --- llarp/service/sendcontext.cpp | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/llarp/service/sendcontext.cpp b/llarp/service/sendcontext.cpp index e3e789bb5..8dfa8b745 100644 --- a/llarp/service/sendcontext.cpp +++ b/llarp/service/sendcontext.cpp @@ -43,22 +43,19 @@ namespace llarp std::unordered_set flushpaths; auto rttRMS = 0ms; { - do + while (auto maybe = m_SendQueue.tryPopFront()) { - auto maybe = m_SendQueue.tryPopFront(); - if (not maybe) - break; - auto& item = *maybe; - item.first->S = item.second->NextSeqNo(); - if (item.second->SendRoutingMessage(*item.first, r)) + auto& [msg, path] = *maybe; + msg->S = path->NextSeqNo(); + if (path->SendRoutingMessage(*msg, r)) { lastGoodSend = r->Now(); - flushpaths.emplace(item.second); - m_Endpoint->ConvoTagTX(item.first->T.T); - const auto rtt = (item.second->intro.latency + remoteIntro.latency) * 2; + flushpaths.emplace(path); + m_Endpoint->ConvoTagTX(msg->T.T); + const auto rtt = (path->intro.latency + remoteIntro.latency) * 2; rttRMS += rtt * rtt.count(); } - } while (not m_SendQueue.empty()); + } } // flush the select path's upstream for (const auto& path : flushpaths) From 846577a0aad68c2c6e73724d6892403c816b65fb Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 14 Nov 2021 14:57:34 -0400 Subject: [PATCH 40/53] Add missing inbound traffic pump trigger --- llarp/service/endpoint.cpp | 1 + llarp/service/endpoint.hpp | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index a83b23932..33407d0cc 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -1167,6 +1167,7 @@ namespace llarp || (msg->proto == ProtocolType::QUIC and m_quic)) { m_InboundTrafficQueue.tryPushBack(std::move(msg)); + Router()->TriggerPump(); return true; } if (msg->proto == ProtocolType::Control) diff --git a/llarp/service/endpoint.hpp b/llarp/service/endpoint.hpp index 56f255f69..cfb5a2ef9 100644 --- a/llarp/service/endpoint.hpp +++ b/llarp/service/endpoint.hpp @@ -244,10 +244,6 @@ namespace llarp std::string service, std::function)> resultHandler) override; - /// called when something needs a pump to trigger the pump idempotently - void - TriggerPump(); - /// called on event loop pump virtual void Pump(llarp_time_t now); From bce7e2e56d6cf5f715f31fad0d6d3c8789726542 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 14 Nov 2021 15:12:29 -0400 Subject: [PATCH 41/53] Pump outbound context --- llarp/service/endpoint.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index 33407d0cc..15d1be3f2 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -1621,7 +1621,8 @@ namespace llarp return true; } - void Endpoint::Pump(llarp_time_t) + void + Endpoint::Pump(llarp_time_t now) { FlushRecvData(); // send downstream packets to user for snode @@ -1660,7 +1661,10 @@ namespace llarp auto router = Router(); // TODO: locking on this container for (const auto& [addr, outctx] : m_state->m_RemoteSessions) + { outctx->FlushUpstream(); + outctx->Pump(now); + } // TODO: locking on this container for (const auto& [router, session] : m_state->m_SNodeSessions) session->FlushUpstream(); From fbc5cb0a6d95ef6cd559ca893244c6d40d0038d1 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 14 Nov 2021 15:42:15 -0400 Subject: [PATCH 42/53] Add another missing pump trigger --- llarp/service/endpoint.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index 15d1be3f2..085f3847a 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -1097,6 +1097,7 @@ namespace llarp m_router->loop()->call_soon([this] { FlushRecvData(); }); } m_RecvQueue.tryPushBack(std::move(ev)); + Router()->TriggerPump(); } bool From 70553c762771ba5d7d7ca3ea5b7721982944d31d Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 15 Nov 2021 10:29:38 -0400 Subject: [PATCH 43/53] Remove hopeless flush The TriggerPump just below this is *already* going to trigger a flush, so the extra flush call here can't do anything useful (and in particular, it won't clear up the queue *immediately*, which is what this code looks like it was aimed at doing). --- llarp/service/endpoint.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index 085f3847a..f51a7ddc0 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -1092,10 +1092,6 @@ namespace llarp void Endpoint::QueueRecvData(RecvDataEvent ev) { - if (m_RecvQueue.full() or m_RecvQueue.empty()) - { - m_router->loop()->call_soon([this] { FlushRecvData(); }); - } m_RecvQueue.tryPushBack(std::move(ev)); Router()->TriggerPump(); } From 0fe7153f6efb7081e71de5f1230199b69795bc32 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 15 Nov 2021 10:20:57 -0400 Subject: [PATCH 44/53] Fix client latency bug; De-shared_ptr IHopHandler queues - Replace m_FlushWakeup with a call to the router's god mode pump method. m_FlushWakeup apparently isn't enough to get things out, and we can end up with incoming packets that don't get properly handled right away without it. - The shared_ptr around the ihophandler queues isn't needed and is just adding a layer of obfuscation; instead just exchange the list directly into the lambda. - Use std::exchange rather than swap - A couple other small code cleanups. --- llarp/path/ihophandler.cpp | 10 ++------- llarp/path/ihophandler.hpp | 9 ++++---- llarp/path/path.cpp | 32 ++++++++++++++-------------- llarp/path/path.hpp | 4 ++-- llarp/path/transit_hop.cpp | 26 +++++++++++------------ llarp/path/transit_hop.hpp | 4 ++-- llarp/service/endpoint.cpp | 9 +++----- llarp/service/sendcontext.cpp | 39 +++++++++++++++++------------------ llarp/service/sendcontext.hpp | 2 -- 9 files changed, 59 insertions(+), 76 deletions(-) diff --git a/llarp/path/ihophandler.cpp b/llarp/path/ihophandler.cpp index e55e0d850..3b7055cb9 100644 --- a/llarp/path/ihophandler.cpp +++ b/llarp/path/ihophandler.cpp @@ -9,10 +9,7 @@ namespace llarp bool IHopHandler::HandleUpstream(const llarp_buffer_t& X, const TunnelNonce& Y, AbstractRouter* r) { - if (m_UpstreamQueue == nullptr) - m_UpstreamQueue = std::make_shared(); - m_UpstreamQueue->emplace_back(); - auto& pkt = m_UpstreamQueue->back(); + auto& pkt = m_UpstreamQueue.emplace_back(); pkt.first.resize(X.sz); std::copy_n(X.base, X.sz, pkt.first.begin()); pkt.second = Y; @@ -24,10 +21,7 @@ namespace llarp bool IHopHandler::HandleDownstream(const llarp_buffer_t& X, const TunnelNonce& Y, AbstractRouter* r) { - if (m_DownstreamQueue == nullptr) - m_DownstreamQueue = std::make_shared(); - m_DownstreamQueue->emplace_back(); - auto& pkt = m_DownstreamQueue->back(); + auto& pkt = m_DownstreamQueue.emplace_back(); pkt.first.resize(X.sz); std::copy_n(X.base, X.sz, pkt.first.begin()); pkt.second = Y; diff --git a/llarp/path/ihophandler.hpp b/llarp/path/ihophandler.hpp index 7fb7e740d..0465ca9fb 100644 --- a/llarp/path/ihophandler.hpp +++ b/llarp/path/ihophandler.hpp @@ -26,7 +26,6 @@ namespace llarp { using TrafficEvent_t = std::pair, TunnelNonce>; using TrafficQueue_t = std::list; - using TrafficQueue_ptr = std::shared_ptr; virtual ~IHopHandler() = default; @@ -74,16 +73,16 @@ namespace llarp protected: uint64_t m_SequenceNum = 0; - TrafficQueue_ptr m_UpstreamQueue; - TrafficQueue_ptr m_DownstreamQueue; + TrafficQueue_t m_UpstreamQueue; + TrafficQueue_t m_DownstreamQueue; util::DecayingHashSet m_UpstreamReplayFilter; util::DecayingHashSet m_DownstreamReplayFilter; virtual void - UpstreamWork(TrafficQueue_ptr queue, AbstractRouter* r) = 0; + UpstreamWork(TrafficQueue_t queue, AbstractRouter* r) = 0; virtual void - DownstreamWork(TrafficQueue_ptr queue, AbstractRouter* r) = 0; + DownstreamWork(TrafficQueue_t queue, AbstractRouter* r) = 0; virtual void HandleAllUpstream(std::vector msgs, AbstractRouter* r) = 0; diff --git a/llarp/path/path.cpp b/llarp/path/path.cpp index 36491671c..d0b5ea72f 100644 --- a/llarp/path/path.cpp +++ b/llarp/path/path.cpp @@ -29,7 +29,7 @@ namespace llarp std::weak_ptr pathset, PathRole startingRoles, std::string shortName) - : m_PathSet{pathset}, _role{startingRoles}, m_shortName{std::move(shortName)} + : m_PathSet{std::move(pathset)}, _role{startingRoles}, m_shortName{std::move(shortName)} { hops.resize(h.size()); @@ -492,11 +492,11 @@ namespace llarp } void - Path::UpstreamWork(TrafficQueue_ptr msgs, AbstractRouter* r) + Path::UpstreamWork(TrafficQueue_t msgs, AbstractRouter* r) { - std::vector sendmsgs(msgs->size()); + std::vector sendmsgs(msgs.size()); size_t idx = 0; - for (auto& ev : *msgs) + for (auto& ev : msgs) { const llarp_buffer_t buf(ev.first); TunnelNonce n = ev.second; @@ -519,24 +519,22 @@ namespace llarp void Path::FlushUpstream(AbstractRouter* r) { - if (m_UpstreamQueue && not m_UpstreamQueue->empty()) + if (not m_UpstreamQueue.empty()) { - TrafficQueue_ptr data = nullptr; - std::swap(m_UpstreamQueue, data); - r->QueueWork( - [self = shared_from_this(), data, r]() { self->UpstreamWork(std::move(data), r); }); + r->QueueWork([self = shared_from_this(), + data = std::exchange(m_UpstreamQueue, {}), + r]() mutable { self->UpstreamWork(std::move(data), r); }); } } void Path::FlushDownstream(AbstractRouter* r) { - if (m_DownstreamQueue && not m_DownstreamQueue->empty()) + if (not m_DownstreamQueue.empty()) { - TrafficQueue_ptr data = nullptr; - std::swap(m_DownstreamQueue, data); - r->QueueWork( - [self = shared_from_this(), data, r]() { self->DownstreamWork(std::move(data), r); }); + r->QueueWork([self = shared_from_this(), + data = std::exchange(m_DownstreamQueue, {}), + r]() mutable { self->DownstreamWork(std::move(data), r); }); } } @@ -570,11 +568,11 @@ namespace llarp } void - Path::DownstreamWork(TrafficQueue_ptr msgs, AbstractRouter* r) + Path::DownstreamWork(TrafficQueue_t msgs, AbstractRouter* r) { - std::vector sendMsgs(msgs->size()); + std::vector sendMsgs(msgs.size()); size_t idx = 0; - for (auto& ev : *msgs) + for (auto& ev : msgs) { const llarp_buffer_t buf(ev.first); sendMsgs[idx].Y = ev.second; diff --git a/llarp/path/path.hpp b/llarp/path/path.hpp index da9d2c2f4..d1396b8f2 100644 --- a/llarp/path/path.hpp +++ b/llarp/path/path.hpp @@ -388,10 +388,10 @@ namespace llarp protected: void - UpstreamWork(TrafficQueue_ptr queue, AbstractRouter* r) override; + UpstreamWork(TrafficQueue_t queue, AbstractRouter* r) override; void - DownstreamWork(TrafficQueue_ptr queue, AbstractRouter* r) override; + DownstreamWork(TrafficQueue_t queue, AbstractRouter* r) override; void HandleAllUpstream(std::vector msgs, AbstractRouter* r) override; diff --git a/llarp/path/transit_hop.cpp b/llarp/path/transit_hop.cpp index a04c3537e..f2cd2d1aa 100644 --- a/llarp/path/transit_hop.cpp +++ b/llarp/path/transit_hop.cpp @@ -104,7 +104,7 @@ namespace llarp } void - TransitHop::DownstreamWork(TrafficQueue_ptr msgs, AbstractRouter* r) + TransitHop::DownstreamWork(TrafficQueue_t msgs, AbstractRouter* r) { auto flushIt = [self = shared_from_this(), r]() { std::vector msgs; @@ -114,7 +114,7 @@ namespace llarp } self->HandleAllDownstream(std::move(msgs), r); }; - for (auto& ev : *msgs) + for (auto& ev : msgs) { RelayDownstreamMessage msg; const llarp_buffer_t buf(ev.first); @@ -140,9 +140,9 @@ namespace llarp } void - TransitHop::UpstreamWork(TrafficQueue_ptr msgs, AbstractRouter* r) + TransitHop::UpstreamWork(TrafficQueue_t msgs, AbstractRouter* r) { - for (auto& ev : *msgs) + for (auto& ev : msgs) { const llarp_buffer_t buf(ev.first); RelayUpstreamMessage msg; @@ -223,25 +223,23 @@ namespace llarp void TransitHop::FlushUpstream(AbstractRouter* r) { - if (m_UpstreamQueue && not m_UpstreamQueue->empty()) + if (not m_UpstreamQueue.empty()) { - r->QueueWork([self = shared_from_this(), data = std::move(m_UpstreamQueue), r]() mutable { - self->UpstreamWork(std::move(data), r); - }); + r->QueueWork([self = shared_from_this(), + data = std::exchange(m_UpstreamQueue, {}), + r]() mutable { self->UpstreamWork(std::move(data), r); }); } - m_UpstreamQueue = nullptr; } void TransitHop::FlushDownstream(AbstractRouter* r) { - if (m_DownstreamQueue && not m_DownstreamQueue->empty()) + if (not m_DownstreamQueue.empty()) { - r->QueueWork([self = shared_from_this(), data = std::move(m_DownstreamQueue), r]() mutable { - self->DownstreamWork(std::move(data), r); - }); + r->QueueWork([self = shared_from_this(), + data = std::exchange(m_DownstreamQueue, {}), + r]() mutable { self->DownstreamWork(std::move(data), r); }); } - m_DownstreamQueue = nullptr; } /// this is where a DHT message is handled at the end of a path, that is, diff --git a/llarp/path/transit_hop.hpp b/llarp/path/transit_hop.hpp index b9a8f8566..798c2d063 100644 --- a/llarp/path/transit_hop.hpp +++ b/llarp/path/transit_hop.hpp @@ -190,10 +190,10 @@ namespace llarp protected: void - UpstreamWork(TrafficQueue_ptr queue, AbstractRouter* r) override; + UpstreamWork(TrafficQueue_t queue, AbstractRouter* r) override; void - DownstreamWork(TrafficQueue_ptr queue, AbstractRouter* r) override; + DownstreamWork(TrafficQueue_t queue, AbstractRouter* r) override; void HandleAllUpstream(std::vector msgs, AbstractRouter* r) override; diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index f51a7ddc0..aae60423d 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -1079,14 +1079,11 @@ namespace llarp void Endpoint::FlushRecvData() { - do + while (auto maybe = m_RecvQueue.tryPopFront()) { - auto maybe = m_RecvQueue.tryPopFront(); - if (not maybe) - return; - auto ev = std::move(*maybe); + auto& ev = *maybe; ProtocolMessage::ProcessAsync(ev.fromPath, ev.pathid, ev.msg); - } while (true); + } } void diff --git a/llarp/service/sendcontext.cpp b/llarp/service/sendcontext.cpp index 8dfa8b745..16b524c16 100644 --- a/llarp/service/sendcontext.cpp +++ b/llarp/service/sendcontext.cpp @@ -21,19 +21,20 @@ namespace llarp , m_Endpoint(ep) , createdAt(ep->Now()) , m_SendQueue(SendContextQueueSize) - { - m_FlushWakeup = ep->Loop()->make_waker([this] { FlushUpstream(); }); - } + {} bool SendContext::Send(std::shared_ptr msg, path::Path_ptr path) { - if (not path->IsReady()) - return false; - m_FlushWakeup->Trigger(); - return m_SendQueue.tryPushBack(std::make_pair( - std::make_shared(*msg, remoteIntro.pathID), path)) - == thread::QueueReturn::Success; + if (path->IsReady() + and m_SendQueue.tryPushBack(std::make_pair( + std::make_shared(*msg, remoteIntro.pathID), path)) + == thread::QueueReturn::Success) + { + m_Endpoint->Router()->TriggerPump(); + return true; + } + return false; } void @@ -42,19 +43,17 @@ namespace llarp auto r = m_Endpoint->Router(); std::unordered_set flushpaths; auto rttRMS = 0ms; + while (auto maybe = m_SendQueue.tryPopFront()) { - while (auto maybe = m_SendQueue.tryPopFront()) + auto& [msg, path] = *maybe; + msg->S = path->NextSeqNo(); + if (path->SendRoutingMessage(*msg, r)) { - auto& [msg, path] = *maybe; - msg->S = path->NextSeqNo(); - if (path->SendRoutingMessage(*msg, r)) - { - lastGoodSend = r->Now(); - flushpaths.emplace(path); - m_Endpoint->ConvoTagTX(msg->T.T); - const auto rtt = (path->intro.latency + remoteIntro.latency) * 2; - rttRMS += rtt * rtt.count(); - } + lastGoodSend = r->Now(); + flushpaths.emplace(path); + m_Endpoint->ConvoTagTX(msg->T.T); + const auto rtt = (path->intro.latency + remoteIntro.latency) * 2; + rttRMS += rtt * rtt.count(); } } // flush the select path's upstream diff --git a/llarp/service/sendcontext.hpp b/llarp/service/sendcontext.hpp index cc0ab4a69..0728d7fc5 100644 --- a/llarp/service/sendcontext.hpp +++ b/llarp/service/sendcontext.hpp @@ -56,8 +56,6 @@ namespace llarp std::function authResultListener; - std::shared_ptr m_FlushWakeup; - virtual bool ShiftIntroduction(bool rebuild = true) { From f63fff37a336a0211699ab78adc1cfbaa4c72e83 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 15 Nov 2021 16:20:07 -0400 Subject: [PATCH 45/53] Remove broken test This test isn't very useful, but also broken with the other changes on this branch. --- test/CMakeLists.txt | 1 - test/iwp/test_iwp_session.cpp | 312 ---------------------------------- 2 files changed, 313 deletions(-) delete mode 100644 test/iwp/test_iwp_session.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0c6c43870..8ac70868a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -22,7 +22,6 @@ add_executable(testAll crypto/test_llarp_crypto.cpp crypto/test_llarp_key_manager.cpp dns/test_llarp_dns_dns.cpp - iwp/test_iwp_session.cpp net/test_ip_address.cpp net/test_llarp_net.cpp net/test_sock_addr.cpp diff --git a/test/iwp/test_iwp_session.cpp b/test/iwp/test_iwp_session.cpp deleted file mode 100644 index 9cf248d07..000000000 --- a/test/iwp/test_iwp_session.cpp +++ /dev/null @@ -1,312 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include "ev/ev.hpp" - -#undef LOG_TAG -#define LOG_TAG __FILE__ - -namespace iwp = llarp::iwp; -namespace util = llarp::util; - -/// make an iwp link -template -static llarp::LinkLayer_ptr -make_link(Args&&... args) -{ - if (inbound) - return iwp::NewInboundLink(std::forward(args)...); - return iwp::NewOutboundLink(std::forward(args)...); -} - -/// a single iwp link with associated keys and members to make unit tests work -struct IWPLinkContext -{ - llarp::RouterContact rc; - llarp::IpAddress localAddr; - llarp::LinkLayer_ptr link; - std::shared_ptr keyManager; - llarp::LinkMessageParser m_Parser; - llarp::EventLoop_ptr m_Loop; - /// is the test done on this context ? - bool gucci = false; - - IWPLinkContext(std::string_view addr, llarp::EventLoop_ptr loop) - : localAddr{std::move(addr)} - , keyManager{std::make_shared()} - , m_Parser{nullptr} - , m_Loop{std::move(loop)} - { - // generate keys - llarp::CryptoManager::instance()->identity_keygen(keyManager->identityKey); - llarp::CryptoManager::instance()->encryption_keygen(keyManager->encryptionKey); - llarp::CryptoManager::instance()->encryption_keygen(keyManager->transportKey); - - // set keys in rc - rc.pubkey = keyManager->identityKey.toPublic(); - rc.enckey = keyManager->encryptionKey.toPublic(); - } - - template - void - Call(Func_t work) - { - m_Loop->call_soon(std::move(work)); - } - - bool - HandleMessage(llarp::ILinkSession* from, const llarp_buffer_t& buf) - { - return m_Parser.ProcessFrom(from, buf); - } - - /// initialize link - template - void - InitLink(std::function established) - { - link = make_link( - keyManager, - m_Loop, - // getrc - [&]() -> const llarp::RouterContact& { return rc; }, - // link message handler - util::memFn(&IWPLinkContext::HandleMessage, this), - // sign buffer - [&](llarp::Signature& sig, const llarp_buffer_t& buf) { - REQUIRE(llarp::CryptoManager::instance()->sign(sig, keyManager->identityKey, buf)); - return true; - }, - // before connect - nullptr, - // established handler - [established](llarp::ILinkSession* s, bool linkIsInbound) { - REQUIRE(s != nullptr); - REQUIRE(inbound == linkIsInbound); - established(s); - return true; - }, - // renegotiate handler - [](llarp::RouterContact newrc, llarp::RouterContact oldrc) { - REQUIRE(newrc.pubkey == oldrc.pubkey); - return true; - }, - // timeout handler - [&](llarp::ILinkSession*) { - m_Loop->stop(); - FAIL("session timeout"); - }, - // session closed handler - [](llarp::RouterID) {}, - // pump done handler - []() {}, - // do work function - [l = m_Loop](llarp::Work_t work) { l->call_soon(work); }); - REQUIRE(link->Configure( - m_Loop, llarp::net::LoopbackInterfaceName(), AF_INET, *localAddr.getPort())); - - if (inbound) - { - // only add address info on the recipient's rc - rc.addrs.emplace_back(); - REQUIRE(link->GetOurAddressInfo(rc.addrs.back())); - } - // sign rc - REQUIRE(rc.Sign(keyManager->identityKey)); - REQUIRE(keyManager != nullptr); - } -}; - -using Context_ptr = std::shared_ptr; - -/// run an iwp unit test after setup -/// call take 2 parameters, test and a timeout -/// -/// test is a callable that takes 5 arguments: -/// 0) std::function that starts the iwp links and gives an event loop to call with -/// 1) std::function that ends the unit test if we are done -/// 2) std::function that ends the unit test right now as a success -/// 3) client iwp link context (shared_ptr) -/// 4) relay iwp link context (shared_ptr) -/// -/// timeout is a std::chrono::duration that tells the driver how long to run the unit test for -/// before it should assume failure of unit test -template -void -RunIWPTest(Func_t test, Duration_t timeout = 10s) -{ - // shut up logs - llarp::LogSilencer shutup; - // set up event loop - auto loop = llarp::EventLoop::create(); - - llarp::LogContext::Instance().Initialize( - llarp::eLogDebug, llarp::LogType::File, "stdout", "unit test", [loop](auto work) { - loop->call_soon(work); - }); - - // turn off bogon blocking - auto oldBlockBogons = llarp::RouterContact::BlockBogons; - llarp::RouterContact::BlockBogons = false; - - // set up cryptography - llarp::sodium::CryptoLibSodium crypto{}; - llarp::CryptoManager manager{&crypto}; - - // set up client - auto initiator = std::make_shared("127.0.0.1:3001", loop); - // set up server - auto recipient = std::make_shared("127.0.0.1:3002", loop); - - // function for ending unit test on success - auto endIfDone = [initiator, recipient, loop]() { - if (initiator->gucci and recipient->gucci) - { - loop->stop(); - } - }; - // function to start test and give loop to unit test - auto start = [initiator, recipient, loop]() { - REQUIRE(initiator->link->Start()); - REQUIRE(recipient->link->Start()); - return loop; - }; - - // function to end test immediately - auto endTest = [loop] { loop->stop(); }; - - loop->call_later(timeout, [] { FAIL("test timeout"); }); - test(start, endIfDone, endTest, initiator, recipient); - loop->run(); - llarp::RouterContact::BlockBogons = oldBlockBogons; -} - -/// ensure clients can connect to relays -TEST_CASE("IWP handshake", "[iwp]") -{ - RunIWPTest([](std::function start, - std::function endIfDone, - [[maybe_unused]] std::function endTestNow, - Context_ptr alice, - Context_ptr bob) { - // set up initiator - alice->InitLink([=](auto remote) { - REQUIRE(remote->GetRemoteRC() == bob->rc); - alice->gucci = true; - endIfDone(); - }); - // set up recipient - bob->InitLink([=](auto remote) { - REQUIRE(remote->GetRemoteRC() == alice->rc); - bob->gucci = true; - endIfDone(); - }); - // start unit test - auto loop = start(); - // try establishing a session - loop->call([link = alice->link, rc = bob->rc]() { REQUIRE(link->TryEstablishTo(rc)); }); - }); -} - -/// ensure relays cannot connect to clients -TEST_CASE("IWP handshake reverse", "[iwp]") -{ - RunIWPTest([](std::function start, - [[maybe_unused]] std::function endIfDone, - std::function endTestNow, - Context_ptr alice, - Context_ptr bob) { - alice->InitLink([](auto) {}); - bob->InitLink([](auto) {}); - // start unit test - auto loop = start(); - // try establishing a session in the wrong direction - loop->call([link = bob->link, rc = alice->rc, endTestNow] { - REQUIRE(not link->TryEstablishTo(rc)); - endTestNow(); - }); - }); -} - -/// ensure iwp can send messages between sessions -TEST_CASE("IWP send messages", "[iwp]") -{ - int aliceNumSent = 0; - int bobNumSent = 0; - RunIWPTest([&aliceNumSent, &bobNumSent](std::function start, - std::function endIfDone, - std::function endTestNow, - Context_ptr alice, - Context_ptr bob) { - constexpr int numSend = 64; - // when alice makes a session to bob send `aliceNumSend` messages to him - alice->InitLink([endIfDone, alice, &aliceNumSent](auto session) { - for (auto index = 0; index < numSend; index++) - { - alice->Call([session, endIfDone, alice, &aliceNumSent]() { - // generate a discard message that is 512 bytes long - llarp::DiscardMessage msg; - std::vector msgBuff(512); - llarp_buffer_t buf(msgBuff); - // add random padding - llarp::CryptoManager::instance()->randomize(buf); - // encode the discard message - msg.BEncode(&buf); - // send the message - session->SendMessageBuffer(msgBuff, [endIfDone, alice, &aliceNumSent](auto status) { - if (status == llarp::ILinkSession::DeliveryStatus::eDeliverySuccess) - { - // on successful transmit increment the number we sent - aliceNumSent++; - } - // if we sent all the messages sucessfully we end the unit test - alice->gucci = aliceNumSent == numSend; - endIfDone(); - }); - }); - } - }); - bob->InitLink([endIfDone, bob, &bobNumSent](auto session) { - for (auto index = 0; index < numSend; index++) - { - bob->Call([session, endIfDone, bob, &bobNumSent]() { - // generate a discard message that is 512 bytes long - llarp::DiscardMessage msg; - std::vector msgBuff(512); - llarp_buffer_t buf(msgBuff); - // add random padding - llarp::CryptoManager::instance()->randomize(buf); - // encode the discard message - msg.BEncode(&buf); - // send the message - session->SendMessageBuffer(msgBuff, [endIfDone, bob, &bobNumSent](auto status) { - if (status == llarp::ILinkSession::DeliveryStatus::eDeliverySuccess) - { - // on successful transmit increment the number we sent - bobNumSent++; - } - // if we sent all the messages sucessfully we end the unit test - bob->gucci = bobNumSent == numSend; - endIfDone(); - }); - }); - } - }); - // start unit test - auto loop = start(); - // try establishing a session from alice to bob - loop->call([link = alice->link, rc = bob->rc, endTestNow]() { - REQUIRE(link->TryEstablishTo(rc)); - }); - }); -} From 711038d1506b2a64416ff4d6773de236df4f3033 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 15 Nov 2021 16:57:17 -0400 Subject: [PATCH 46/53] Remove debug comment --- llarp/handlers/tun.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/llarp/handlers/tun.cpp b/llarp/handlers/tun.cpp index 5c49ca216..4e1ed390f 100644 --- a/llarp/handlers/tun.cpp +++ b/llarp/handlers/tun.cpp @@ -1070,7 +1070,6 @@ namespace llarp // send icmp unreachable as we dont have any exits for this ip if (const auto icmp = pkt.MakeICMPUnreachable()) { - /// ? HandleWriteIPPacket(icmp->ConstBuffer(), dst, src, 0); } return; From 687b54f860d025aafbdb1db8a4be3fbdaa754952 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 15 Nov 2021 17:15:16 -0400 Subject: [PATCH 47/53] Abstract & simplify logic --- llarp/handlers/tun.cpp | 8 ++------ llarp/net/ip_range.hpp | 8 ++++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/llarp/handlers/tun.cpp b/llarp/handlers/tun.cpp index 4e1ed390f..5677fc12b 100644 --- a/llarp/handlers/tun.cpp +++ b/llarp/handlers/tun.cpp @@ -1077,11 +1077,7 @@ namespace llarp service::Address addr{}; for (const auto& [range, exitAddr] : exitEntries) { - if ( - // we permit this because it matches our rules and we allow bogons: - (range.BogonRange() and range.Contains(dst)) - // allow because the destination is not a bogon and the mapped range is not a bogon - or not IsBogon(dst)) + if (not IsBogon(dst) or range.BogonContains(dst)) { addr = exitAddr; } @@ -1274,7 +1270,7 @@ namespace llarp bool allow = false; for (const auto& [range, exitAddr] : mapped) { - if ((range.BogonRange() and range.Contains(src)) or not IsBogon(src)) + if (not IsBogon(src) or range.BogonContains(src)) { // allow if this address matches the endpoint we think it should be allow = exitAddr == fromAddr; diff --git a/llarp/net/ip_range.hpp b/llarp/net/ip_range.hpp index 7a3f9e6df..1a9b98683 100644 --- a/llarp/net/ip_range.hpp +++ b/llarp/net/ip_range.hpp @@ -52,6 +52,14 @@ namespace llarp return IsBogon(addr) or IsBogon(HighestAddr()); } + /// return true if we intersect with a bogon range *and* we contain the given address + template + bool + BogonContains(Addr&& addr) const + { + return BogonRange() and Contains(std::forward(addr)); + } + /// return the number of bits set in the hostmask constexpr int HostmaskBits() const From 09c2b31b250895cfdc92c399209188e6d9ea134c Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 15 Nov 2021 18:23:42 -0400 Subject: [PATCH 48/53] Move paths pumping into PumpLL --- llarp/router/router.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 948f3fb1c..8729bf0a0 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -69,11 +69,7 @@ namespace llarp _running.store(false); _lastTick = llarp::time_now_ms(); m_NextExploreAt = Clock_t::now(); - m_Pump = _loop->make_waker([this]() { - paths.PumpDownstream(); - paths.PumpUpstream(); - PumpLL(); - }); + m_Pump = _loop->make_waker([this]() { PumpLL(); }); } Router::~Router() @@ -87,6 +83,8 @@ namespace llarp llarp::LogTrace("Router::PumpLL() start"); if (_stopping.load()) return; + paths.PumpDownstream(); + paths.PumpUpstream(); _hiddenServiceContext.Pump(); _outboundMessageHandler.Pump(); _linkManager.PumpLinks(); From 7a04911b9b49d2d271aa85599ce41c24ef546213 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 9 Nov 2021 13:52:03 -0400 Subject: [PATCH 49/53] Avoid std::function in hot pump code We're seeing overhead here of the std::function invocation, which we can easily avoid in this case by not using a std::function around the callback. --- llarp/path/path_context.hpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/llarp/path/path_context.hpp b/llarp/path/path_context.hpp index 50a791521..b53e11a06 100644 --- a/llarp/path/path_context.hpp +++ b/llarp/path/path_context.hpp @@ -125,8 +125,11 @@ namespace llarp Mutex_t first; // protects second TransitHopsMap_t second GUARDED_BY(first); + /// Invokes a callback for each transit path; visit must be invokable with a `const + /// TransitHop_ptr&` argument. + template void - ForEach(std::function visit) EXCLUDES(first) + ForEach(TransitHopVisitor&& visit) EXCLUDES(first) { Lock_t lock(first); for (const auto& item : second) @@ -142,8 +145,11 @@ namespace llarp util::Mutex first; // protects second OwnedPathsMap_t second GUARDED_BY(first); + /// Invokes a callback for each owned path; visit must be invokable with a `const Path_ptr&` + /// argument. + template void - ForEach(std::function visit) + ForEach(OwnedHopVisitor&& visit) { util::Lock lock(first); for (const auto& item : second) From b2ee003329e01a160f28678416b82abe46f54db3 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 9 Nov 2021 15:06:20 -0400 Subject: [PATCH 50/53] Optimization: eliminate unneeded shared_ptrs We have a few cases where we're making an extra shared_ptr which we copy into a lambda, which then results in an extra unnecessary refcount decrement in the parent; this changes them to give an rvalue reference to the lambda to avoid the extra incr/decr instead. The one in Session::Pump is particularly noticeable and shows up in profiling. --- llarp/exit/session.cpp | 3 +++ llarp/iwp/session.cpp | 14 +++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/llarp/exit/session.cpp b/llarp/exit/session.cpp index 485ca768f..b2370002a 100644 --- a/llarp/exit/session.cpp +++ b/llarp/exit/session.cpp @@ -137,6 +137,9 @@ namespace llarp void BaseSession::CallPendingCallbacks(bool success) { + if (m_PendingCallbacks.empty()) + return; + if (success) { auto self = shared_from_this(); diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp index b3999895e..a3e504222 100644 --- a/llarp/iwp/session.cpp +++ b/llarp/iwp/session.cpp @@ -97,9 +97,8 @@ namespace llarp m_RemoteRC = msg->rc; GotLIM = util::memFn(&Session::GotRenegLIM, this); - auto self = shared_from_this(); - assert(self.use_count() > 1); - SendOurLIM([self](ILinkSession::DeliveryStatus st) { + assert(shared_from_this().use_count() > 1); + SendOurLIM([self = shared_from_this()](ILinkSession::DeliveryStatus st) { if (st == ILinkSession::DeliveryStatus::eDeliverySuccess) { self->m_State = State::Ready; @@ -259,17 +258,18 @@ namespace llarp } } } - auto self = shared_from_this(); - assert(self.use_count() > 1); + assert(shared_from_this().use_count() > 1); if (not m_EncryptNext.empty()) { - m_Parent->QueueWork([self, data = m_EncryptNext] { self->EncryptWorker(data); }); + m_Parent->QueueWork( + [self = shared_from_this(), data = m_EncryptNext] { self->EncryptWorker(data); }); m_EncryptNext.clear(); } if (not m_DecryptNext.empty()) { - m_Parent->QueueWork([self, data = m_DecryptNext] { self->DecryptWorker(data); }); + m_Parent->QueueWork( + [self = shared_from_this(), data = m_DecryptNext] { self->DecryptWorker(data); }); m_DecryptNext.clear(); } } From 51151620661e6b2a204a139a72fea86f839ea123 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 9 Nov 2021 15:37:42 -0400 Subject: [PATCH 51/53] Use libuv's cached current time for `time_now()` We are calling time_now() a huge amount, and it is a major consumer of CPU cycles, but we don't need it: most of the time the current event loop time is enough. --- llarp/ev/ev.hpp | 8 ++++---- llarp/ev/ev_libuv.hpp | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/llarp/ev/ev.hpp b/llarp/ev/ev.hpp index f39d3041f..898d526a3 100644 --- a/llarp/ev/ev.hpp +++ b/llarp/ev/ev.hpp @@ -75,11 +75,11 @@ namespace llarp virtual bool running() const = 0; + // Returns a current steady clock time value representing the current time with event loop tick + // granularity. That is, the value is typically only updated at the beginning of an event loop + // tick. virtual llarp_time_t - time_now() const - { - return llarp::time_now_ms(); - } + time_now() const = 0; // Calls a function/lambda/etc. If invoked from within the event loop itself this calls the // given lambda immediately; otherwise it passes it to `call_soon()` to be queued to run at the diff --git a/llarp/ev/ev_libuv.hpp b/llarp/ev/ev_libuv.hpp index 86dacac79..62c3d5a4c 100644 --- a/llarp/ev/ev_libuv.hpp +++ b/llarp/ev/ev_libuv.hpp @@ -31,6 +31,12 @@ namespace llarp::uv bool running() const override; + llarp_time_t + time_now() const override + { + return m_Impl->now(); + } + void call_later(llarp_time_t delay_ms, std::function callback) override; From e7d282993a3ba467fda499536b87af712b18c2a4 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 16 Nov 2021 09:35:09 -0500 Subject: [PATCH 52/53] in static linux ci pipeline disable building lokinet-bootstrap and bundle a bootstrap.signed --- .drone.jsonnet | 4 ++-- contrib/ci/drone-static-upload.sh | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 744ad55bc..5758d9a57 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -305,7 +305,7 @@ local mac_builder(name, '-DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 ' + '-DCMAKE_CXX_FLAGS="-march=x86-64 -mtune=haswell" ' + '-DCMAKE_C_FLAGS="-march=x86-64 -mtune=haswell" ' + - '-DNATIVE_BUILD=OFF -DWITH_SYSTEMD=OFF', + '-DNATIVE_BUILD=OFF -DWITH_SYSTEMD=OFF -DWITH_BOOTSTRAP=OFF', extra_cmds=[ '../contrib/ci/drone-check-static-libs.sh', '../contrib/ci/drone-static-upload.sh', @@ -317,7 +317,7 @@ local mac_builder(name, deps=['g++', 'python3-dev', 'automake', 'libtool'], cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON ' + '-DCMAKE_CXX_FLAGS="-march=armv7-a+fp -Wno-psabi" -DCMAKE_C_FLAGS="-march=armv7-a+fp" ' + - '-DNATIVE_BUILD=OFF -DWITH_SYSTEMD=OFF', + '-DNATIVE_BUILD=OFF -DWITH_SYSTEMD=OFF -DWITH_BOOTSTRAP=OFF', extra_cmds=[ '../contrib/ci/drone-check-static-libs.sh', 'UPLOAD_OS=linux-armhf ../contrib/ci/drone-static-upload.sh', diff --git a/contrib/ci/drone-static-upload.sh b/contrib/ci/drone-static-upload.sh index c5d35bfaa..f2b445ddb 100755 --- a/contrib/ci/drone-static-upload.sh +++ b/contrib/ci/drone-static-upload.sh @@ -44,7 +44,8 @@ elif [ -e lokinet.apk ] ; then archive="$base.apk" cp -av lokinet.apk "$archive" else - cp -av daemon/lokinet daemon/lokinet-vpn daemon/lokinet-bootstrap "$base" + cp -av daemon/lokinet daemon/lokinet-vpn "$base" + cp -av ../contrib/bootstrap/mainnet.signed "$base/bootstrap.signed" # tar dat shiz up yo archive="$base.tar.xz" tar cJvf "$archive" "$base" From d24f89192d143d5d6f13fcd45fbe3eab8c7abeea Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 16 Nov 2021 19:50:52 -0400 Subject: [PATCH 53/53] Bump version --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ab561a0d..4d0244044 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,7 +22,7 @@ endif() project(lokinet - VERSION 0.9.7 + VERSION 0.9.8 DESCRIPTION "lokinet - IP packet onion router" LANGUAGES ${LANGS})