From 4eb275324ddcc866dbbc3967c2151a3d8b61aa07 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 10 Nov 2020 06:25:37 -0400 Subject: [PATCH] Fix deb snapshot builds + misc other CI updates (#1462) * Remove unneeded libcurl-dev build dep * Make apt-get operations quieter * apt repo updates - store the apt repo gpg pubkey in the repo rather than having to fetch it every time - add the apt repo for all the deb builds - update apt repo URL to deb.loki.network instead of deb.imaginary.stream * Use http so that we don't need ca-certificates The repo packages are already signed so using https really isn't important. * Add hack to ignore jsonnet conflicts in deb builds * Add build machine name to build output * We don't install any shared libs currently --- .drone.jsonnet | 72 ++++++++++++++++++++--------------- contrib/deb.loki.network.gpg | Bin 0 -> 2213 bytes 2 files changed, 41 insertions(+), 31 deletions(-) create mode 100644 contrib/deb.loki.network.gpg diff --git a/.drone.jsonnet b/.drone.jsonnet index 08861ac93..80217a54c 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -1,4 +1,4 @@ -local default_deps_base='libsystemd-dev python3-dev libcurl4-openssl-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev'; +local default_deps_base='libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev'; local default_deps_nocxx='libsodium-dev ' + default_deps_base; // libsodium-dev needs to be >= 1.0.18 local default_deps='g++ ' + default_deps_nocxx; // g++ sometimes needs replacement local default_windows_deps='mingw-w64 zip nsis'; @@ -10,6 +10,8 @@ local submodules = { commands: ['git fetch --tags', 'git submodule update --init --recursive --depth=1'] }; +local apt_get_quiet = 'apt-get -o=Dpkg::Use-Pty=0 -q'; + // Regular build on a debian-like system: local debian_pipeline(name, image, arch='amd64', @@ -19,7 +21,7 @@ local debian_pipeline(name, image, werror=true, cmake_extra='', extra_cmds=[], - imaginary_repo=false, + loki_repo=false, allow_fail=false) = { kind: 'pipeline', type: 'docker', @@ -34,18 +36,19 @@ local debian_pipeline(name, image, [if allow_fail then "failure"]: "ignore", environment: { SSH_KEY: { from_secret: "SSH_KEY" } }, commands: [ + 'echo "Building on ${DRONE_STAGE_MACHINE}"', 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections', - 'apt-get update', - 'apt-get install -y eatmydata', - 'eatmydata apt-get dist-upgrade -y', - ] + (if imaginary_repo then [ - 'eatmydata apt-get install -y gpg curl lsb-release', - 'echo deb https://deb.imaginary.stream $$(lsb_release -sc) main >/etc/apt/sources.list.d/imaginary.stream.list', - 'curl -s https://deb.imaginary.stream/public.gpg | apt-key add -', - 'eatmydata apt-get update' + apt_get_quiet + ' update', + apt_get_quiet + ' install -y eatmydata', + ] + (if loki_repo then [ + 'eatmydata ' + apt_get_quiet + ' install -y lsb-release', + 'cp contrib/deb.loki.network.gpg /etc/apt/trusted.gpg.d', + 'echo deb http://deb.loki.network $$(lsb_release -sc) main >/etc/apt/sources.list.d/loki.network.list', + 'eatmydata ' + apt_get_quiet + ' update' ] else [] ) + [ - 'eatmydata apt-get install -y gdb cmake git ninja-build pkg-config ccache ' + deps, + 'eatmydata ' + apt_get_quiet + ' dist-upgrade -y', + 'eatmydata ' + apt_get_quiet + ' install -y gdb cmake git ninja-build pkg-config ccache ' + deps, 'mkdir build', 'cd build', 'cmake .. -G Ninja -DCMAKE_CXX_FLAGS=-fdiagnostics-color=always -DCMAKE_BUILD_TYPE='+build_type+' ' + @@ -83,10 +86,11 @@ local windows_cross_pipeline(name, image, [if allow_fail then "failure"]: "ignore", environment: { SSH_KEY: { from_secret: "SSH_KEY" }, WINDOWS_BUILD_NAME: toolchain+"bit" }, commands: [ + 'echo "Building on ${DRONE_STAGE_MACHINE}"', 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections', - 'apt-get update', - 'apt-get install -y eatmydata', - 'eatmydata apt install -y build-essential cmake git ninja-build pkg-config ccache mingw-w64 nsis zip', + apt_get_quiet + ' update', + apt_get_quiet + ' install -y eatmydata', + 'eatmydata ' + apt_get_quiet + ' install -y build-essential cmake git ninja-build pkg-config ccache mingw-w64 nsis zip', 'update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix', 'update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix', 'git clone https://github.com/despair86/libuv.git win32-setup/libuv', @@ -104,7 +108,7 @@ local windows_cross_pipeline(name, image, }; // Builds a snapshot .deb on a debian-like system by merging into the debian/* or ubuntu/* branch -local deb_builder(image, distro, distro_branch, arch='amd64', imaginary_repo=false) = { +local deb_builder(image, distro, distro_branch, arch='amd64', loki_repo=true) = { kind: 'pipeline', type: 'docker', name: 'DEB (' + distro + (if arch == 'amd64' then '' else '/' + arch) + ')', @@ -118,15 +122,15 @@ local deb_builder(image, distro, distro_branch, arch='amd64', imaginary_repo=fal failure: 'ignore', environment: { SSH_KEY: { from_secret: "SSH_KEY" } }, commands: [ - 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections', - 'apt-get update', - 'apt-get install -y eatmydata', - 'eatmydata apt-get install -y git devscripts equivs ccache git-buildpackage python3-dev' + (if imaginary_repo then ' gpg' else'') - ] + (if imaginary_repo then [ // Some distros need the imaginary.stream repo for backported sodium, etc. - 'echo deb https://deb.imaginary.stream $${distro} main >/etc/apt/sources.list.d/imaginary.stream.list', - 'curl -s https://deb.imaginary.stream/public.gpg | apt-key add -', - 'eatmydata apt-get update' + 'echo "Building on ${DRONE_STAGE_MACHINE}"', + 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' + ] + (if loki_repo then [ + 'cp contrib/deb.loki.network.gpg /etc/apt/trusted.gpg.d', + 'echo deb http://deb.loki.network $${distro} main >/etc/apt/sources.list.d/loki.network.list' ] else []) + [ + apt_get_quiet + ' update', + apt_get_quiet + ' install -y eatmydata', + 'eatmydata ' + apt_get_quiet + ' install -y git devscripts equivs ccache git-buildpackage python3-dev', ||| # Look for the debian branch in this repo first, try upstream if that fails. if ! git checkout $${distro_branch}; then @@ -134,15 +138,18 @@ local deb_builder(image, distro, distro_branch, arch='amd64', imaginary_repo=fal git checkout $${distro_branch} fi |||, + # Tell the merge how to resolve conflicts in the source .drone.jsonnet (we don't + # care about it at all since *this* .drone.jsonnet is already loaded). + 'git config merge.ours.driver true', + 'echo .drone.jsonnet merge=ours >>.gitattributes', + 'git merge ${DRONE_COMMIT}', 'export DEBEMAIL="${DRONE_COMMIT_AUTHOR_EMAIL}" DEBFULLNAME="${DRONE_COMMIT_AUTHOR_NAME}"', 'gbp dch -S -s "HEAD^" --spawn-editor=never -U low', - 'eatmydata mk-build-deps --install --remove --tool "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y"', + 'eatmydata mk-build-deps --install --remove --tool "' + apt_get_quiet + ' -o Debug::pkgProblemResolver=yes --no-install-recommends -y"', 'export DEB_BUILD_OPTIONS="parallel=$$(nproc)"', - 'grep -q lib debian/lokinet-bin.install || echo "/usr/lib/lib*.so*" >>debian/lokinet-bin.install', + #'grep -q lib debian/lokinet-bin.install || echo "/usr/lib/lib*.so*" >>debian/lokinet-bin.install', 'debuild -e CCACHE_DIR -b', - 'pwd', - 'find ./contrib/ci', './contrib/ci/drone-debs-upload.sh ' + distro, ] } @@ -161,6 +168,7 @@ local mac_builder(name, build_type='Release', werror=true, cmake_extra='', extra name: 'build', environment: { SSH_KEY: { from_secret: "SSH_KEY" } }, commands: [ + 'echo "Building on ${DRONE_STAGE_MACHINE}"', // If you don't do this then the C compiler doesn't have an include path containing // basic system headers. WTF apple: 'export SDKROOT="$(xcrun --sdk macosx --show-sdk-path)"', @@ -185,8 +193,10 @@ local mac_builder(name, build_type='Release', werror=true, cmake_extra='', extra steps: [{ name: 'build', image: 'debian:sid', commands: [ - 'apt-get update', 'apt-get install -y eatmydata', - 'eatmydata apt-get install -y git clang-format-9', + 'echo "Building on ${DRONE_STAGE_MACHINE}"', + apt_get_quiet + ' update', + apt_get_quiet + ' install -y eatmydata', + 'eatmydata ' + apt_get_quiet + ' install -y git clang-format-9', './contrib/ci/drone-format-verify.sh'] }] }, @@ -199,7 +209,7 @@ local mac_builder(name, build_type='Release', werror=true, cmake_extra='', extra debian_pipeline("Debian buster (i386)", "i386/debian:buster", cmake_extra='-DDOWNLOAD_SODIUM=ON'), debian_pipeline("Ubuntu focal (amd64)", "ubuntu:focal"), debian_pipeline("Ubuntu bionic (amd64)", "ubuntu:bionic", deps='g++-8 ' + default_deps_nocxx, - cmake_extra='-DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8', imaginary_repo=true), + cmake_extra='-DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8', loki_repo=true), // ARM builds (ARM64 and armhf) debian_pipeline("Debian sid (ARM64)", "debian:sid", arch="arm64"), @@ -227,7 +237,7 @@ local mac_builder(name, build_type='Release', werror=true, cmake_extra='', extra // Deb builds: deb_builder("debian:sid", "sid", "debian/sid"), - deb_builder("debian:buster", "buster", "debian/buster", imaginary_repo=true), + deb_builder("debian:buster", "buster", "debian/buster"), deb_builder("ubuntu:focal", "focal", "ubuntu/focal"), deb_builder("debian:sid", "sid", "debian/sid", arch='arm64'), diff --git a/contrib/deb.loki.network.gpg b/contrib/deb.loki.network.gpg new file mode 100644 index 0000000000000000000000000000000000000000..6ec4e25ad9a5f27c4cf558a455888e8087153f9d GIT binary patch literal 2213 zcmV;W2wL}<0u2OKF+Xtu5CE?&fm>>)yi@QSe>CUvyh@(yR~rMoHPnm@yQ)}lpqkMf zD?8<&y2iLh@H(Ncc^4S;)>v2`VyM%K48uy?Hm4i`cW^n<9xI`P`k|xhmc>GxhsBVaU3%-K%Fsi@4anlD}?+XzVZtkUWlK9Oy%mk zK7NJ7?iE9>EM#<>R_NBkXI4vd5`1Gj8c8HH-FT86?#Gj^%?08cQf1IR5jG%#ZXgKD zg+Wm2dD1nIZw1&kNqF&k4y1w3T?Z!kOVBFE$wZE|I{8`aD5(CY7o+UJ`tv_;u9n%H zLS2uAMJ~0yKh)T8i(NpiJ{GDQvMlARUq%>H-AVyWlL=<0++mOSL2=r#=q5_e&WBf; zo&cKEaSF{7hOIr}DX^5LQn;H*IyF_fD_aFKZ_sf)Ym5_koQX6yb{5@~!lTDQ<^?VT zVo*=fO@xy|MABwa>{N&SlGJ~huxV7Pn_x{@UMePG_`syb`&$w4v&h{cr;FlK-#A>D z*0tfCuxf-a)w(5@t{0-vrJ;fjb;tglI<`T{ID6>1juEPmvTExc^@koU#T15Y}5 z3~z8-&kGw-w@s&0VZR?_NNKBxqMH3i+@}mt`|88~?KFm>Tjd5%y!(vW1aJT{p6v=P zP@;vjY{;5gEvo<#0RRECCQ4y*Z*Cw`XlZU`Y+-I>WpW@qYGHG4Za`^mVP|P>VRCsc zV_`mt0yqQ{0RjLb1p-zvKXC#Z0|pBT2nPcK1{DYb2?`4Y76JnS0v-VZ7k~f?2@u4Y zEa#|=LSko35B@W1W>tT)7-CsDjswdHnH&BedhVRfMKUq~nh`E@@ORw+{kDqoYyN@! zv#Q3XiR0yopGKib1^oxCZLVf5L2Jq-{BS@h4qAgo`gIU5u z3nOqmW;@|rm3j*7wg^ow8P%~v?nE8%c^gm^Se&vRS zn(HdcusYW?Xj|3iST3E%P`n#rK%=t}*OxHHHf*%LoP5m&xX^C(4{MgfP?wTHwCG!N z3N*Q6#p){(kK;L73f`Pm)tT*AB8@_2V$E(!Y%5vb>gFK_%xpKpx9z*RjSaMn0jW~ z4*AeBu^e5Z`LSf*3t&sUbXXHNS<|H3XZ!00r*GIMm(r5I)ucUp*m^EQ)<`#6 zE*&R?6q%5idYuu)WK~YtKAJw4i^hWl;L=n1HDh*b1>Z#(?DPEacNHqunS2=ZD`e8299CYx~rbxm&e9Ypjw&6RWWo0xevuI#5Y4v zg7rl1xZsxECMmVcJ{a(f$$%5b5@jJ`FT0Rjmm3bHoe{=}WmJkT(L zTdoAcYT@XJCaEd}4~t+Fp6OY9zmyjI1E>1i8xX?$ERz{#`AKNn09 zn|>{TqRl@|vyy;v?sno*#LE+q$cNQ%re3_6a~0olmz)^0IlkdT?ZJ$NF9MCa9Zq31 z0CWd+&oxI*h_%@gf;#r)aV=3aMq-o$#iYHG1&XdPa>MBAQ>$C2IZ89ZPRy~}oNlIEZ)O@& zTAQzFr0$^a2iHSM2)P#AGGz4gglF!^bwg4<5h4{O2DQ>GoDE!hzSQFIM(+u+{Yd12 zQ9#higkbklZ`(i{z`>sATHHV8DP_vp+Ay4#6dr=F+BM|@FPRe4jyZi_EWv@^(kR-O z|CSgF*+&*w@C?x?uU3p7^im$a6U$0ZvQi0%{|rtN!nDA|ok|BysaU%9aEB5@BBiDq nq1%?36-cF__u)W3ywPi5!&0SfcP(HNlEj8=N;Vhp-76(J2Bs0L literal 0 HcmV?d00001