Docker image updates

- split debian sid into base/sid/clang images
- similarly for debian stable
- add jsonnet to lint
- add `--pull` to docker build so that we always pull the latest images
  (otherwise we were building on whatever local cache we have for
  `debian:sid`, etc., which made the base image update layer much
  larger).
- don't install Recommends by default
- add libzmq3-dev
- split android into android (base) and flutter
- hard-code registry.oxen.rocks into the dockerfile stuff because that
  seems to be the only way to properly depend on other docker builds.
- update a few CI builds that should have been using our images but
  weren't.
- Update a few CI images to bullseye instead of buster

Add openssh-client (for sftp to upload builds)
pull/1774/head
Jason Rhinelander 3 years ago
parent 6f58648161
commit ca9d979361

@ -193,14 +193,14 @@ local deb_builder(image, distro, distro_branch, arch='amd64', loki_repo=true) =
local clang(version) = debian_pipeline(
'Debian sid/clang-' + version + ' (amd64)',
docker_base + 'debian-sid',
docker_base + 'debian-sid-clang',
deps=['clang-' + version] + default_deps_nocxx,
cmake_extra='-DCMAKE_C_COMPILER=clang-' + version + ' -DCMAKE_CXX_COMPILER=clang++-' + version + ' '
);
local full_llvm(version) = debian_pipeline(
'Debian sid/llvm-' + version + ' (amd64)',
'debian:sid',
docker_base + 'debian-sid-clang',
deps=['clang-' + version, ' lld-' + version, ' libc++-' + version + '-dev', 'libc++abi-' + version + '-dev']
+ default_deps_nocxx,
cmake_extra='-DCMAKE_C_COMPILER=clang-' + version +
@ -249,7 +249,7 @@ local mac_builder(name,
type: 'docker',
steps: [{
name: 'build',
image: 'registry.oxen.rocks/lokinet-ci-lint',
image: docker_base + 'lint',
commands: [
'echo "Building on ${DRONE_STAGE_MACHINE}"',
apt_get_quiet + ' update',
@ -261,21 +261,21 @@ local mac_builder(name,
},
// Various debian builds
debian_pipeline('Debian sid (amd64)', 'debian:sid'),
debian_pipeline('Debian sid/Debug (amd64)', 'debian:sid', build_type='Debug'),
debian_pipeline('Debian sid (amd64)', docker_base + 'debian-sid'),
debian_pipeline('Debian sid/Debug (amd64)', docker_base + 'debian-sid', build_type='Debug'),
clang(13),
full_llvm(13),
debian_pipeline('Debian buster (i386)', 'i386/debian:buster', cmake_extra='-DDOWNLOAD_SODIUM=ON'),
debian_pipeline('Debian bullseye (i386)', 'i386/debian:bullseye', cmake_extra='-DDOWNLOAD_SODIUM=ON'),
debian_pipeline('Ubuntu focal (amd64)', docker_base + 'ubuntu-focal'),
debian_pipeline('Ubuntu bionic (amd64)',
'ubuntu:bionic',
docker_base + 'ubuntu-bionic',
deps=['g++-8'] + default_deps_nocxx,
cmake_extra='-DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8',
loki_repo=true),
// ARM builds (ARM64 and armhf)
debian_pipeline('Debian sid (ARM64)', 'debian:sid', arch='arm64', jobs=4),
debian_pipeline('Debian buster (armhf)', 'arm32v7/debian:buster', arch='arm64', cmake_extra='-DDOWNLOAD_SODIUM=ON', jobs=4),
debian_pipeline('Debian bullseye (armhf)', 'arm32v7/debian:bullseye', arch='arm64', cmake_extra='-DDOWNLOAD_SODIUM=ON', jobs=4),
// Static armhf build (gets uploaded)
debian_pipeline('Static (buster armhf)',
'arm32v7/debian:buster',
@ -290,7 +290,7 @@ local mac_builder(name,
],
jobs=4),
// android apk builder
apk_builder('android apk', 'registry.oxen.rocks/lokinet-ci-android', extra_cmds=['UPLOAD_OS=android ./contrib/ci/drone-static-upload.sh']),
apk_builder('android apk', docker_base + 'flutter', extra_cmds=['UPLOAD_OS=android ./contrib/ci/drone-static-upload.sh']),
// Windows builds (x64)
windows_cross_pipeline('Windows (amd64)',
@ -321,9 +321,10 @@ local mac_builder(name,
cmake_extra='-DWITH_HIVE=ON'),
// Deb builds:
deb_builder('debian:sid', 'sid', 'debian/sid'),
deb_builder('debian:buster', 'buster', 'debian/buster'),
deb_builder('ubuntu:focal', 'focal', 'ubuntu/focal'),
deb_builder(docker_base + 'debian-sid', 'sid', 'debian/sid'),
deb_builder(docker_base + 'debian-bullseye', 'bullseye', 'debian/bullseye'),
deb_builder(docker_base + 'ubuntu-impish', 'impish', 'ubuntu/impish'),
deb_builder(docker_base + 'ubuntu-focal', 'focal', 'ubuntu/focal'),
deb_builder('debian:sid', 'sid', 'debian/sid', arch='arm64'),
// Macos builds:

@ -0,0 +1,3 @@
FROM debian:bullseye
RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y'

@ -0,0 +1,3 @@
FROM debian:buster
RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y'

@ -0,0 +1,3 @@
FROM debian:sid
RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y'

@ -0,0 +1 @@
00-debian-bullseye-base.dockerfile

@ -0,0 +1,3 @@
FROM debian:testing
RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y'

@ -0,0 +1,2 @@
FROM registry.oxen.rocks/lokinet-ci-debian-bullseye-base
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client'

@ -0,0 +1,2 @@
FROM registry.oxen.rocks/lokinet-ci-debian-buster-base
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client'

@ -0,0 +1,2 @@
FROM registry.oxen.rocks/lokinet-ci-debian-sid-base
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake make git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client'

@ -0,0 +1 @@
10-debian-bullseye.dockerfile

@ -1,3 +1,3 @@
FROM ubuntu:bionic
RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q --no-install-recommends install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++-8 python3-dev automake libtool autoconf make qttools5-dev file gperf patch openssh-client'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q --no-install-recommends install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++-8 python3-dev automake libtool autoconf make qttools5-dev file gperf patch openssh-client lsb-release libzmq3-dev libpgm-dev libuv1-dev openssh-client && mkdir -p /usr/lib/x86_64-linux-gnu/pgm-5.2/include'

@ -0,0 +1,3 @@
FROM ubuntu:focal
RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client'

@ -0,0 +1,3 @@
FROM registry.oxen.rocks/lokinet-ci-debian-testing-base
RUN /bin/bash -c 'sed -i "s/main/main contrib/g" /etc/apt/sources.list'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y android-sdk google-android-ndk-installer wget git pkg-config automake libtool cmake ccache curl zip xz-utils openssh-client && git clone https://github.com/Shadowstyler/android-sdk-licenses.git /tmp/android-sdk-licenses && cp -a /tmp/android-sdk-licenses/*-license /usr/lib/android-sdk/licenses && rm -rf /tmp/android-sdk-licenses'

@ -1,7 +0,0 @@
FROM debian:testing
RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
RUN /bin/bash -c 'sed -i "s/main/main contrib/g" /etc/apt/sources.list'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install -y android-sdk google-android-ndk-installer wget git pkg-config automake libtool cmake ccache curl zip'
RUN /bin/bash -c 'git clone https://github.com/Shadowstyler/android-sdk-licenses.git /tmp/android-sdk-licenses && cp -a /tmp/android-sdk-licenses/*-license /usr/lib/android-sdk/licenses && rm -rf /tmp/android-sdk-licenses'
RUN /bin/bash -c 'wget https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_2.2.2-stable.tar.xz -O /tmp/flutter.tar.xz && cd /opt && tar -xJvf /tmp/flutter.tar.xz && rm /tmp/flutter.tar.xz && ln -s /opt/flutter/bin/flutter /usr/local/bin/'
RUN /bin/bash -c 'flutter precache'

@ -0,0 +1,2 @@
FROM registry.oxen.rocks/lokinet-ci-debian-sid
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y clang-13 lld-13 libc++-13-dev libc++abi-13-dev'

@ -1,3 +0,0 @@
FROM debian:sid
RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache clang-11 libsodium-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev'

@ -1,3 +0,0 @@
FROM debian:stable
RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev'

@ -1,4 +1,4 @@
FROM debian:testing
RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install -y eatmydata build-essential cmake git ninja-build pkg-config ccache g++-mingw-w64-x86-64-posix nsis zip automake libtool autoconf make qttools5-dev file gperf patch openssh-client'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata build-essential cmake git ninja-build pkg-config ccache g++-mingw-w64-x86-64-posix nsis zip automake libtool autoconf make qttools5-dev file gperf patch openssh-client'
RUN /bin/bash -c 'update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix && update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix'

@ -0,0 +1,2 @@
FROM registry.oxen.rocks/lokinet-ci-android
RUN /bin/bash -c 'cd /opt && curl https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_2.2.2-stable.tar.xz | tar xJv && ln -s /opt/flutter/bin/flutter /usr/local/bin/ && flutter precache'

@ -1,3 +1,2 @@
FROM debian:sid
RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q install -y eatmydata git clang-format-11'
FROM registry.oxen.rocks/lokinet-ci-debian-sid-base
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata git clang-format-11 jsonnet'

@ -1,3 +1,3 @@
FROM node:14.16.1
RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ wine'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ wine openssh-client'

@ -1,8 +1,13 @@
## drone-ci docker jizz
To rebuild all ci images and push them to a registry server do:
To rebuild all ci images and push them to the oxen registry server do:
$ docker login your.registry.here
$ ./rebuild-docker-images.sh your.registry.here *.dockerfile
$ docker login registry.oxen.rocks
$ ./rebuild-docker-images.sh *.dockerfile
The docker images will be `your.registry.here/lokinet-ci-*`for each *.dockerfile in this directory
If you aren't part of the Oxen team, you'll likely need to set up your own registry and change
registry.oxen.rocks to your own domain name in order to do anything useful with this.
The docker images will be `registry.oxen.rocks/lokinet-ci-*`for each \*.dockerfile in this
directory, with the leading numeric `NN-` removed, if present (so that you can ensure proper
ordering using two-digit numeric prefixes).

@ -1,13 +1,19 @@
#!/bin/bash
# the registry server to use
registry=$1
set -o errexit
test "x$registry" != "x" || exit 1
registry=registry.oxen.rocks
for file in ${@:2} ; do
name="$(echo $file | cut -d'.' -f1)"
echo "rebuild $name"
docker build -f $file -t $registry/lokinet-ci-$name .
if [[ $# -eq 0 ]]; then
files=(*.dockerfile)
else
files=("$@")
fi
for file in "${files[@]}"; do
name="${file#[0-9][0-9]-}" # s/^\d\d-//
name="${name%.dockerfile}" # s/\.dockerfile$//
echo -e "\e[32;1mrebuilding $name\e[0m"
docker build --pull -f $file -t $registry/lokinet-ci-$name .
docker push $registry/lokinet-ci-$name
done

@ -1,3 +0,0 @@
FROM ubuntu:focal
RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev'
Loading…
Cancel
Save