diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index dac6b25..baca9bc 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -16,6 +16,57 @@ on: workflow_dispatch: jobs: + # ── One Go patch in every image that ships Go (no build needed) ──── + # A Go build cache is keyed by the toolchain that wrote it: lanes on two + # patches that share one cache split it, each rebuilding what the other + # cached. keploy-ci's GO_VERSION (an X.Y.Z, with go.dev's SHA-256 for each + # tarball) is the patch; go-build and golint must be FROM that patch's + # golang image by digest (a bare golang:1.27-bookworm floats to each new + # patch), and ebpf-objects must fetch and check that patch's tarball. The + # builds below then run each image's go and compare. + go-patch: + name: One Go patch in every image + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Every Dockerfile that ships Go names keploy-ci's GO_VERSION + run: | + set -eu + rc=0 + bad() { echo "::error file=$1::$2"; rc=1; } + want="$(sed -n 's/^ARG GO_VERSION=//p' keploy-ci/Dockerfile)" + printf '%s\n' "$want" | grep -qxE '[0-9]+\.[0-9]+\.[0-9]+' \ + || bad keploy-ci/Dockerfile "GO_VERSION='$want' is not one X.Y.Z patch" + for a in AMD64 ARM64; do + grep -qxE "ARG GO_SHA256_$a=[0-9a-f]{64}" keploy-ci/Dockerfile \ + || bad keploy-ci/Dockerfile "no ARG GO_SHA256_$a=" + done + grep -qF 'sha256sum -c' keploy-ci/Dockerfile || bad keploy-ci/Dockerfile "the Go tarball is not checked against GO_SHA256_*" + v="$(printf '%s' "$want" | sed 's/\./\\./g')" + for f in keploy-ci-go-build/Dockerfile keploy-ci-golint/Dockerfile; do + from="$(grep -m1 '^FROM ' "$f")" + printf '%s\n' "$from" | grep -qxE "FROM golang:$v-bookworm@sha256:[0-9a-f]{64}" \ + || bad "$f" "'$from': want FROM golang:$want-bookworm@sha256:" + done + f=keploy-ci-ebpf-objects/Dockerfile + grep -qF "https://go.dev/dl/go$want.linux-amd64.tar.gz" "$f" || bad "$f" "does not fetch go$want.linux-amd64.tar.gz" + grep -qE '^ADD --checksum=sha256:[0-9a-f]{64} ' "$f" || bad "$f" "its Go tarball is not ADDed with --checksum" + grep -qF "go version | grep -qF 'go$want linux/amd64'" "$f" || bad "$f" "its build does not check go version is go$want" + # No other Go named in any instruction (comments aside): another + # golang image (by tag, digest or variable), a Go tarball (binary or + # source) of another version written out, from any host, Debian's + # golang packages, or a GOTOOLCHAIN that names a toolchain, quoted + # or not. A tarball named through a variable (keploy-ci's + # go${GO_VERSION}) is the runtime check's to catch. + others="$(cat keploy-ci*/Dockerfile | grep -v '^[[:space:]]*#' \ + | grep -oE 'golang[:@][^[:space:]]*|/go[0-9][0-9.]*[0-9]\.(linux-[a-z0-9]+|src)|golang-go|golang-[0-9.]+-go|GOTOOLCHAIN=["'"'"']?go[^[:space:]"'"'"']*' \ + | grep -vxE "golang:$v-bookworm@sha256:[0-9a-f]{64}|/go$v\.linux-amd64" | sort -u || true)" + [ -z "$others" ] || bad . "other Go named: $others" + [ "$rc" = 0 ] && echo "every image that ships Go names go$want" + exit "$rc" + # ── Stage 0: the minio-cache helper and the s3.sh client it speaks through # are duplicated per build context (Docker cannot COPY outside a context). # Drift between copies is how the ILM rule-spam regression shipped, so fail @@ -152,9 +203,11 @@ jobs: - context: ./keploy-ci tag-prefix: '' arch: amd64 + go: true - context: ./keploy-ci tag-prefix: '' arch: arm64 + go: true - context: ./keploy-ci-slim tag-prefix: 'slim-' arch: amd64 @@ -170,9 +223,11 @@ jobs: - context: ./keploy-ci-golint tag-prefix: 'golint-' arch: amd64 + go: true - context: ./keploy-ci-golint tag-prefix: 'golint-' arch: arm64 + go: true - context: ./keploy-ci-awscli tag-prefix: 'awscli-' arch: amd64 @@ -189,6 +244,7 @@ jobs: - context: ./keploy-ci-go-build tag-prefix: 'go-build-' arch: amd64 + go: true steps: - name: Checkout repository @@ -207,8 +263,19 @@ jobs: context: ${{ matrix.context }} platforms: linux/${{ matrix.arch }} push: false + load: ${{ matrix.go || false }} tags: keploy-ci:${{ matrix.tag-prefix }}ci-${{ matrix.arch }} + - name: Its go is keploy-ci's GO_VERSION + if: matrix.go + run: | + set -eu + want="go$(sed -n 's/^ARG GO_VERSION=//p' keploy-ci/Dockerfile)" + got="$(docker run --rm --platform "linux/${{ matrix.arch }}" --entrypoint go \ + "keploy-ci:${{ matrix.tag-prefix }}ci-${{ matrix.arch }}" env GOVERSION)" + echo "keploy-ci declares $want; ${{ matrix.tag-prefix || 'base' }} (${{ matrix.arch }}) runs $got" + [ "$got" = "$want" ] || { echo "::error::${{ matrix.tag-prefix || 'base' }} ships $got, not $want"; exit 1; } + # ── A tier-1 image no other image needs, in a job of its own ──────── # As in publish.yml: derived-images needs base-images, so a leg there that # fails (this one fetches from snapshot.ubuntu.com and go.dev) would skip @@ -224,6 +291,7 @@ jobs: - context: ./keploy-ci-ebpf-objects tag-prefix: 'ebpf-objects-' arch: amd64 + go: true steps: - name: Checkout repository @@ -238,8 +306,19 @@ jobs: context: ${{ matrix.context }} platforms: linux/${{ matrix.arch }} push: false + load: ${{ matrix.go || false }} tags: keploy-ci:${{ matrix.tag-prefix }}ci-${{ matrix.arch }} + - name: Its go is keploy-ci's GO_VERSION + if: matrix.go + run: | + set -eu + want="go$(sed -n 's/^ARG GO_VERSION=//p' keploy-ci/Dockerfile)" + got="$(docker run --rm --platform "linux/${{ matrix.arch }}" --entrypoint go \ + "keploy-ci:${{ matrix.tag-prefix }}ci-${{ matrix.arch }}" env GOVERSION)" + echo "keploy-ci declares $want; ${{ matrix.tag-prefix || 'base' }} (${{ matrix.arch }}) runs $got" + [ "$got" = "$want" ] || { echo "::error::${{ matrix.tag-prefix || 'base' }} ships $got, not $want"; exit 1; } + # ── Stage 2: images that derive FROM keploy-ci ───────────────────── # Exercise the BASE_TAG threading that publish.yml relies on. # diff --git a/README.md b/README.md index e62bc11..d71482c 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,9 @@ A Docker-in-Docker image with Go and common CI dependencies for running Keploy t **Features:** - Based on `debian:trixie-slim`, with Docker Engine 29.1 from Docker's apt repository (dockerd, docker CLI, buildx, compose, containerd) -- Go 1.27.0; no C toolchain (Linux builds use `CGO_ENABLED=0`; Windows - cross-compiles use `keploy-ci-go-build`) +- Go 1.27.2, the one Go patch of every image here that has Go; no C + toolchain (Linux builds use `CGO_ENABLED=0`; Windows cross-compiles use + `keploy-ci-go-build`) - Common CI utilities (bash, curl, git, jq, sudo, zstd, etc.); no MinIO client (see below) - Helper script `start-docker` to start Docker daemon inside containers @@ -64,7 +65,7 @@ mirror to install it. **Contents:** Ubuntu 22.04 (by index digest), `clang-14` and `llvm-14` 1:14.0.0-1ubuntu1.1, `libbpf-dev` 1:0.5.0-1ubuntu22.04.1, `linux-libc-dev`, `libc6-dev` and their arm64 cross packages, `git`, `ca-certificates`, and Go -1.27.1 (`GOTOOLCHAIN=local`). Every package installed comes from one +1.27.2 (`GOTOOLCHAIN=local`). Every package installed comes from one snapshot.ubuntu.com snapshot and the recipe's packages are pinned with `=version`, so a rebuild resolves the same versions or fails; Go is checked against its SHA-256. amd64 only: the snapshot service does not serve diff --git a/keploy-ci-ebpf-objects/Dockerfile b/keploy-ci-ebpf-objects/Dockerfile index 610967d..f170dc3 100644 --- a/keploy-ci-ebpf-objects/Dockerfile +++ b/keploy-ci-ebpf-objects/Dockerfile @@ -1,7 +1,7 @@ # keploy-ci-ebpf-objects: the toolchain keploy/ebpf's committed BPF objects are # built with (the recipe in its compile.sh header: Ubuntu 22.04, clang-14 and # llvm-14, libbpf-dev 0.5.0, the native and arm64 cross kernel and libc -# headers, Go 1.27.1, git), pinned, so that its objects-reproducible lane +# headers, Go 1.27.2, git), pinned, so that its objects-reproducible lane # rebuilds every committed object without apt or a Go download. # Used by: keploy/ebpf .woodpecker/objects-reproducible.yml, by digest. # @@ -31,8 +31,8 @@ # pinned the same way. FROM ubuntu@sha256:0a9887931b0e38af2e3487ed39e29436b0535c4f3f52e57745baaf79d1898956 AS go -ADD --checksum=sha256:63d339f0da5ab53635a56f2490a7984dfe12dfcff22ad749f63edaf590168445 \ - https://go.dev/dl/go1.27.1.linux-amd64.tar.gz /tmp/go.tgz +ADD --checksum=sha256:ecbadb99091a3f46e31f5f934b068b1864eafa7995211b39eaddf76996045fe5 \ + https://go.dev/dl/go1.27.2.linux-amd64.tar.gz /tmp/go.tgz RUN tar -C /usr/local -xzf /tmp/go.tgz FROM ubuntu@sha256:0a9887931b0e38af2e3487ed39e29436b0535c4f3f52e57745baaf79d1898956 @@ -90,7 +90,7 @@ ENV PATH=/usr/local/go/bin:$PATH GOTOOLCHAIN=local # target with compile.sh's include paths, from the kernel and libbpf headers, # and strip it. RUN set -eux; \ - go version | grep -qF 'go1.27.1 linux/amd64'; \ + go version | grep -qF 'go1.27.2 linux/amd64'; \ printf '#include \n#include \n#include \n#include \nSEC("tracepoint/x") int probe(void *ctx) { return 0; }\nchar LICENSE[] SEC("license") = "GPL";\n' > /tmp/probe.c; \ clang -target bpf -O2 -g -D__x86_64__ -I/usr/include/x86_64-linux-gnu -c /tmp/probe.c -o /tmp/probe-x86.o; \ clang -target bpf -O2 -g -D__aarch64__ -I/usr/aarch64-linux-gnu/include -I/usr/include/aarch64-linux-gnu -c /tmp/probe.c -o /tmp/probe-arm64.o; \ diff --git a/keploy-ci-go-build/Dockerfile b/keploy-ci-go-build/Dockerfile index d40bba6..be7d351 100644 --- a/keploy-ci-go-build/Dockerfile +++ b/keploy-ci-go-build/Dockerfile @@ -1,6 +1,8 @@ # keploy-ci-go-build: Go + MinGW cross-compilers # Used by: go-test.yml download-and-build, prepare-and-run.yml build-and-upload-keploy -FROM golang:1.27-bookworm +# By digest: golang:1.27-bookworm floats to each new patch, and this image's Go +# must be the patch every CI image ships (keploy-ci/Dockerfile's GO_VERSION). +FROM golang:1.27.2-bookworm@sha256:5cf287a799e6b94384bad13d16b14904c531f51ba65792237e122ce42b392f61 RUN set -eux; \ apt-get update; \ diff --git a/keploy-ci-golint/Dockerfile b/keploy-ci-golint/Dockerfile index d18ac8d..a1e13ec 100644 --- a/keploy-ci-golint/Dockerfile +++ b/keploy-ci-golint/Dockerfile @@ -2,7 +2,9 @@ # the other Linux architecture's cross compiler that enterprise's cgo typecheck # needs (see below). # Used by: golangci-lint.yml -FROM golang:1.27-bookworm +# By digest: golang:1.27-bookworm floats to each new patch, and this image's Go +# must be the patch every CI image ships (keploy-ci/Dockerfile's GO_VERSION). +FROM golang:1.27.2-bookworm@sha256:5cf287a799e6b94384bad13d16b14904c531f51ba65792237e122ce42b392f61 # The C toolchain pieces enterprise's lint needs for its cgo, which the # typecheck runs (cgo code is type-checked from what cgo generates): diff --git a/keploy-ci/Dockerfile b/keploy-ci/Dockerfile index eaf8833..a46ac03 100644 --- a/keploy-ci/Dockerfile +++ b/keploy-ci/Dockerfile @@ -6,7 +6,15 @@ # Linux binaries with CGO_ENABLED=0. FROM debian:trixie-slim -ARG GO_VERSION=1.27.0 +# One Go patch in every CI image that ships Go: this one (and every image FROM +# it), go-build, golint and ebpf-objects. A Go build cache is keyed by the +# toolchain that wrote it, so a cache shared across lanes on different patches +# splits, each lane rebuilding what the other cached. docker-build.yml holds +# them to one. GO_SHA256_* are go.dev's SHA-256 of each release tarball +# (https://go.dev/dl/?mode=json), checked before it is unpacked. +ARG GO_VERSION=1.27.2 +ARG GO_SHA256_AMD64=ecbadb99091a3f46e31f5f934b068b1864eafa7995211b39eaddf76996045fe5 +ARG GO_SHA256_ARM64=94f3e30b8e374bc285e7dadc11e0865726b9bc6e85b841ccceaabc0214c6b7c8 # Keep Docker "major.minor" aligned with the latest stable release. ARG DOCKER_MAJOR_MINOR=29.1 # Provided automatically by BuildKit/buildx; fallback to dpkg when unset. @@ -79,6 +87,8 @@ RUN set -eux; \ *) echo "Unsupported architecture: $ARCH"; exit 1 ;; \ esac; \ curl -fsSL --retry 5 --retry-delay 2 --retry-all-errors "https://go.dev/dl/go${GO_VERSION}.linux-${ARCH}.tar.gz" -o /tmp/go.tgz; \ + case "$ARCH" in amd64) sum="$GO_SHA256_AMD64" ;; arm64) sum="$GO_SHA256_ARM64" ;; esac; \ + echo "$sum /tmp/go.tgz" | sha256sum -c -; \ rm -rf /usr/local/go; \ tar -C /usr/local -xzf /tmp/go.tgz; \ rm -f /tmp/go.tgz diff --git a/keploy-ci/start-docker b/keploy-ci/start-docker index 0315678..a9776e9 100755 --- a/keploy-ci/start-docker +++ b/keploy-ci/start-docker @@ -14,12 +14,11 @@ # Each daemon is waited for while it is alive and making progress: until it # answers, it exits, or its log has not grown for START_DOCKER_SILENCE_CHECKS # looks in a row (600 by default; a hang guard, not a budget for a start that -# works). A look is a second. The probe (`ctr version`, `docker info`) is +# works). A look is a second. The probe (`ctr version`, dockerd's /_ping) is # never cut short: one runs at a time, and each look sees whether it has # answered. dockerd listens on its socket before its init and answers only -# after, and docker info asks containerd, the daemon that starves, for its -# system info: a probe that takes long is still waited for, and its answer -# taken when it comes. Only a daemon that stops progressing ends the wait, +# after: a probe that takes long is still waited for, and its answer taken +# when it comes. Only a daemon that stops progressing ends the wait, # and then the probe in flight is killed. A stall is said as it goes on, # every START_DOCKER_NOTICE_LOOKS quiet looks (30), with the daemon's last # log line, so a CI step cut by its own cap still says why. @@ -27,8 +26,14 @@ # at a socket no pidfile names, which it has nothing else to watch on: a # daemon there that answers is reused, a socket that refuses (nothing # listens) is taken over, and one that listens and does not answer within -# the bound fails the start, said: its daemon may be slow, not gone, and a -# second one on the same root is not started. +# the bound, or answers as no such daemon does, fails the start, said: its +# daemon may be slow, not gone, and a second one on the same root is not +# started. curl asks (dockerd's /_ping; containerd's Version, over gRPC, as +# ctr version does; with -q, so no ~/.curlrc changes the request): its exit +# says which of those it was, and its bound starts as it dials. A client's +# own start does not count: docker info runs its CLI plugins (buildx, +# compose) before it dials, and on a starved host that alone could take the +# bound and pass a socket that refuses for a mute one. # # containerd runs as dockerd would run it: its root and state under # /var/lib/docker and /var/run/docker (the image's /var/lib/docker volume), @@ -99,7 +104,7 @@ for f in $FLAGS; do c=${g%"${g#?}"} g=${g#?} case "$c" in H | p | g) fixed DOCKERD_FLAGS "-$c" ;; - l | G | b | e) break ;; + l | G | b | e | s) break ;; esac done ;; @@ -111,11 +116,41 @@ fi mkdir -p /var/run /var/lib/docker "$CONTAINERD_ROOT" "$CONTAINERD_STATE" /var/log -# probe CMD...: CMD answers within PROBE_SECONDS. Only for a daemon no -# pidfile names, which start-docker has nothing else to watch on (a hang -# guard); a daemon it waits on is probed without a bound (await). -probe() { - timeout "$PROBE_SECONDS" "$@" >/dev/null 2>&1 9>&- +# answers_once DAEMON SOCK: the one look at a socket no pidfile names, which +# start-docker has nothing else to watch on, bounded (PROBE_SECONDS, from +# the dial: a hang guard); a daemon it waits on is probed without a bound +# (await). Returns 0 when DAEMON (dockerd or containerd) answers there as it +# does, and 1 when nothing listens (the socket refuses: curl's 7). One that +# listens and does not answer in time (curl's 28), or answers as no DAEMON +# does, fails the start, said: nothing is started over it. +answers_once() { + _rc=0 _why="" + if [ "$1" = dockerd ]; then + _code=$(curl -q -sf -o /dev/null -w '%{http_code}' --unix-socket "$2" --max-time "$PROBE_SECONDS" http://docker/_ping 9>&-) || _rc=$? + [ "${_code:-000}" != 000 ] || _code=none + _why="curl's exit $_rc on its /_ping, HTTP status $_code" + else + # containerd's Version, an empty gRPC message (a 0 flag and a 0 length); + # its grpc-status, in the trailers curl prints with the headers, says + # whether it answered. + _hdr=$(printf '\0\0\0\0\0' | curl -q -s -o /dev/null -D - --http2-prior-knowledge --unix-socket "$2" --max-time "$PROBE_SECONDS" \ + -H 'content-type: application/grpc' -H 'te: trailers' --data-binary @- \ + http://containerd/containerd.services.version.v1.Version/Version 9>&-) || _rc=$? + _why="curl's exit $_rc on its Version call" + if [ "$_rc" = 0 ]; then + printf '%s\n' "$_hdr" | grep -q '^grpc-status: 0[[:space:]]*$' && return 0 + _st=$(printf '%s\n' "$_hdr" | grep -m 1 '^grpc-status:' | tr -d '\r') + _why="its Version call answered ${_st:-no grpc-status}" + _rc=1 + fi + fi + case $_rc in + 0) return 0 ;; + 7) return 1 ;; + 28) echo "start-docker: something no pidfile names listens on $2 and did not answer in ${PROBE_SECONDS}s: not starting a second $1 over it" ;; + *) echo "start-docker: something no pidfile names answers on $2, but not as $1 does ($_why): not starting a second $1 over it" ;; + esac + exit 1 } # alive PID: the process runs (a zombie, exited but not yet reaped, does not). @@ -236,7 +271,7 @@ fi old=$(pidfile_names "$DOCKERD_STARTED" dockerd || pidfile_names "$DOCKERD_PIDFILE" dockerd || true) if [ -n "$old" ]; then echo "dockerd ($old) is already running; waiting for it to answer" - if ! await dockerd "$old" "$DOCKERD_LOG" docker -H "unix://$DOCKER_SOCK" info; then + if ! await dockerd "$old" "$DOCKERD_LOG" curl -q -sf -o /dev/null --unix-socket "$DOCKER_SOCK" http://docker/_ping; then fail_start "Docker daemon failed to start" fi echo "Docker daemon is ready" @@ -244,20 +279,11 @@ if [ -n "$old" ]; then fi # A socket no pidfile names: one look, bounded. What answers is reused; what # listens and does not answer may be a slow daemon, not a gone one, and is -# not started over; a socket that refuses (nothing listens) is taken over. -if [ -S "$DOCKER_SOCK" ]; then - rc=0 - probe docker -H "unix://$DOCKER_SOCK" info || rc=$? - case $rc in - 0) - echo "Docker daemon already running" - exit 0 - ;; - 124) - echo "start-docker: something no pidfile names listens on $DOCKER_SOCK and did not answer in ${PROBE_SECONDS}s: not starting a second dockerd over it" - exit 1 - ;; - esac +# not started over, nor is what answers as no dockerd does; a socket that +# refuses (nothing listens) is taken over. +if [ -S "$DOCKER_SOCK" ] && answers_once dockerd "$DOCKER_SOCK"; then + echo "Docker daemon already running" + exit 0 fi # containerd logs at dockerd's level, as dockerd gives the containerd it @@ -297,19 +323,14 @@ if [ -n "$w" ] && is_watcher "$w" && ! containerd_named; then fi fi containerd_pid="" -rc=1 if containerd_pid=$(pidfile_names "$CONTAINERD_PIDFILE" containerd); then echo "containerd ($containerd_pid) is already running; waiting for it to answer" if ! await containerd "$containerd_pid" "$CONTAINERD_LOG" ctr --address "$CONTAINERD_SOCK" version; then fail_start "containerd failed to start" fi echo "containerd ($containerd_pid) already answers on $CONTAINERD_SOCK" -elif [ -S "$CONTAINERD_SOCK" ] && { rc=0; probe ctr --address "$CONTAINERD_SOCK" version || rc=$?; [ "$rc" = 0 ] || [ "$rc" = 124 ]; }; then +elif [ -S "$CONTAINERD_SOCK" ] && answers_once containerd "$CONTAINERD_SOCK"; then containerd_pid="" - if [ "$rc" = 124 ]; then - echo "start-docker: something no pidfile names listens on $CONTAINERD_SOCK and did not answer in ${PROBE_SECONDS}s: not starting a second containerd over it" - exit 1 - fi echo "containerd already answers on $CONTAINERD_SOCK (no pidfile names it: it has no watcher)" else containerd_pid="" @@ -364,7 +385,7 @@ dockerd --containerd="$CONTAINERD_SOCK" ${FLAGS} >>"$DOCKERD_LOG" 2>&1 9>&- & dockerd_pid=$! started="$dockerd_pid $started" echo "$dockerd_pid" >"$DOCKERD_STARTED" -if ! await dockerd "$dockerd_pid" "$DOCKERD_LOG" docker -H "unix://$DOCKER_SOCK" info; then +if ! await dockerd "$dockerd_pid" "$DOCKERD_LOG" curl -q -sf -o /dev/null --unix-socket "$DOCKER_SOCK" http://docker/_ping; then fail_start "Docker daemon failed to start" fi echo "Docker daemon is ready" diff --git a/keploy-ci/start-docker-test.sh b/keploy-ci/start-docker-test.sh index 870bcec..b6e48b8 100755 --- a/keploy-ci/start-docker-test.sh +++ b/keploy-ci/start-docker-test.sh @@ -86,7 +86,7 @@ run "settings" 'for s in START_DOCKER_SILENCE_CHECKS=0 START_DOCKER_PROBE_SECOND env "$s" start-docker; echo "$s: rc=$?" done mkdir -p /etc/docker && printf "{\"data-root\": \"/x\"}\n" >/etc/docker/daemon.json && start-docker; echo "daemon.json: rc=$?"; rm -f /etc/docker/daemon.json -DOCKERD_FLAGS="-ldebug --dns 1.1.1.1" START_DOCKER_SILENCE_CHECKS=1 start-docker >/tmp/fine.out 2>&1; grep -q "sets" /tmp/fine.out && echo "a fine flag refused" +DOCKERD_FLAGS="-ldebug -spool --dns 1.1.1.1" START_DOCKER_SILENCE_CHECKS=1 start-docker >/tmp/fine.out 2>&1; grep -q "sets" /tmp/fine.out && echo "a fine flag refused" pgrep -x containerd >/dev/null && echo "a start began" || echo "nothing started"; pkill -9 -x dockerd; pkill -9 -x containerd; true' if grep -q "START_DOCKER_SILENCE_CHECKS=0: rc=2" <<<"$out" && grep -q "START_DOCKER_PROBE_SECONDS=x: rc=2" <<<"$out" \ && grep -q "START_DOCKER_NOTICE_LOOKS=-1: rc=2" <<<"$out" && grep -q "whole numbers, 1 or more" <<<"$out" \ @@ -216,9 +216,10 @@ if grep -q "first rc=1" <<<"$out" && grep -q "containerd was started again; exit else fail "a start that fails after containerd was started again leaves nothing: the watcher stops and reaps the containerd it started"; fi # dockerd listens before its init and answers after: a stand-in stops the -# real one once its socket is there, so docker info blocks, as on a slow +# real one once its socket is there, so its /_ping blocks, as on a slow # init. Saying what it does (its log grows), it is waited for, a blocked -# probe being no answer that look; saying nothing, it fails, said. +# probe being no answer that look (and one wedged, saying nothing, fails: +# dockerd-wedged, below, with a stand-in of its own). dockerd_shim='mkdir -p /opt/shim && cat >/opt/shim/dockerd <<"EOF" #!/bin/sh /usr/bin/dockerd "$@" & d=$! @@ -236,50 +237,180 @@ PROGRESS=1 STEPS=75 PATH=/opt/shim:$PATH START_DOCKER_SILENCE_CHECKS=60 start-do if [ "$rc" = 0 ] && grep -q "Docker daemon is ready" <<<"$out"; then pass "a dockerd whose init is slow but says what it does is waited for, its blocked probes no answer, not the end" else fail "a dockerd whose init is slow but says what it does is waited for, its blocked probes no answer, not the end"; fi -run "dockerd-wedged" "$dockerd_shim"' -PATH=/opt/shim:$PATH START_DOCKER_SILENCE_CHECKS=4 START_DOCKER_NOTICE_LOOKS=2 start-docker' -if [ "$rc" = 1 ] && grep -q "dockerd has not answered, and its log has not grown in 2 looks; still waiting" <<<"$out" \ +# Wedged, it is given 4 looks of silence. containerd answers before the +# start (one no pidfile names: taken after one bounded look, not waited on), +# so the 4 looks are dockerd's alone, none spent on a containerd starting. +# The wedged dockerd is a stand-in that listens on dockerd's socket, as +# dockerd does before its init, and never answers nor logs: the real one, +# stopped once its socket is there, could finish its init first on a +# starved host. +wedged_shim='mkdir -p /opt/shim && cat >/opt/shim/dockerd <<"EOF" +#!/usr/bin/python3 +import os, socket, time +s = socket.socket(socket.AF_UNIX) +s.bind("/var/run/docker.sock.tmp") +s.listen() +os.rename("/var/run/docker.sock.tmp", "/var/run/docker.sock") +while True: + time.sleep(3600) +EOF +chmod +x /opt/shim/dockerd' +run "dockerd-wedged" "$wedged_shim"' +mkdir -p /var/lib/docker/containerd/daemon /var/run/docker/containerd +containerd --root /var/lib/docker/containerd/daemon --state /var/run/docker/containerd/daemon --address /var/run/docker/containerd/containerd.sock >/var/log/own-containerd.log 2>&1 & c=$! +until ctr --address /var/run/docker/containerd/containerd.sock version >/dev/null 2>&1 || ! kill -0 "$c" 2>/dev/null; do sleep 0.1; done +PATH=/opt/shim:$PATH START_DOCKER_SILENCE_CHECKS=4 START_DOCKER_NOTICE_LOOKS=2 START_DOCKER_PROBE_SECONDS=60 start-docker' +if [ "$rc" = 1 ] && grep -q "containerd already answers on /var/run/docker/containerd/containerd.sock (no pidfile names it" <<<"$out" \ + && grep -q "dockerd has not answered, and its log has not grown in 2 looks; still waiting" <<<"$out" \ && grep -q "dockerd has not answered, and its log has not grown in 4 looks$" <<<"$out"; then pass "a dockerd wedged in its init, its probes blocked and its log still, is said to stall as it goes on, and fails the start, said" else fail "a dockerd wedged in its init, its probes blocked and its log still, is said to stall as it goes on, and fails the start, said"; fi -# A probe that takes long is waited for, not cut: a docker that answers in 8 s -# (as docker info does while it waits on a starved containerd), with 30 looks -# of silence allowed and the bound on one-shot probes at 2 s. -run "slow-probe" 'mkdir -p /opt/shim && printf "#!/bin/sh\n[ \"\$1\" != info ] || sleep 8\nexec /usr/bin/docker \"\$@\"\n" >/opt/shim/docker && chmod +x /opt/shim/docker +# A probe that takes long is waited for, not cut: dockerd's socket reached +# through a relay that holds each connection 8 s before it passes it on (as +# a starved host answers late), with 60 looks of silence allowed and the +# bound on one-shot probes at 2 s. The 8 s are the probe's own, within +# curl's transfer: a bound on it (timeout, or curl's --max-time) cuts it. +run "slow-probe" 'mkdir -p /opt/shim && cat >/opt/relay.py <<"PY" +import socket, threading, time +def pump(a, b): + try: + while True: + x = a.recv(65536) + if not x: + break + b.sendall(x) + except OSError: + pass + for s in (a, b): + try: + s.shutdown(socket.SHUT_RDWR) + except OSError: + pass +def relay(c): + time.sleep(8) + d = socket.socket(socket.AF_UNIX) + try: + d.connect("/var/run/docker.sock") + except OSError: + c.close() + return + threading.Thread(target=pump, args=(d, c), daemon=True).start() + pump(c, d) +l = socket.socket(socket.AF_UNIX) +l.bind("/tmp/slow.sock") +l.listen(64) +while True: + threading.Thread(target=relay, args=(l.accept()[0],), daemon=True).start() +PY +cat >/opt/shim/curl <<"EOF" +#!/bin/sh +for a; do shift; [ "$a" != /var/run/docker.sock ] || a=/tmp/slow.sock; set -- "$@" "$a"; done +exec /usr/bin/curl "$@" +EOF +chmod +x /opt/shim/curl +python3 /opt/relay.py & until [ -S /tmp/slow.sock ]; do sleep 0.05; done PATH=/opt/shim:$PATH START_DOCKER_SILENCE_CHECKS=60 START_DOCKER_PROBE_SECONDS=2 start-docker' if [ "$rc" = 0 ] && grep -q "Docker daemon is ready" <<<"$out"; then pass "a probe that takes 8 s to answer is waited for, not cut short, and its answer taken" else fail "a probe that takes 8 s to answer is waited for, not cut short, and its answer taken"; fi -# A socket no pidfile names gets one look, bounded -# (START_DOCKER_PROBE_SECONDS). One that refuses (nothing listens: a daemon -# gone without removing it) is taken over: the start goes on, and its own -# dockerd binds the socket. One that listens and does not answer (a daemon -# start-docker knows nothing of, slow or wedged) fails the start, said, and -# nothing is started over it. -run "dead-socket" 'python3 -c "import socket; socket.socket(socket.AF_UNIX).bind(\"/var/run/docker.sock\")" -START_DOCKER_PROBE_SECONDS=2 start-docker' +# A socket no pidfile names, dockerd's or containerd's, gets one look, +# bounded (START_DOCKER_PROBE_SECONDS). One that refuses (nothing listens: a +# daemon gone without removing it) is taken over: the start goes on, and its +# own daemon binds the socket. The bound is the dial's: clients that take 3 s +# to start (a starved host's), against a bound of 2, do not make a socket +# that refuses look mute. One that listens and does not answer (a daemon +# start-docker knows nothing of, slow or wedged), or answers as no such +# daemon does, fails the start, said, and nothing is started over it. +# listen.py SOCK dead|mute|http404 binds SOCK and: listens on nothing (dead); +# listens and never accepts (mute); answers every request with a 404 +# (http404). A listening socket is bound beside SOCK and renamed onto it once +# it listens: SOCK is never there unlistened, which start-docker would take +# for a dead one and take over. +listener='mkdir -p /opt /var/run/docker/containerd && cat >/opt/listen.py <<"PY" +import socket, sys, time +import os +s = socket.socket(socket.AF_UNIX) +if sys.argv[2] == "dead": + s.bind(sys.argv[1]) + sys.exit(0) +s.bind(sys.argv[1] + ".tmp") +s.listen() +os.rename(sys.argv[1] + ".tmp", sys.argv[1]) +if sys.argv[2] == "mute": + time.sleep(600) +while True: + c, _ = s.accept() + c.recv(65536) + c.sendall(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + c.close() +PY +CSOCK=/var/run/docker/containerd/containerd.sock +listen() { python3 /opt/listen.py "$1" "$2" & until [ -S "$1" ]; do sleep 0.05; done; }' +run "dead-socket" "$listener"' +listen /var/run/docker.sock dead; listen "$CSOCK" dead +mkdir -p /opt/shim && for c in curl docker ctr; do printf "#!/bin/sh\nsleep 3\nexec /usr/bin/$c \"\$@\"\n" >"/opt/shim/$c"; chmod +x "/opt/shim/$c"; done +PATH=/opt/shim:$PATH START_DOCKER_PROBE_SECONDS=2 start-docker' if [ "$rc" = 0 ] && grep -q "Starting containerd" <<<"$out" && grep -q "Docker daemon is ready" <<<"$out"; then - pass "a socket no pidfile names that refuses is taken over, and the start goes on" -else fail "a socket no pidfile names that refuses is taken over, and the start goes on"; fi -run "mute-socket" 'python3 -c "import socket, time; s = socket.socket(socket.AF_UNIX); s.bind(\"/var/run/docker.sock\"); s.listen(); time.sleep(600)" & until [ -S /var/run/docker.sock ]; do sleep 0.05; done -START_DOCKER_PROBE_SECONDS=2 start-docker; echo "rc=$?" -pgrep -x containerd >/dev/null && echo "a containerd was started" || echo "nothing started"' -if grep -q "rc=1" <<<"$out" && grep -q "listens on /var/run/docker.sock and did not answer in 2s: not starting a second dockerd" <<<"$out" \ + pass "sockets no pidfile names that refuse, dockerd's and containerd's, are taken over, clients slow to start notwithstanding, and the start goes on" +else fail "sockets no pidfile names that refuse, dockerd's and containerd's, are taken over, clients slow to start notwithstanding, and the start goes on"; fi +run "mute-socket" "$listener"' +listen /var/run/docker.sock mute; m=$! +START_DOCKER_PROBE_SECONDS=2 start-docker; echo "dockerd: rc=$?" +kill "$m"; wait "$m"; rm -f /var/run/docker.sock +listen "$CSOCK" mute +START_DOCKER_PROBE_SECONDS=2 start-docker; echo "containerd: rc=$?" +pgrep -x containerd >/dev/null || pgrep -x dockerd >/dev/null && echo "a daemon was started" || echo "nothing started"' +if grep -q "dockerd: rc=1" <<<"$out" && grep -q "listens on /var/run/docker.sock and did not answer in 2s: not starting a second dockerd over it" <<<"$out" \ + && grep -q "containerd: rc=1" <<<"$out" && grep -q "listens on /var/run/docker/containerd/containerd.sock and did not answer in 2s: not starting a second containerd over it" <<<"$out" \ && grep -q "nothing started" <<<"$out"; then - pass "a socket no pidfile names that listens and does not answer fails the start, said, and nothing is started over it" -else fail "a socket no pidfile names that listens and does not answer fails the start, said, and nothing is started over it"; fi + pass "sockets no pidfile names that listen and do not answer, dockerd's and containerd's, fail the start, said, and nothing is started over them" +else fail "sockets no pidfile names that listen and do not answer, dockerd's and containerd's, fail the start, said, and nothing is started over them"; fi +# Answering as no such daemon does: an HTTP server on either socket, and a +# containerd without its Version service (its gRPC answers, Unimplemented). +run "foreign-socket" "$listener"' +listen /var/run/docker.sock http404; m=$! +START_DOCKER_PROBE_SECONDS=2 start-docker; echo "dockerd: rc=$?" +kill "$m"; wait "$m"; rm -f /var/run/docker.sock +listen "$CSOCK" http404; m=$! +START_DOCKER_PROBE_SECONDS=2 start-docker; echo "containerd, http: rc=$?" +kill "$m"; wait "$m"; rm -f "$CSOCK" +printf "version = 2\ndisabled_plugins = [\"io.containerd.grpc.v1.version\", \"io.containerd.grpc.v1.cri\"]\n" >/tmp/no-version.toml +containerd --config /tmp/no-version.toml --root /tmp/croot --state /tmp/cstate --address "$CSOCK" >/tmp/c.log 2>&1 & c=$! +until curl -s -o /dev/null --http2-prior-knowledge --unix-socket "$CSOCK" http://containerd/ || ! kill -0 "$c" 2>/dev/null; do sleep 0.1; done +START_DOCKER_PROBE_SECONDS=2 start-docker; echo "containerd, no Version: rc=$?" +echo "containerds: $(pgrep -cx containerd)"; pgrep -x dockerd >/dev/null && echo "a dockerd was started" || echo "no dockerd started"' +if grep -q "dockerd: rc=1" <<<"$out" && grep -q "answers on /var/run/docker.sock, but not as dockerd does (curl's exit 22 on its /_ping, HTTP status 404): not starting a second dockerd over it" <<<"$out" \ + && grep -q "containerd, http: rc=1" <<<"$out" && grep -q "answers on /var/run/docker/containerd/containerd.sock, but not as containerd does (curl's exit [0-9]* on its Version call): not starting a second containerd over it" <<<"$out" \ + && grep -q "containerd, no Version: rc=1" <<<"$out" && grep -q "but not as containerd does (its Version call answered grpc-status: 12)" <<<"$out" \ + && grep -q "containerds: 1" <<<"$out" && grep -q "no dockerd started" <<<"$out"; then + pass "sockets no pidfile names that answer as no such daemon does, dockerd's and containerd's, fail the start, said, and nothing is started over them" +else fail "sockets no pidfile names that answer as no such daemon does, dockerd's and containerd's, fail the start, said, and nothing is started over them"; fi +# A dockerd that answers and that no pidfile names (one a lane started some +# other way, or whose pidfiles are gone) is reused after one look: no second +# dockerd on the same socket. A ~/.curlrc that would change curl's request +# (HTTP/2, which dockerd's socket does not speak) changes neither start's. +run "unnamed-dockerd" 'echo --http2-prior-knowledge >/root/.curlrc && +START_DOCKER_SILENCE_CHECKS=60 start-docker >/dev/null && rm -f /var/run/docker.pid /var/run/docker/start-docker-dockerd.pid && +start-docker; echo "second rc=$?"; echo "dockerds: $(pgrep -cx dockerd)"' +if grep -q "Docker daemon already running" <<<"$out" && grep -q "second rc=0" <<<"$out" && grep -q "dockerds: 1" <<<"$out"; then + pass "a dockerd that answers and that no pidfile names is reused after one look, and no second one is started" +else fail "a dockerd that answers and that no pidfile names is reused after one look, and no second one is started"; fi # A start killed after it started dockerd, before dockerd wrote its own -# pidfile (dockerd slow to begin, as a starved one is): the next start finds -# it by the pid start-docker wrote, and waits for it. -run "killed-before-dockerd-pidfile" 'mkdir -p /opt/shim && printf "#!/bin/sh\nsleep 5\nexec /usr/bin/dockerd \"\$@\"\n" >/opt/shim/dockerd && chmod +x /opt/shim/dockerd +# pidfile (dockerd slow to begin, as a starved one is; here held until the +# case lets it go, once the next start has looked): the next start finds it +# by the pid start-docker wrote, and waits for it. +run "killed-before-dockerd-pidfile" 'mkdir -p /opt/shim && printf "#!/bin/sh\nuntil [ -e /tmp/go ]; do sleep 0.05; done\nexec /usr/bin/dockerd \"\$@\"\n" >/opt/shim/dockerd && chmod +x /opt/shim/dockerd PATH=/opt/shim:$PATH start-docker >/dev/null 2>&1 & first=$! until [ -s /var/run/docker/start-docker-dockerd.pid ]; do sleep 0.05; done kill -9 "$first"; wait "$first" 2>/dev/null +PATH=/opt/shim:$PATH start-docker >/tmp/second.out 2>&1 & second=$! +until grep -q "is already running; waiting for it to answer\|Starting dockerd" /tmp/second.out || ! kill -0 "$second" 2>/dev/null; do sleep 0.05; done [ -s /var/run/docker.pid ] || echo "dockerd has written no pidfile yet" -PATH=/opt/shim:$PATH start-docker; echo "second rc=$?"' +touch /tmp/go +wait "$second"; echo "second rc=$?"; cat /tmp/second.out' if grep -q "dockerd has written no pidfile yet" <<<"$out" && grep -q "dockerd ([0-9]*) is already running; waiting for it to answer" <<<"$out" \ && grep -q "second rc=0" <<<"$out"; then pass "a start killed before dockerd wrote its pidfile leaves its pid: the next start waits for that dockerd"