diff options
Diffstat (limited to 'test')
| -rwxr-xr-x | test | 274 |
1 files changed, 274 insertions, 0 deletions
@@ -0,0 +1,274 @@ +#!/bin/sh +# Drives the real image end to end against a local object store: the container +# is destroyed and the forge comes back from the bucket alone. +# +# Needs docker, go, git, curl and openssl here, nothing extra inside the image. +# fake-r2.go stands in for the object handler in worker.js, so no Cloudflare +# account is needed. +set -eu + +IMAGE=${IMAGE:-git-forge-cf:smoke} +PORT=${PORT:-18081} +R2PORT=${R2PORT:-19000} +BASE=http://127.0.0.1:$PORT +NAME=git-forge-cf-smoke + +WORK=$(mktemp -d) +BUCKET=$WORK/bucket +R2PID="" +cleanup() { + docker rm -f "$NAME" >/dev/null 2>&1 || true + [ -n "$R2PID" ] && kill "$R2PID" 2>/dev/null || true + rm -rf "$WORK" +} +trap cleanup EXIT + +ok() { printf 'ok %s\n' "$1"; } +no() { + printf 'FAIL %s\n' "$1" >&2 + docker logs "$NAME" 2>&1 | tail -25 >&2 || true + exit 1 +} +want() { grep -q -- "$2" "$3" || no "$1"; ok "$1"; } +code() { + description=$1 expected=$2 + shift 2 + got=$(curl -s -o "$WORK/body" -w '%{http_code}' "$@") + [ "$got" = "$expected" ] || no "$description: got $got, wanted $expected" + ok "$description" +} +csrf() { grep -o 'name="csrf" value="[^"]*"' "$1" | head -1 | cut -d'"' -f4; } + +# Same trick git-forge's own test uses: the secret is the base32 of the ascii +# "12345678901234567890", so a code needs openssl and no base32 decoder. +SECRET=GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ +KEYHEX=3132333435363738393031323334353637383930 +totp() { + step=$(( $(date -u +%s) / 30 )) + # The counter as eight raw bytes. Octal escapes rather than \x, which is not + # POSIX and which dash does not interpret. + msg="" + for byte in $(printf '%016x' "$step" | sed 's/../& /g'); do + msg="$msg$(printf '\\%03o' "$((0x$byte))")" + done + digest=$(printf "$msg" | openssl dgst -sha1 -mac HMAC -macopt hexkey:$KEYHEX | sed 's/.*= *//') + offset=$(( 0x$(printf %s "$digest" | cut -c40) * 2 + 1 )) + chunk=$(printf %s "$digest" | cut -c${offset}-$((offset + 7))) + printf '%06d' $(( (0x$chunk & 0x7fffffff) % 1000000 )) +} + +export GIT_TERMINAL_PROMPT=0 GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null +export GIT_AUTHOR_NAME=Smoke GIT_AUTHOR_EMAIL=smoke@example.com +export GIT_COMMITTER_NAME=Smoke GIT_COMMITTER_EMAIL=smoke@example.com +# alice's cookie jar, as two curl flags. Left unquoted at every use on purpose. +alice="-c $WORK/alice -b $WORK/alice" + +# start <description> brings up a container against the same bucket. No volume, +# so the datadir it starts from is whatever it can fetch. +# +# GIT_FORGE_DATADIR is deliberately wrong: syncd mirrors /var/lib/git-forge and +# is what tells git-forge where to write, so a git-forge that took this value +# would serve normally while every flush went nowhere. +start() { + docker run -d --name "$NAME" \ + -p "$PORT:8081" \ + --add-host=host.docker.internal:host-gateway \ + -e SYNCD_R2="http://host.docker.internal:$R2PORT" \ + -e GIT_FORGE_DATADIR=/tmp/not-mirrored \ + -e GIT_FORGE_BASE_URL="$BASE" \ + -e GIT_FORGE_COOKIE_KEY=0123456789abcdef0123456789abcdef \ + -e SYNCD_LOOSE_LIMIT=1 \ + -e SYNCD_LOGS="http://host.docker.internal:$R2PORT/" \ + "$IMAGE" >/dev/null + i=0 + while [ $i -lt 100 ]; do + if curl -fsS "$BASE/healthz" >/dev/null 2>&1; then + ok "$1" + return + fi + i=$((i + 1)) + sleep 0.2 + done + docker logs "$NAME" >&2 || true + no "$1" +} + +echo "== build" +./fetch-forge >/dev/null +docker build -q -f Containerfile -t "$IMAGE" . >/dev/null +ok "the image builds" + +# Built rather than `go run`, so that R2PID is the server and not a wrapper +# whose child would outlive the kill and hold the port. +go build -o "$WORK/fake-r2" fake-r2.go +"$WORK/fake-r2" -dir "$BUCKET" -listen ":$R2PORT" >"$WORK/r2.log" 2>&1 & +R2PID=$! +sleep 1 +[ -d "$BUCKET" ] || no "the object store did not start" +ok "the object store is up" + +echo "== first start, on an empty bucket" +start "the container starts with nothing to restore" + +docker logs "$NAME" 2>&1 | grep -o 'token: .*' | cut -d' ' -f2 >"$WORK/tok" +TOKEN=$(cat "$WORK/tok") +[ -n "$TOKEN" ] || no "no setup token in the log" +curl -s $alice "$BASE/setup" >"$WORK/p" +code "the setup form makes the first admin" 303 $alice -o /dev/null \ + -d "csrf=$(csrf "$WORK/p")&token=$TOKEN&name=alice&password=hunter2hunter2&again=hunter2hunter2" \ + "$BASE/setup" + +curl -s $alice "$BASE/settings/totp" >"$WORK/p" +curl -s $alice -o /dev/null \ + -d "csrf=$(csrf "$WORK/p")&secret=$SECRET&code=$(totp)¤t=hunter2hunter2" \ + "$BASE/settings/totp" +code "the admin holds a second factor" 200 $alice "$BASE/admin" + +curl -s $alice "$BASE/admin" >"$WORK/p" +code "a repository is created" 303 $alice -o /dev/null \ + -d "csrf=$(csrf "$WORK/p")&name=notes&desc=Notes&visibility=public&issues=members&wiki=members" \ + "$BASE/admin/repo/create" + +curl -s $alice "$BASE/settings" >"$WORK/p" +curl -s $alice -d "csrf=$(csrf "$WORK/p")&repo=notes" "$BASE/settings/tokens" >"$WORK/minted" +PUSH=$(sed -n 's/.*<pre id="token">\(.*\)<\/pre>.*/\1/p' "$WORK/minted") +[ -n "$PUSH" ] || no "no push token was minted" +ok "a push token is minted" + +echo "== push" +git init -q "$WORK/src" +echo "first" >"$WORK/src/a.txt" +git -C "$WORK/src" add a.txt +git -C "$WORK/src" -c commit.gpgsign=false commit -qm "first" +git -C "$WORK/src" push -q "http://alice:$PUSH@127.0.0.1:$PORT/notes.git" master:master +ok "push over http succeeds" +HEAD=$(git -C "$WORK/src" rev-parse HEAD) + +# The push returned, so it is already in the bucket. Nothing is waited for: a +# sleep would hide exactly the bug this checks for. +[ -f "$BUCKET/repos/notes.git/REFS" ] || no "the refs are not in the bucket" +grep -q "$HEAD" "$BUCKET/repos/notes.git/REFS" || no "the bucket has the wrong head" +ok "the push was durable before the response returned" +find "$BUCKET/repos/notes.git/objects" -type f >"$WORK/objs" 2>/dev/null || true +[ -s "$WORK/objs" ] || no "no objects in the bucket" +ok "objects are in the bucket" +[ -f "$BUCKET/config/users.json" ] || no "the account table is not in the bucket" +ok "the config is in the bucket" + +# git sends anything past http.postBuffer, a megabyte by default, with chunked +# transfer encoding, and that body crosses syncd's proxy. Every other push here +# is a few hundred bytes. Random, so the pack is still over the buffer after +# compression. +head -c 3000000 /dev/urandom >"$WORK/src/big.bin" +git -C "$WORK/src" add big.bin +git -C "$WORK/src" -c commit.gpgsign=false commit -qm "big" +git -C "$WORK/src" push -q "http://alice:$PUSH@127.0.0.1:$PORT/notes.git" master:master +ok "a push past git's post buffer succeeds" +# Reassigned, so everything after the restart is checked against the large push +# rather than the small one. +HEAD=$(git -C "$WORK/src" rev-parse HEAD) +grep -q "$HEAD" "$BUCKET/repos/notes.git/REFS" || no "the chunked push is not in the bucket" +ok "and it was durable before the response returned too" + +echo "== destroy the container, keep only the bucket" +docker rm -f "$NAME" >/dev/null +ok "the container and its disk are gone" + +start "a fresh container starts from the bucket alone" + +# Lazy restore: the directory has to exist so git-forge's startup scan finds it, +# but nothing has asked for the repository yet, so it must still be empty. +docker exec "$NAME" ls /var/lib/git-forge/repos/notes.git >"$WORK/before" 2>&1 || true +[ -s "$WORK/before" ] && no "the repository was fetched before anything asked for it" +ok "the repository is a placeholder until it is asked for" + +curl -s "$BASE/" >"$WORK/home" +want "the index lists it anyway" "notes" "$WORK/home" + +echo "== hydrate on first touch" +git clone -q "http://127.0.0.1:$PORT/notes.git" "$WORK/clone" 2>"$WORK/cloneerr" \ + || { cat "$WORK/cloneerr" >&2; no "the clone failed"; } +[ "$(git -C "$WORK/clone" rev-parse HEAD)" = "$HEAD" ] || no "the clone is at the wrong commit" +ok "the clone comes back at the same commit" +want "and the content survived" "first" "$WORK/clone/a.txt" + +docker exec "$NAME" ls /var/lib/git-forge/repos/notes.git >"$WORK/after" 2>&1 || true +grep -q objects "$WORK/after" || no "the repository was not filled in" +ok "the repository is filled in only after it was asked for" + +echo "== the accounts survived too" +# The push token is a digest in users.json, so a push that authenticates proves +# the account table came back, and proves a restored repository takes writes. +echo "second" >"$WORK/clone/b.txt" +git -C "$WORK/clone" add b.txt +git -C "$WORK/clone" -c commit.gpgsign=false commit -qm "second" +git -C "$WORK/clone" push -q "http://alice:$PUSH@127.0.0.1:$PORT/notes.git" master:master +ok "the push token still authenticates after the restore" +SECOND=$(git -C "$WORK/clone" rev-parse HEAD) +grep -q "$SECOND" "$BUCKET/repos/notes.git/REFS" || no "the second push is not in the bucket" +ok "and the second push is durable too" + +# alice enrolled a second factor before the restart, so a password alone must +# now be answered with a demand for a code rather than a session. +curl -s -o "$WORK/login" -w '%{http_code}' \ + -d "name=alice&password=hunter2hunter2" "$BASE/login" >"$WORK/lc" +grep -qx 200 "$WORK/lc" || no "signing in did not reach the second factor" +want "the second factor survived the restore" 'name="code"' "$WORK/login" + +echo "== compaction packs the audit log and prunes what it replaced" +# SYNCD_LOOSE_LIMIT is 1 here, so a sweep packs rather than waiting for the five +# hundred a deployment would. Long enough for two of them, because pruning waits +# for the pass after the one that finds an object gone. +loose() { find "$BUCKET/config/git/objects" -type f -not -path '*/pack/*' 2>/dev/null | wc -l; } +before=$(loose) +[ "$before" -gt 0 ] || no "there were no loose objects to pack" +sleep 35 +ls "$BUCKET"/config/git/objects/pack/*.pack >/dev/null 2>&1 || no "the audit log was not packed" +ok "the audit log is packed" +after=$(loose) +[ "$after" -lt "$before" ] || no "nothing was pruned: $before loose objects before, $after after" +ok "what packing replaced was pruned from the bucket ($before down to $after)" + +# Pruning deletes from object storage, so the question is whether it deleted +# something still reachable. +docker rm -f "$NAME" >/dev/null +start "a third container starts from the compacted bucket" +curl -s "$BASE/" >"$WORK/home3" +want "the index still lists the repository" "notes" "$WORK/home3" +git clone -q "http://127.0.0.1:$PORT/notes.git" "$WORK/clone3" 2>"$WORK/err3" \ + || { cat "$WORK/err3" >&2; no "the clone failed after compaction"; } +[ "$(git -C "$WORK/clone3" rev-parse HEAD)" = "$SECOND" ] || no "the clone is at the wrong commit after compaction" +ok "and a clone after compaction is still at the right commit" + +echo "== an idle forge writes nothing" +# The sweep runs every fifteen seconds and must find nothing to do. +before=$(grep -c 'PUT ' "$WORK/r2.log" || true) +sleep 20 +after=$(grep -c 'PUT ' "$WORK/r2.log" || true) +[ "$before" = "$after" ] || no "an idle sweep wrote $((after - before)) objects, it should write none" +ok "a sweep over unchanged state writes nothing" + +echo "== a graceful stop is durable and says so" +# SIGTERM, not a kill: syncd drains git-forge, flushes once more, and ships +# those last lines rather than losing them with the process. +grep -q 'container: syncd:' "$WORK/r2.log" || no "no container logs were shipped at all" +ok "container logs are shipped" +docker stop -t 30 "$NAME" >/dev/null +grep -q 'container: syncd: draining' "$WORK/r2.log" || no "the drain was never reported" +ok "SIGTERM drains git-forge" +grep -q 'container: syncd: flushed and stopping' "$WORK/r2.log" || no "the final flush was never reported" +ok "and the final flush is reported before the process exits" + +echo "== a container that refuses to start says why" +# Container stdout is forwarded nowhere, so a fatal must ship its line before it +# goes. No cookie key is the cheapest way to make it refuse. +docker run --rm --name "$NAME-fatal" \ + --add-host=host.docker.internal:host-gateway \ + -e SYNCD_R2="http://host.docker.internal:$R2PORT" \ + -e SYNCD_LOGS="http://host.docker.internal:$R2PORT/" \ + "$IMAGE" >/dev/null 2>&1 || true +sleep 1 +want "the reason it refused to start reaches the log" \ + 'container: syncd: GIT_FORGE_COOKIE_KEY must be set' "$WORK/r2.log" + +printf '\nall good\n' |