Enter git-forge-cloudflareHEADmaster

Runs git-forge on Cloudflare Containers with R2 for durable storage. A
Worker routes every request to one container through a Durable Object.
Inside it, syncd mirrors the data directory to the bucket and restores
it at boot, holding a write until it is durable.

git-forge is neither modified nor vendored: the image builds it from
its own repository at the commit named in forge.rev.

1 files changed, 1113 insertions, 0 deletions

diff --git a/sync_test.go b/sync_test.go
new file mode 100644
index 0000000..2158626
--- /dev/null
+++ b/sync_test.go
@@ -0,0 +1,1113 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// TestMirrorable pins which files under objects/ belong in the bucket. The
+// interesting cases are all state git is in the middle of writing.
+func TestMirrorable(t *testing.T) {
+	root := t.TempDir()
+
+	// mirrorable stats the index beside a pack, so these must be real files.
+	write := func(rel string) {
+		path := filepath.Join(root, filepath.FromSlash(rel))
+		if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(path, []byte("x"), 0600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	write("ab/cdef0123456789")
+	write("pack/pack-done.pack")
+	write("pack/pack-done.idx")
+	write("pack/pack-half.pack") // no index yet
+	write("pack/pack-held.pack")
+	write("pack/pack-held.idx")
+	write("pack/pack-held.keep") // receive-pack still has it open
+	write("pack/tmp_pack_writing")
+	write("info/commit-graph")
+	// receive-pack's quarantine: a temporary directory holding files whose own
+	// names say nothing about being temporary.
+	write("tmp_objdir-incoming-HBbblc/pack/pack-incoming.idx")
+	write("tmp_objdir-incoming-HBbblc/pack/pack-incoming.pack")
+	write("tmp_objdir-incoming-HBbblc/pack/pack-incoming.keep")
+
+	cases := []struct {
+		rel  string
+		want bool
+	}{
+		{"ab/cdef0123456789", true},
+		{"pack/pack-done.pack", true},
+		{"pack/pack-done.idx", true},
+		{"pack/pack-half.pack", false},
+		{"pack/pack-held.pack", true},
+		{"pack/pack-held.keep", false},
+		{"pack/tmp_pack_writing", false},
+		{"info/commit-graph", false},
+		{"tmp_objdir-incoming-HBbblc/pack/pack-incoming.idx", false},
+		{"tmp_objdir-incoming-HBbblc/pack/pack-incoming.pack", false},
+		{"tmp_objdir-incoming-HBbblc/pack/pack-incoming.keep", false},
+	}
+	for _, c := range cases {
+		if got := mirrorable(filepath.FromSlash(c.rel), root); got != c.want {
+			t.Errorf("mirrorable(%q) = %v, want %v", c.rel, got, c.want)
+		}
+	}
+}
+
+// TestScanRepos pins what the manifest holds, and so what a restore brings
+// back: repositories at the root, one section deep, and the issue and wiki
+// siblings.
+func TestScanRepos(t *testing.T) {
+	root := t.TempDir()
+	for _, d := range []string{
+		"notes.git", "notes.issues.git", "notes.wiki.git",
+		"team/api.git", "team/deeper/nested.git", "empty",
+	} {
+		if err := os.MkdirAll(filepath.Join(root, filepath.FromSlash(d)), 0700); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// Named like a repository and not a directory, so not one.
+	if err := os.WriteFile(filepath.Join(root, "afile.git"), []byte("x"), 0600); err != nil {
+		t.Fatal(err)
+	}
+
+	got, err := scanRepos(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// team/deeper/nested.git is absent on purpose: a section is one level deep,
+	// which is the rule git-forge scans by.
+	want := "notes.git notes.issues.git notes.wiki.git team/api.git"
+	if strings.Join(got, " ") != want {
+		t.Errorf("scanRepos = %q, want %q", strings.Join(got, " "), want)
+	}
+}
+
+// TestListFollowsTheCursor pins that a caller gets every key rather than the
+// first page. A repository with more than a thousand loose objects is ordinary,
+// and a listing that stops short is the one input that turns a hydration into a
+// deletion: what it did not fetch reads afterwards as an object gc replaced.
+func TestListFollowsTheCursor(t *testing.T) {
+	b, url := newBucket(t)
+	const prefix = "repos/x.git/objects/"
+	b.mu.Lock()
+	for i := range bucketPage*3 + 1 {
+		b.objs[fmt.Sprintf("%s%02d/object", prefix, i)] = []byte("x")
+	}
+	b.mu.Unlock()
+
+	keys, err := newR2(url).list(prefix)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(keys) != bucketPage*3+1 {
+		t.Errorf("a listing of %d keys came back with %d", bucketPage*3+1, len(keys))
+	}
+}
+
+// TestCheckKeys pins the one thing standing between a tampered bucket and a
+// file written outside the datadir.
+func TestCheckKeys(t *testing.T) {
+	const prefix = "repos/x.git/objects/"
+	cases := []struct {
+		key  string
+		want bool
+	}{
+		{prefix + "ab/cdef0123456789", true},
+		{prefix + "pack/pack-1.pack", true},
+		{prefix + "../../../../etc/passwd", false},
+		{prefix + "/etc/passwd", false},
+		{prefix, false},
+	}
+	for _, c := range cases {
+		if got := checkKeys([]string{c.key}, prefix) == nil; got != c.want {
+			t.Errorf("checkKeys(%q) accepted = %v, want %v", c.key, got, c.want)
+		}
+	}
+}
+
+// How many keys a listing answers at once. See the list case below.
+const bucketPage = 25
+
+// bucket is the object handler from worker.js, in memory. Object gets can be
+// held open and any request made to fail, which turns races into tests.
+type bucket struct {
+	mu      sync.Mutex
+	objs    map[string][]byte
+	failing func(method, key string) bool
+	gate    chan struct{} // object gets wait on this while it is not nil
+	blocked chan struct{} // closed once one has
+	once    sync.Once
+}
+
+func newBucket(t *testing.T) (*bucket, string) {
+	t.Helper()
+	b := &bucket{objs: map[string][]byte{}, blocked: make(chan struct{})}
+	srv := httptest.NewServer(b)
+	t.Cleanup(srv.Close)
+	return b, srv.URL
+}
+
+func (b *bucket) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	key := strings.TrimPrefix(r.URL.Path, "/")
+
+	b.mu.Lock()
+	failing, gate := b.failing, b.gate
+	b.mu.Unlock()
+	if failing != nil && failing(r.Method, key) {
+		http.Error(w, "the bucket is having a moment", http.StatusInternalServerError)
+		return
+	}
+	if gate != nil && strings.Contains(key, "/objects/") {
+		b.once.Do(func() { close(b.blocked) })
+		<-gate
+	}
+
+	switch {
+	case r.Method == http.MethodGet && key == "":
+		prefix := r.URL.Query().Get("prefix")
+		keys := []string{}
+		b.mu.Lock()
+		for k := range b.objs {
+			if strings.HasPrefix(k, prefix) {
+				keys = append(keys, k)
+			}
+		}
+		b.mu.Unlock()
+		// Paged, because r2 pages at a thousand and a listing that stops short
+		// hydrates part of a repository, which the pass after it reads as
+		// objects gc replaced and deletes. Small, so every test that lists
+		// anything with a repository in it follows a cursor.
+		slices.Sort(keys)
+		from := 0
+		if c := r.URL.Query().Get("cursor"); c != "" {
+			from, _ = strconv.Atoi(c)
+		}
+		to := min(from+bucketPage, len(keys))
+		next := ""
+		if to < len(keys) {
+			next = strconv.Itoa(to)
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{"keys": keys[from:to], "cursor": next})
+	case r.Method == http.MethodGet:
+		b.mu.Lock()
+		data, found := b.objs[key]
+		b.mu.Unlock()
+		if !found {
+			http.Error(w, "not found", http.StatusNotFound)
+			return
+		}
+		w.Header().Set("Content-Length", strconv.Itoa(len(data)))
+		_, _ = w.Write(data)
+	case r.Method == http.MethodPut:
+		data, err := io.ReadAll(r.Body)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		b.mu.Lock()
+		b.objs[key] = data
+		b.mu.Unlock()
+		w.WriteHeader(http.StatusNoContent)
+	case r.Method == http.MethodDelete:
+		b.mu.Lock()
+		delete(b.objs, key)
+		b.mu.Unlock()
+		w.WriteHeader(http.StatusNoContent)
+	default:
+		http.Error(w, "bad method", http.StatusMethodNotAllowed)
+	}
+}
+
+// hold blocks every object get until release, and reports when the first one is
+// waiting, which is a repository part way through being hydrated.
+func (b *bucket) hold() <-chan struct{} {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.gate = make(chan struct{})
+	return b.blocked
+}
+
+func (b *bucket) release() {
+	b.mu.Lock()
+	gate := b.gate
+	b.gate = nil
+	b.mu.Unlock()
+	close(gate)
+}
+
+func (b *bucket) fail(when func(method, key string) bool) {
+	b.mu.Lock()
+	b.failing = when
+	b.mu.Unlock()
+}
+
+func (b *bucket) object(key string) string {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return string(b.objs[key])
+}
+
+func (b *bucket) under(prefix string) []string {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	var keys []string
+	for k := range b.objs {
+		if strings.HasPrefix(k, prefix) {
+			keys = append(keys, k)
+		}
+	}
+	return keys
+}
+
+// gitIdentity keeps these tests independent of whoever runs them: git refuses
+// to write a commit without one.
+func gitIdentity(t *testing.T) {
+	t.Helper()
+	for _, v := range []string{"GIT_AUTHOR_NAME", "GIT_COMMITTER_NAME"} {
+		t.Setenv(v, "syncd test")
+	}
+	for _, v := range []string{"GIT_AUTHOR_EMAIL", "GIT_COMMITTER_EMAIL"} {
+		t.Setenv(v, "test@example.com")
+	}
+}
+
+// seed leaves the bucket holding what a forge that had taken one push would
+// have, through the same flush, and returns the commit master points at.
+func seed(t *testing.T, b *bucket, url string) string {
+	t.Helper()
+	dir := t.TempDir()
+	s := newSync(newR2(url), dir)
+	if err := s.restore(); err != nil {
+		t.Fatal(err)
+	}
+	repo := filepath.Join(dir, "repos", "x.git")
+	if err := gitInit(repo, true); err != nil {
+		t.Fatal(err)
+	}
+	head := commitOne(t, repo)
+	if err := s.flush(); err != nil {
+		t.Fatal(err)
+	}
+	if refs := b.object("repos/x.git/REFS"); !strings.Contains(refs, head) {
+		t.Fatalf("the seed did not store the refs, the bucket holds %q", refs)
+	}
+	return head
+}
+
+// commitOne puts one empty commit on master and returns it.
+func commitOne(t *testing.T, repo string) string {
+	t.Helper()
+	tree, err := git(repo, "", []byte{}, "mktree")
+	if err != nil {
+		t.Fatalf("mktree: %v: %s", err, tree)
+	}
+	commit, err := git(repo, "", nil, "commit-tree", strings.TrimSpace(string(tree)), "-m", "first")
+	if err != nil {
+		t.Fatalf("commit-tree: %v: %s", err, commit)
+	}
+	head := strings.TrimSpace(string(commit))
+	if out, err := git(repo, "", nil, "update-ref", "refs/heads/master", head); err != nil {
+		t.Fatalf("update-ref: %v: %s", err, out)
+	}
+	return head
+}
+
+// commitBlobs puts one commit holding n blobs on master. Enough of them land in
+// most of the fanout directories, which a gc racing a walk needs.
+func commitBlobs(t *testing.T, repo string, n int) string {
+	t.Helper()
+	var paths, entries strings.Builder
+	dir := t.TempDir()
+	for i := range n {
+		name := filepath.Join(dir, fmt.Sprintf("f%d", i))
+		if err := os.WriteFile(name, fmt.Appendf(nil, "content %d\n", i), 0600); err != nil {
+			t.Fatal(err)
+		}
+		paths.WriteString(name + "\n")
+	}
+	blobs, err := git(repo, "", []byte(paths.String()), "hash-object", "-w", "--stdin-paths")
+	if err != nil {
+		t.Fatalf("hash-object: %v: %s", err, blobs)
+	}
+	for i, sha := range strings.Fields(string(blobs)) {
+		fmt.Fprintf(&entries, "100644 blob %s\tf%d\n", sha, i)
+	}
+	tree, err := git(repo, "", []byte(entries.String()), "mktree")
+	if err != nil {
+		t.Fatalf("mktree: %v: %s", err, tree)
+	}
+	commit, err := git(repo, "", nil, "commit-tree", strings.TrimSpace(string(tree)), "-m", "many")
+	if err != nil {
+		t.Fatalf("commit-tree: %v: %s", err, commit)
+	}
+	head := strings.TrimSpace(string(commit))
+	if out, err := git(repo, "", nil, "update-ref", "refs/heads/master", head); err != nil {
+		t.Fatalf("update-ref: %v: %s", err, out)
+	}
+	return head
+}
+
+// Pruning waits a pass: deleting on the pass that first notices an object gone
+// can take one whose replacement is still on disk alone.
+func TestPruneWaitsForAPass(t *testing.T) {
+	gitIdentity(t)
+	b, url := newBucket(t)
+
+	dir := t.TempDir()
+	s := newSync(newR2(url), dir)
+	if err := s.restore(); err != nil {
+		t.Fatal(err)
+	}
+	repo := filepath.Join(dir, "repos", "x.git")
+	if err := gitInit(repo, true); err != nil {
+		t.Fatal(err)
+	}
+	head := commitOne(t, repo)
+	if err := s.flush(); err != nil {
+		t.Fatal(err)
+	}
+	out, err := git(repo, "", nil, "rev-parse", head+"^{tree}")
+	if err != nil {
+		t.Fatalf("rev-parse: %v: %s", err, out)
+	}
+	tree := strings.TrimSpace(string(out))
+	key := "repos/x.git/objects/" + tree[:2] + "/" + tree[2:]
+	if b.object(key) == "" {
+		t.Fatalf("the flush did not store the tree at %s", key)
+	}
+
+	// What gc does to a loose object the moment it has packed it.
+	if err := os.Remove(filepath.Join(repo, "objects", tree[:2], tree[2:])); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.flush(); err != nil {
+		t.Fatal(err)
+	}
+	if b.object(key) == "" {
+		t.Fatal("the pass that first found the object gone deleted it")
+	}
+	if err := s.flush(); err != nil {
+		t.Fatal(err)
+	}
+	if b.object(key) != "" {
+		t.Error("the pass after that did not delete it")
+	}
+}
+
+// A repository the manifest no longer lists leaves the bucket after the grace,
+// not at the first boot that notices: a manifest restored from a backup, or
+// written short, would otherwise take every repository it omits with it.
+func TestPruneWaitsOutTheGrace(t *testing.T) {
+	gitIdentity(t)
+	b, url := newBucket(t)
+
+	dir := t.TempDir()
+	s := newSync(newR2(url), dir)
+	if err := s.restore(); err != nil {
+		t.Fatal(err)
+	}
+	heads := map[string]string{}
+	for _, name := range []string{"keep.git", "drop.git"} {
+		repo := filepath.Join(dir, "repos", name)
+		if err := gitInit(repo, true); err != nil {
+			t.Fatal(err)
+		}
+		heads[name] = commitOne(t, repo)
+	}
+	if err := s.flush(); err != nil {
+		t.Fatal(err)
+	}
+
+	// What DeleteRepo leaves behind: the directory gone and the manifest
+	// rewritten without it, while everything it named is still in the bucket.
+	if err := os.RemoveAll(filepath.Join(dir, "repos", "drop.git")); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.flush(); err != nil {
+		t.Fatal(err)
+	}
+	if n := len(b.under("repos/drop.git/")); n == 0 {
+		t.Fatal("the flush that rewrote the manifest already removed the objects")
+	}
+
+	fresh := t.TempDir()
+	f := newSync(newR2(url), fresh)
+	if err := f.restore(); err != nil {
+		t.Fatal(err)
+	}
+	if err := f.pruneDeleted(); err != nil {
+		t.Fatal(err)
+	}
+	if len(b.under("repos/drop.git/")) == 0 {
+		t.Fatal("the first boot to find it unlisted removed it")
+	}
+	if !strings.Contains(b.object(staleKey), "drop.git") {
+		t.Fatalf("the bucket does not record when it went unlisted: %q", b.object(staleKey))
+	}
+
+	// Listed again, by a manifest put back: the record goes and nothing is
+	// removed.
+	f.setManifest([]string{"keep.git", "drop.git"})
+	if err := f.pruneDeleted(); err != nil {
+		t.Fatal(err)
+	}
+	if got := b.object(staleKey); got != "" {
+		t.Errorf("a repository listed again is still recorded as stale: %q", got)
+	}
+	if len(b.under("repos/drop.git/")) == 0 {
+		t.Fatal("a repository listed again was removed")
+	}
+
+	// Unlisted for longer than the grace.
+	f.setManifest([]string{"keep.git"})
+	b.mu.Lock()
+	b.objs[staleKey] = []byte(`{"drop.git":"2020-01-01T00:00:00Z"}`)
+	b.mu.Unlock()
+	if err := f.pruneDeleted(); err != nil {
+		t.Fatal(err)
+	}
+	if keys := b.under("repos/drop.git/"); len(keys) > 0 {
+		t.Errorf("the bucket still holds %d keys for a repository unlisted past the grace: %v", len(keys), keys)
+	}
+	if got := b.object(staleKey); got != "" {
+		t.Errorf("the record outlived the deletion: %q", got)
+	}
+
+	// The question worth asking of anything that deletes: whether it took
+	// something still reachable.
+	if keys := b.under("repos/keep.git/"); len(keys) == 0 {
+		t.Fatal("a repository the manifest lists was pruned too")
+	}
+	if err := f.hydrate("keep"); err != nil {
+		t.Fatalf("hydrating what the prune left: %v", err)
+	}
+	out, _ := git(filepath.Join(fresh, "repos", "keep.git"), "", nil, "show-ref")
+	if !strings.Contains(string(out), heads["keep.git"]) {
+		t.Errorf("the repository that survived is missing its refs, show-ref says %q", out)
+	}
+}
+
+// gc packs loose objects and deletes them, detached, while the flush a push
+// triggered is still uploading. Skipping what it took would write refs naming
+// objects the bucket never received, so the flush has to walk again.
+func TestFlushWhileGitRepacks(t *testing.T) {
+	gitIdentity(t)
+	b, url := newBucket(t)
+
+	dir := t.TempDir()
+	s := newSync(newR2(url), dir)
+	if err := s.restore(); err != nil {
+		t.Fatal(err)
+	}
+	repo := filepath.Join(dir, "repos", "x.git")
+	if err := gitInit(repo, true); err != nil {
+		t.Fatal(err)
+	}
+
+	// Enough reachable objects that the uploads outlast the walk: sixteen are in
+	// flight at a time, so the rest are still queued when repack runs.
+	commitBlobs(t, repo, 60)
+
+	blocked := b.hold()
+	done := make(chan error, 1)
+	go func() { done <- s.flush() }()
+	<-blocked
+	if out, err := git(repo, "", nil, "repack", "-a", "-d", "-q"); err != nil {
+		t.Fatalf("repack: %v: %s", err, out)
+	}
+	b.release()
+	if err := <-done; err != nil {
+		t.Fatalf("flush: %v", err)
+	}
+
+	// Not what the bucket holds but whether it is a repository, which only a
+	// fresh container answers.
+	fresh := t.TempDir()
+	f := newSync(newR2(url), fresh)
+	if err := f.restore(); err != nil {
+		t.Fatal(err)
+	}
+	if err := f.hydrate("x"); err != nil {
+		t.Fatalf("hydrating what the flush published: %v", err)
+	}
+	out, err := git(filepath.Join(fresh, "repos", "x.git"), "", nil, "rev-list", "--objects", "--all")
+	if err != nil {
+		t.Fatalf("the published repository is missing objects its refs name: %v: %s", err, out)
+	}
+	if n := len(strings.Split(strings.TrimSpace(string(out)), "\n")); n < 61 {
+		t.Errorf("only %d objects came back, want the commit, its tree and sixty blobs", n)
+	}
+}
+
+// flushRepo reads the ref state before it walks the objects. A push that lands
+// during the upload is left to the next flush rather than published by this
+// one: refs read afterwards would name a commit the walk never saw.
+func TestFlushWhileAPushLands(t *testing.T) {
+	gitIdentity(t)
+	b, url := newBucket(t)
+
+	dir := t.TempDir()
+	s := newSync(newR2(url), dir)
+	if err := s.restore(); err != nil {
+		t.Fatal(err)
+	}
+	repo := filepath.Join(dir, "repos", "x.git")
+	if err := gitInit(repo, true); err != nil {
+		t.Fatal(err)
+	}
+	// Enough that the uploads are still running when the commit below lands.
+	head := commitBlobs(t, repo, 60)
+
+	blocked := b.hold()
+	done := make(chan error, 1)
+	go func() { done <- s.flush() }()
+	<-blocked
+
+	// The walk is over by the time the first upload blocks, so this commit is
+	// exactly what a push arriving mid-flush leaves behind.
+	late := commitOne(t, repo)
+	b.release()
+	if err := <-done; err != nil {
+		t.Fatalf("flush: %v", err)
+	}
+
+	refs := b.object("repos/x.git/REFS")
+	if strings.Contains(refs, late) {
+		t.Fatalf("the flush published a commit it did not walk, refs hold %q", refs)
+	}
+	if !strings.Contains(refs, head) {
+		t.Fatalf("the flush published no refs at all, the bucket holds %q", refs)
+	}
+
+	// Whether the published state is servable, which only a fresh container
+	// answers.
+	fresh := t.TempDir()
+	f := newSync(newR2(url), fresh)
+	if err := f.restore(); err != nil {
+		t.Fatal(err)
+	}
+	if err := f.hydrate("x"); err != nil {
+		t.Fatalf("hydrating what the flush published: %v", err)
+	}
+	out, err := git(filepath.Join(fresh, "repos", "x.git"), "", nil, "rev-list", "--objects", head)
+	if err != nil {
+		t.Fatalf("the published repository is missing objects its refs name: %v: %s", err, out)
+	}
+}
+
+// The other half of a detached gc: it removes the fanout directories it
+// emptied, and a walk inside one when it goes stops there. Treating that as a
+// finished walk publishes refs naming objects the bucket never received.
+//
+// The window is small, so this takes several rounds. A flush that reports the
+// walk was short is a correct answer; success over a bucket that cannot serve
+// what it published is not.
+func TestFlushWhileGitPrunes(t *testing.T) {
+	gitIdentity(t)
+
+	published := 0
+	for round := range 8 {
+		_, url := newBucket(t)
+		dir := t.TempDir()
+		s := newSync(newR2(url), dir)
+		if err := s.restore(); err != nil {
+			t.Fatal(err)
+		}
+		repo := filepath.Join(dir, "repos", "x.git")
+		if err := gitInit(repo, true); err != nil {
+			t.Fatal(err)
+		}
+		head := commitBlobs(t, repo, 600)
+
+		// Where gc is the moment before it prunes: everything reachable is in a
+		// pack, and the loose copies it replaced are still on disk.
+		if out, err := git(repo, "", nil, "repack", "-a", "-q"); err != nil {
+			t.Fatalf("repack: %v: %s", err, out)
+		}
+
+		// prune-packed alongside the flush: the listing the flush does first is
+		// long enough for this to be well under way.
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			root := filepath.Join(repo, "objects")
+			fanout, err := os.ReadDir(root)
+			if err != nil {
+				return
+			}
+			for _, d := range fanout {
+				if d.IsDir() && len(d.Name()) == 2 {
+					_ = os.RemoveAll(filepath.Join(root, d.Name()))
+					time.Sleep(20 * time.Microsecond)
+				}
+			}
+		}()
+		err := s.flush()
+		<-done
+		if err != nil {
+			continue
+		}
+		published++
+
+		fresh := t.TempDir()
+		f := newSync(newR2(url), fresh)
+		if err := f.restore(); err != nil {
+			t.Fatal(err)
+		}
+		if err := f.hydrate("x"); err != nil {
+			t.Fatalf("round %d: the flush published refs the bucket cannot serve: %v", round, err)
+		}
+		out, err := git(filepath.Join(fresh, "repos", "x.git"), "", nil, "rev-list", "--objects", head)
+		if err != nil {
+			t.Fatalf("round %d: the published repository is missing objects its refs name: %v: %s", round, err, out)
+		}
+	}
+	// Every round refusing is a pass that checked nothing, and a flush that
+	// always refuses is its own bug.
+	if published == 0 {
+		t.Error("no round got as far as publishing, so nothing above was checked")
+	}
+}
+
+// Deleting a repository must not fail the flush of everything else. DeleteRepo
+// removes the directory, and the section with it when it held the last one, so
+// a pass can list one and find it gone by the time it reads it.
+//
+// The directories are renamed away rather than removed, so each stops existing
+// at an instant: that is the moment the pass has to survive. A removal caught
+// half done leaves something git will not read, which flush still fails on.
+//
+// The deletions start when the bucket sees the first upload, so they land while
+// the pass is inside the repositories.
+func TestFlushWhileRepositoriesAreDeleted(t *testing.T) {
+	gitIdentity(t)
+
+	for round := range 20 {
+		b := &bucket{objs: map[string][]byte{}, blocked: make(chan struct{})}
+		dir := t.TempDir()
+		away := t.TempDir()
+
+		var doomed []string
+		var once sync.Once
+		upload, deleted := make(chan struct{}), make(chan struct{})
+		release := func() { once.Do(func() { close(upload) }) }
+		go func() {
+			defer close(deleted)
+			<-upload
+			for i, p := range doomed {
+				_ = os.Rename(p, filepath.Join(away, strconv.Itoa(i)))
+				// The section goes with the last repository in it.
+				_ = os.Remove(filepath.Dir(p))
+				time.Sleep(time.Millisecond)
+			}
+		}()
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/repos/") {
+				release()
+			}
+			b.ServeHTTP(w, r)
+		}))
+
+		s := newSync(newR2(srv.URL), dir)
+		if err := s.restore(); err != nil {
+			t.Fatal(err)
+		}
+		keep := filepath.Join(dir, "repos", "keep.git")
+		if err := gitInit(keep, true); err != nil {
+			t.Fatal(err)
+		}
+		head := commitOne(t, keep)
+		// Sorted after keep.git, so the pass is still working when they go.
+		for i := range 12 {
+			p := filepath.Join(dir, "repos", fmt.Sprintf("team%d", i), "api.git")
+			if err := gitInit(p, true); err != nil {
+				t.Fatal(err)
+			}
+			commitOne(t, p)
+			doomed = append(doomed, p)
+		}
+
+		err := s.flush()
+		// The deleter has to be done with the round's directories before
+		// t.TempDir takes them away underneath it.
+		srv.Close()
+		release()
+		<-deleted
+		if err != nil {
+			t.Fatalf("round %d: a deletion elsewhere failed the flush: %v", round, err)
+		}
+		if refs := b.object("repos/keep.git/REFS"); !strings.Contains(refs, head) {
+			t.Fatalf("round %d: the repository that stayed is missing its refs, the bucket holds %q", round, refs)
+		}
+	}
+}
+
+// A repository being hydrated must be invisible to a flush: an objects
+// directory that git init has just made holds no refs yet, and a flush that saw
+// one would write that emptiness over the real ref state.
+func TestFlushDuringHydration(t *testing.T) {
+	gitIdentity(t)
+	b, url := newBucket(t)
+	head := seed(t, b, url)
+
+	dir := t.TempDir()
+	s := newSync(newR2(url), dir)
+	if err := s.restore(); err != nil {
+		t.Fatal(err)
+	}
+
+	blocked := b.hold()
+	done := make(chan error, 1)
+	go func() { done <- s.hydrate("x") }()
+	<-blocked
+
+	if err := s.flush(); err != nil {
+		t.Fatal(err)
+	}
+	b.release()
+	if err := <-done; err != nil {
+		t.Fatal(err)
+	}
+
+	if refs := b.object("repos/x.git/REFS"); !strings.Contains(refs, head) {
+		t.Errorf("the flush overwrote the stored refs, the bucket holds %q", refs)
+	}
+	out, _ := git(filepath.Join(dir, "repos", "x.git"), "", nil, "show-ref")
+	if !strings.Contains(string(out), head) {
+		t.Errorf("the hydrated repository has no refs, show-ref says %q", out)
+	}
+}
+
+// A move into the placeholder that failed part way must not wedge every attempt
+// after it: rename refuses an existing directory, so the debris of one failure
+// would otherwise be permanent for the life of the container.
+func TestHydrationAfterPartialMove(t *testing.T) {
+	gitIdentity(t)
+	b, url := newBucket(t)
+	head := seed(t, b, url)
+
+	dir := t.TempDir()
+	s := newSync(newR2(url), dir)
+	if err := s.restore(); err != nil {
+		t.Fatal(err)
+	}
+	// What a rename that died in the middle of the loop leaves: part of a
+	// repository, no objects directory, so still a placeholder.
+	for _, d := range []string{"refs/heads", "info"} {
+		if err := os.MkdirAll(filepath.Join(dir, "repos", "x.git", d), 0700); err != nil {
+			t.Fatal(err)
+		}
+	}
+	headfile := filepath.Join(dir, "repos", "x.git", "HEAD")
+	if err := os.WriteFile(headfile, []byte("ref: refs/heads/master\n"), 0600); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := s.hydrate("x"); err != nil {
+		t.Fatalf("hydrating over the debris: %v", err)
+	}
+	out, _ := git(filepath.Join(dir, "repos", "x.git"), "", nil, "show-ref")
+	if !strings.Contains(string(out), head) {
+		t.Errorf("the repository was not hydrated, show-ref says %q", out)
+	}
+}
+
+// A manifest write that failed must be carried up by the next flush: recording
+// it before the put would make one refused write permanent.
+func TestManifestRetriedAfterFailure(t *testing.T) {
+	gitIdentity(t)
+	b, url := newBucket(t)
+
+	dir := t.TempDir()
+	s := newSync(newR2(url), dir)
+	if err := s.restore(); err != nil {
+		t.Fatal(err)
+	}
+	if err := gitInit(filepath.Join(dir, "repos", "x.git"), true); err != nil {
+		t.Fatal(err)
+	}
+
+	b.fail(func(method, key string) bool {
+		return method == http.MethodPut && key == manifestKey
+	})
+	if err := s.flush(); err == nil {
+		t.Fatal("a flush whose manifest the bucket refused reported success")
+	}
+	b.fail(nil)
+
+	if err := s.flush(); err != nil {
+		t.Fatal(err)
+	}
+	if got := b.object(manifestKey); !strings.Contains(got, "x.git") {
+		t.Errorf("the repository is missing from the manifest, which holds %q", got)
+	}
+}
+
+// A forge with no repositories writes no manifest, because the directory set
+// never moved. A restore has to tell that apart from an empty bucket, or the
+// next container comes up with no accounts and flushes that over the real ones.
+func TestConfigRestoredWithoutManifest(t *testing.T) {
+	gitIdentity(t)
+	b, url := newBucket(t)
+
+	dir := t.TempDir()
+	s := newSync(newR2(url), dir)
+	if err := s.restore(); err != nil {
+		t.Fatal(err)
+	}
+	if err := gitInit(filepath.Join(dir, "config"), false); err != nil {
+		t.Fatal(err)
+	}
+	users := `{"alice":{}}`
+	if err := os.WriteFile(filepath.Join(dir, "config", "users.json"), []byte(users), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.flush(); err != nil {
+		t.Fatal(err)
+	}
+	if got := b.object(manifestKey); got != "" {
+		t.Fatalf("a forge with no repositories wrote a manifest holding %q", got)
+	}
+
+	fresh := t.TempDir()
+	f := newSync(newR2(url), fresh)
+	if err := f.restore(); err != nil {
+		t.Fatal(err)
+	}
+	got, err := os.ReadFile(filepath.Join(fresh, "config", "users.json"))
+	if err != nil {
+		t.Fatalf("the account table did not come back: %v", err)
+	}
+	if string(got) != users {
+		t.Fatalf("the account table came back as %q", got)
+	}
+	if err := f.flush(); err != nil {
+		t.Fatal(err)
+	}
+	if got := b.object("config/users.json"); got != users {
+		t.Fatalf("the flush after the restore left %q in the bucket", got)
+	}
+}
+
+// An absent manifest means no repositories, which is true of a forge that has
+// none and a lie about a bucket that has lost one. Believing the lie hands the
+// sweep every repository in the bucket as one nothing lists.
+func TestRestoreRefusesAMissingManifest(t *testing.T) {
+	gitIdentity(t)
+	b, url := newBucket(t)
+	seed(t, b, url)
+
+	b.mu.Lock()
+	delete(b.objs, manifestKey)
+	b.mu.Unlock()
+
+	f := newSync(newR2(url), t.TempDir())
+	if err := f.restore(); err == nil {
+		t.Fatal("a restore believed a bucket that holds repositories and no manifest")
+	}
+	if keys := b.under("repos/x.git/"); len(keys) == 0 {
+		t.Error("the repository was pruned anyway")
+	}
+
+	// And the case it has to stay true for: a forge that never had one.
+	empty, emptyURL := newBucket(t)
+	empty.mu.Lock()
+	empty.objs["config/users.json"] = []byte(`{"alice":{}}`)
+	empty.mu.Unlock()
+	if err := newSync(newR2(emptyURL), t.TempDir()).restore(); err != nil {
+		t.Errorf("a forge with no repositories was refused: %v", err)
+	}
+}
+
+// What a restore knows the bucket holds has to come from the keys it fetched.
+// Taking it from the directory instead would mean a file the bucket does not
+// hold is remembered as stored, and the flush after it leaves it on a disk that
+// does not survive.
+func TestConfigOnDiskOnlyIsStillMirrored(t *testing.T) {
+	gitIdentity(t)
+	b, url := newBucket(t)
+
+	seedDir := t.TempDir()
+	s := newSync(newR2(url), seedDir)
+	if err := s.restore(); err != nil {
+		t.Fatal(err)
+	}
+	if err := gitInit(filepath.Join(seedDir, "config"), false); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(seedDir, "config", "users.json"), []byte(`{"alice":{}}`), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.flush(); err != nil {
+		t.Fatal(err)
+	}
+
+	// A datadir that outlived the syncd that wrote it, holding a file whose
+	// flush never succeeded.
+	dirty := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(dirty, "config"), 0700); err != nil {
+		t.Fatal(err)
+	}
+	repos := `{"notes":{"visibility":"public"}}`
+	if err := os.WriteFile(filepath.Join(dirty, "config", "repos.json"), []byte(repos), 0600); err != nil {
+		t.Fatal(err)
+	}
+
+	f := newSync(newR2(url), dirty)
+	if err := f.restore(); err != nil {
+		t.Fatal(err)
+	}
+	if err := f.flush(); err != nil {
+		t.Fatal(err)
+	}
+	if got := b.object("config/repos.json"); got != repos {
+		t.Errorf("the bucket holds %q for a file only disk had", got)
+	}
+}
+
+// git-forge writes a config file by creating {name}.tmp{n} beside it and
+// renaming over the top, so a pass can catch one between the two. Mirroring it
+// stores a half written account table under a key nothing removes.
+func TestConfigTempFilesAreNotMirrored(t *testing.T) {
+	b, url := newBucket(t)
+
+	dir := t.TempDir()
+	s := newSync(newR2(url), dir)
+	if err := s.restore(); err != nil {
+		t.Fatal(err)
+	}
+	users := `{"alice":{}}`
+	if err := os.WriteFile(filepath.Join(dir, "config", "users.json"), []byte(users), 0600); err != nil {
+		t.Fatal(err)
+	}
+	// What git-forge's writeFile leaves beside it for the length of an fsync.
+	temp := "users.json.tmp3877658021"
+	if err := os.WriteFile(filepath.Join(dir, "config", temp), []byte(`{"alice":{"par`), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.flush(); err != nil {
+		t.Fatal(err)
+	}
+	if got := b.object("config/" + temp); got != "" {
+		t.Errorf("the bucket holds a file git-forge was still writing: %q", got)
+	}
+	if got := b.object("config/users.json"); got != users {
+		t.Errorf("the account table is not in the bucket, which holds %q", got)
+	}
+
+	// And one already there is left where it is rather than restored, because
+	// putting it back on disk is what makes it permanent.
+	b.mu.Lock()
+	b.objs["config/"+temp] = []byte(`{"alice":{"par`)
+	b.mu.Unlock()
+
+	fresh := t.TempDir()
+	f := newSync(newR2(url), fresh)
+	if err := f.restore(); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(filepath.Join(fresh, "config", temp)); err == nil {
+		t.Error("a restore wrote a half written config file back to disk")
+	}
+	if got, err := os.ReadFile(filepath.Join(fresh, "config", "users.json")); err != nil || string(got) != users {
+		t.Errorf("the account table did not come back: %q: %v", got, err)
+	}
+}
+
+// Only syncd writes this bucket, so a key of a shape it never writes means the
+// bucket has been tampered with.
+func TestRestoreRefusesAKeyItDidNotWrite(t *testing.T) {
+	for _, key := range []string{
+		"config/.git/config",
+		"config/.git/hooks/pre-commit",
+		"config/git/hooks/pre-commit",
+		"config/sub/dir/users.json",
+	} {
+		b, url := newBucket(t)
+		b.mu.Lock()
+		b.objs["config/users.json"] = []byte(`{}`)
+		b.objs[key] = []byte("#!/bin/sh\nid\n")
+		b.mu.Unlock()
+
+		dir := t.TempDir()
+		s := newSync(newR2(url), dir)
+		if err := s.restore(); err == nil {
+			t.Errorf("a restore believed %s", key)
+		}
+	}
+}
+
+// git-forge serves without an audit log when git will not initialise. The
+// working tree still has to be mirrored: it is what a restore reads.
+func TestConfigFlushedWithoutAuditLog(t *testing.T) {
+	b, url := newBucket(t)
+
+	dir := t.TempDir()
+	s := newSync(newR2(url), dir)
+	if err := s.restore(); err != nil {
+		t.Fatal(err)
+	}
+	users := `{"alice":{}}`
+	if err := os.WriteFile(filepath.Join(dir, "config", "users.json"), []byte(users), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.flush(); err != nil {
+		t.Fatal(err)
+	}
+	if got := b.object("config/users.json"); got != users {
+		t.Errorf("the account table is not in the bucket, which holds %q", got)
+	}
+}
+
+// A hydration that fails part way must leave the placeholder alone, so the next
+// request starts over. An objects directory left behind counts as hydrated.
+func TestHydrationRetriedAfterFailure(t *testing.T) {
+	gitIdentity(t)
+	b, url := newBucket(t)
+	head := seed(t, b, url)
+
+	dir := t.TempDir()
+	s := newSync(newR2(url), dir)
+	if err := s.restore(); err != nil {
+		t.Fatal(err)
+	}
+
+	b.fail(func(method, key string) bool {
+		return method == http.MethodGet && strings.Contains(key, "/objects/")
+	})
+	if err := s.hydrate("x"); err == nil {
+		t.Fatal("a hydration that could not read the objects reported success")
+	}
+	b.fail(nil)
+
+	if err := s.hydrate("x"); err != nil {
+		t.Fatalf("second attempt: %v", err)
+	}
+	out, _ := git(filepath.Join(dir, "repos", "x.git"), "", nil, "show-ref")
+	if !strings.Contains(string(out), head) {
+		t.Errorf("the second request did not hydrate it, show-ref says %q", out)
+	}
+	if err := s.flush(); err != nil {
+		t.Fatal(err)
+	}
+	if refs := b.object("repos/x.git/REFS"); !strings.Contains(refs, head) {
+		t.Errorf("the flush that followed overwrote the stored refs with %q", refs)
+	}
+}