path: root/sync_test.go

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"slices"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"
)

// TestMirrorable pins which files under objects/ belong in the bucket. The
// interesting cases are all state git is in the middle of writing.
func TestMirrorable(t *testing.T) {
	root := t.TempDir()

	// mirrorable stats the index beside a pack, so these must be real files.
	write := func(rel string) {
		path := filepath.Join(root, filepath.FromSlash(rel))
		if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(path, []byte("x"), 0600); err != nil {
			t.Fatal(err)
		}
	}
	write("ab/cdef0123456789")
	write("pack/pack-done.pack")
	write("pack/pack-done.idx")
	write("pack/pack-half.pack") // no index yet
	write("pack/pack-held.pack")
	write("pack/pack-held.idx")
	write("pack/pack-held.keep") // receive-pack still has it open
	write("pack/tmp_pack_writing")
	write("info/commit-graph")
	// receive-pack's quarantine: a temporary directory holding files whose own
	// names say nothing about being temporary.
	write("tmp_objdir-incoming-HBbblc/pack/pack-incoming.idx")
	write("tmp_objdir-incoming-HBbblc/pack/pack-incoming.pack")
	write("tmp_objdir-incoming-HBbblc/pack/pack-incoming.keep")

	cases := []struct {
		rel  string
		want bool
	}{
		{"ab/cdef0123456789", true},
		{"pack/pack-done.pack", true},
		{"pack/pack-done.idx", true},
		{"pack/pack-half.pack", false},
		{"pack/pack-held.pack", true},
		{"pack/pack-held.keep", false},
		{"pack/tmp_pack_writing", false},
		{"info/commit-graph", false},
		{"tmp_objdir-incoming-HBbblc/pack/pack-incoming.idx", false},
		{"tmp_objdir-incoming-HBbblc/pack/pack-incoming.pack", false},
		{"tmp_objdir-incoming-HBbblc/pack/pack-incoming.keep", false},
	}
	for _, c := range cases {
		if got := mirrorable(filepath.FromSlash(c.rel), root); got != c.want {
			t.Errorf("mirrorable(%q) = %v, want %v", c.rel, got, c.want)
		}
	}
}

// TestScanRepos pins what the manifest holds, and so what a restore brings
// back: repositories at the root, one section deep, and the issue and wiki
// siblings.
func TestScanRepos(t *testing.T) {
	root := t.TempDir()
	for _, d := range []string{
		"notes.git", "notes.issues.git", "notes.wiki.git",
		"team/api.git", "team/deeper/nested.git", "empty",
	} {
		if err := os.MkdirAll(filepath.Join(root, filepath.FromSlash(d)), 0700); err != nil {
			t.Fatal(err)
		}
	}
	// Named like a repository and not a directory, so not one.
	if err := os.WriteFile(filepath.Join(root, "afile.git"), []byte("x"), 0600); err != nil {
		t.Fatal(err)
	}

	got, err := scanRepos(root)
	if err != nil {
		t.Fatal(err)
	}
	// team/deeper/nested.git is absent on purpose: a section is one level deep,
	// which is the rule git-forge scans by.
	want := "notes.git notes.issues.git notes.wiki.git team/api.git"
	if strings.Join(got, " ") != want {
		t.Errorf("scanRepos = %q, want %q", strings.Join(got, " "), want)
	}
}

// TestListFollowsTheCursor pins that a caller gets every key rather than the
// first page. A repository with more than a thousand loose objects is ordinary,
// and a listing that stops short is the one input that turns a hydration into a
// deletion: what it did not fetch reads afterwards as an object gc replaced.
func TestListFollowsTheCursor(t *testing.T) {
	b, url := newBucket(t)
	const prefix = "repos/x.git/objects/"
	b.mu.Lock()
	for i := range bucketPage*3 + 1 {
		b.objs[fmt.Sprintf("%s%02d/object", prefix, i)] = []byte("x")
	}
	b.mu.Unlock()

	keys, err := newR2(url).list(prefix)
	if err != nil {
		t.Fatal(err)
	}
	if len(keys) != bucketPage*3+1 {
		t.Errorf("a listing of %d keys came back with %d", bucketPage*3+1, len(keys))
	}
}

// TestCheckKeys pins the one thing standing between a tampered bucket and a
// file written outside the datadir.
func TestCheckKeys(t *testing.T) {
	const prefix = "repos/x.git/objects/"
	cases := []struct {
		key  string
		want bool
	}{
		{prefix + "ab/cdef0123456789", true},
		{prefix + "pack/pack-1.pack", true},
		{prefix + "../../../../etc/passwd", false},
		{prefix + "/etc/passwd", false},
		{prefix, false},
	}
	for _, c := range cases {
		if got := checkKeys([]string{c.key}, prefix) == nil; got != c.want {
			t.Errorf("checkKeys(%q) accepted = %v, want %v", c.key, got, c.want)
		}
	}
}

// How many keys a listing answers at once. See the list case below.
const bucketPage = 25

// bucket is the object handler from worker.js, in memory. Object gets can be
// held open and any request made to fail, which turns races into tests.
type bucket struct {
	mu      sync.Mutex
	objs    map[string][]byte
	failing func(method, key string) bool
	gate    chan struct{} // object gets wait on this while it is not nil
	blocked chan struct{} // closed once one has
	once    sync.Once
}

func newBucket(t *testing.T) (*bucket, string) {
	t.Helper()
	b := &bucket{objs: map[string][]byte{}, blocked: make(chan struct{})}
	srv := httptest.NewServer(b)
	t.Cleanup(srv.Close)
	return b, srv.URL
}

func (b *bucket) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	key := strings.TrimPrefix(r.URL.Path, "/")

	b.mu.Lock()
	failing, gate := b.failing, b.gate
	b.mu.Unlock()
	if failing != nil && failing(r.Method, key) {
		http.Error(w, "the bucket is having a moment", http.StatusInternalServerError)
		return
	}
	if gate != nil && strings.Contains(key, "/objects/") {
		b.once.Do(func() { close(b.blocked) })
		<-gate
	}

	switch {
	case r.Method == http.MethodGet && key == "":
		prefix := r.URL.Query().Get("prefix")
		keys := []string{}
		b.mu.Lock()
		for k := range b.objs {
			if strings.HasPrefix(k, prefix) {
				keys = append(keys, k)
			}
		}
		b.mu.Unlock()
		// Paged, because r2 pages at a thousand and a listing that stops short
		// hydrates part of a repository, which the pass after it reads as
		// objects gc replaced and deletes. Small, so every test that lists
		// anything with a repository in it follows a cursor.
		slices.Sort(keys)
		from := 0
		if c := r.URL.Query().Get("cursor"); c != "" {
			from, _ = strconv.Atoi(c)
		}
		to := min(from+bucketPage, len(keys))
		next := ""
		if to < len(keys) {
			next = strconv.Itoa(to)
		}
		_ = json.NewEncoder(w).Encode(map[string]any{"keys": keys[from:to], "cursor": next})
	case r.Method == http.MethodGet:
		b.mu.Lock()
		data, found := b.objs[key]
		b.mu.Unlock()
		if !found {
			http.Error(w, "not found", http.StatusNotFound)
			return
		}
		w.Header().Set("Content-Length", strconv.Itoa(len(data)))
		_, _ = w.Write(data)
	case r.Method == http.MethodPut:
		data, err := io.ReadAll(r.Body)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		b.mu.Lock()
		b.objs[key] = data
		b.mu.Unlock()
		w.WriteHeader(http.StatusNoContent)
	case r.Method == http.MethodDelete:
		b.mu.Lock()
		delete(b.objs, key)
		b.mu.Unlock()
		w.WriteHeader(http.StatusNoContent)
	default:
		http.Error(w, "bad method", http.StatusMethodNotAllowed)
	}
}

// hold blocks every object get until release, and reports when the first one is
// waiting, which is a repository part way through being hydrated.
func (b *bucket) hold() <-chan struct{} {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.gate = make(chan struct{})
	return b.blocked
}

func (b *bucket) release() {
	b.mu.Lock()
	gate := b.gate
	b.gate = nil
	b.mu.Unlock()
	close(gate)
}

func (b *bucket) fail(when func(method, key string) bool) {
	b.mu.Lock()
	b.failing = when
	b.mu.Unlock()
}

func (b *bucket) object(key string) string {
	b.mu.Lock()
	defer b.mu.Unlock()
	return string(b.objs[key])
}

func (b *bucket) under(prefix string) []string {
	b.mu.Lock()
	defer b.mu.Unlock()
	var keys []string
	for k := range b.objs {
		if strings.HasPrefix(k, prefix) {
			keys = append(keys, k)
		}
	}
	return keys
}

// gitIdentity keeps these tests independent of whoever runs them: git refuses
// to write a commit without one.
func gitIdentity(t *testing.T) {
	t.Helper()
	for _, v := range []string{"GIT_AUTHOR_NAME", "GIT_COMMITTER_NAME"} {
		t.Setenv(v, "syncd test")
	}
	for _, v := range []string{"GIT_AUTHOR_EMAIL", "GIT_COMMITTER_EMAIL"} {
		t.Setenv(v, "test@example.com")
	}
}

// seed leaves the bucket holding what a forge that had taken one push would
// have, through the same flush, and returns the commit master points at.
func seed(t *testing.T, b *bucket, url string) string {
	t.Helper()
	dir := t.TempDir()
	s := newSync(newR2(url), dir)
	if err := s.restore(); err != nil {
		t.Fatal(err)
	}
	repo := filepath.Join(dir, "repos", "x.git")
	if err := gitInit(repo, true); err != nil {
		t.Fatal(err)
	}
	head := commitOne(t, repo)
	if err := s.flush(); err != nil {
		t.Fatal(err)
	}
	if refs := b.object("repos/x.git/REFS"); !strings.Contains(refs, head) {
		t.Fatalf("the seed did not store the refs, the bucket holds %q", refs)
	}
	return head
}

// commitOne puts one empty commit on master and returns it.
func commitOne(t *testing.T, repo string) string {
	t.Helper()
	tree, err := git(repo, "", []byte{}, "mktree")
	if err != nil {
		t.Fatalf("mktree: %v: %s", err, tree)
	}
	commit, err := git(repo, "", nil, "commit-tree", strings.TrimSpace(string(tree)), "-m", "first")
	if err != nil {
		t.Fatalf("commit-tree: %v: %s", err, commit)
	}
	head := strings.TrimSpace(string(commit))
	if out, err := git(repo, "", nil, "update-ref", "refs/heads/master", head); err != nil {
		t.Fatalf("update-ref: %v: %s", err, out)
	}
	return head
}

// commitBlobs puts one commit holding n blobs on master. Enough of them land in
// most of the fanout directories, which a gc racing a walk needs.
func commitBlobs(t *testing.T, repo string, n int) string {
	t.Helper()
	var paths, entries strings.Builder
	dir := t.TempDir()
	for i := range n {
		name := filepath.Join(dir, fmt.Sprintf("f%d", i))
		if err := os.WriteFile(name, fmt.Appendf(nil, "content %d\n", i), 0600); err != nil {
			t.Fatal(err)
		}
		paths.WriteString(name + "\n")
	}
	blobs, err := git(repo, "", []byte(paths.String()), "hash-object", "-w", "--stdin-paths")
	if err != nil {
		t.Fatalf("hash-object: %v: %s", err, blobs)
	}
	for i, sha := range strings.Fields(string(blobs)) {
		fmt.Fprintf(&entries, "100644 blob %s\tf%d\n", sha, i)
	}
	tree, err := git(repo, "", []byte(entries.String()), "mktree")
	if err != nil {
		t.Fatalf("mktree: %v: %s", err, tree)
	}
	commit, err := git(repo, "", nil, "commit-tree", strings.TrimSpace(string(tree)), "-m", "many")
	if err != nil {
		t.Fatalf("commit-tree: %v: %s", err, commit)
	}
	head := strings.TrimSpace(string(commit))
	if out, err := git(repo, "", nil, "update-ref", "refs/heads/master", head); err != nil {
		t.Fatalf("update-ref: %v: %s", err, out)
	}
	return head
}

// Pruning waits a pass: deleting on the pass that first notices an object gone
// can take one whose replacement is still on disk alone.
func TestPruneWaitsForAPass(t *testing.T) {
	gitIdentity(t)
	b, url := newBucket(t)

	dir := t.TempDir()
	s := newSync(newR2(url), dir)
	if err := s.restore(); err != nil {
		t.Fatal(err)
	}
	repo := filepath.Join(dir, "repos", "x.git")
	if err := gitInit(repo, true); err != nil {
		t.Fatal(err)
	}
	head := commitOne(t, repo)
	if err := s.flush(); err != nil {
		t.Fatal(err)
	}
	out, err := git(repo, "", nil, "rev-parse", head+"^{tree}")
	if err != nil {
		t.Fatalf("rev-parse: %v: %s", err, out)
	}
	tree := strings.TrimSpace(string(out))
	key := "repos/x.git/objects/" + tree[:2] + "/" + tree[2:]
	if b.object(key) == "" {
		t.Fatalf("the flush did not store the tree at %s", key)
	}

	// What gc does to a loose object the moment it has packed it.
	if err := os.Remove(filepath.Join(repo, "objects", tree[:2], tree[2:])); err != nil {
		t.Fatal(err)
	}
	if err := s.flush(); err != nil {
		t.Fatal(err)
	}
	if b.object(key) == "" {
		t.Fatal("the pass that first found the object gone deleted it")
	}
	if err := s.flush(); err != nil {
		t.Fatal(err)
	}
	if b.object(key) != "" {
		t.Error("the pass after that did not delete it")
	}
}

// A repository the manifest no longer lists leaves the bucket after the grace,
// not at the first boot that notices: a manifest restored from a backup, or
// written short, would otherwise take every repository it omits with it.
func TestPruneWaitsOutTheGrace(t *testing.T) {
	gitIdentity(t)
	b, url := newBucket(t)

	dir := t.TempDir()
	s := newSync(newR2(url), dir)
	if err := s.restore(); err != nil {
		t.Fatal(err)
	}
	heads := map[string]string{}
	for _, name := range []string{"keep.git", "drop.git"} {
		repo := filepath.Join(dir, "repos", name)
		if err := gitInit(repo, true); err != nil {
			t.Fatal(err)
		}
		heads[name] = commitOne(t, repo)
	}
	if err := s.flush(); err != nil {
		t.Fatal(err)
	}

	// What DeleteRepo leaves behind: the directory gone and the manifest
	// rewritten without it, while everything it named is still in the bucket.
	if err := os.RemoveAll(filepath.Join(dir, "repos", "drop.git")); err != nil {
		t.Fatal(err)
	}
	if err := s.flush(); err != nil {
		t.Fatal(err)
	}
	if n := len(b.under("repos/drop.git/")); n == 0 {
		t.Fatal("the flush that rewrote the manifest already removed the objects")
	}

	fresh := t.TempDir()
	f := newSync(newR2(url), fresh)
	if err := f.restore(); err != nil {
		t.Fatal(err)
	}
	if err := f.pruneDeleted(); err != nil {
		t.Fatal(err)
	}
	if len(b.under("repos/drop.git/")) == 0 {
		t.Fatal("the first boot to find it unlisted removed it")
	}
	if !strings.Contains(b.object(staleKey), "drop.git") {
		t.Fatalf("the bucket does not record when it went unlisted: %q", b.object(staleKey))
	}

	// Listed again, by a manifest put back: the record goes and nothing is
	// removed.
	f.setManifest([]string{"keep.git", "drop.git"})
	if err := f.pruneDeleted(); err != nil {
		t.Fatal(err)
	}
	if got := b.object(staleKey); got != "" {
		t.Errorf("a repository listed again is still recorded as stale: %q", got)
	}
	if len(b.under("repos/drop.git/")) == 0 {
		t.Fatal("a repository listed again was removed")
	}

	// Unlisted for longer than the grace.
	f.setManifest([]string{"keep.git"})
	b.mu.Lock()
	b.objs[staleKey] = []byte(`{"drop.git":"2020-01-01T00:00:00Z"}`)
	b.mu.Unlock()
	if err := f.pruneDeleted(); err != nil {
		t.Fatal(err)
	}
	if keys := b.under("repos/drop.git/"); len(keys) > 0 {
		t.Errorf("the bucket still holds %d keys for a repository unlisted past the grace: %v", len(keys), keys)
	}
	if got := b.object(staleKey); got != "" {
		t.Errorf("the record outlived the deletion: %q", got)
	}

	// The question worth asking of anything that deletes: whether it took
	// something still reachable.
	if keys := b.under("repos/keep.git/"); len(keys) == 0 {
		t.Fatal("a repository the manifest lists was pruned too")
	}
	if err := f.hydrate("keep"); err != nil {
		t.Fatalf("hydrating what the prune left: %v", err)
	}
	out, _ := git(filepath.Join(fresh, "repos", "keep.git"), "", nil, "show-ref")
	if !strings.Contains(string(out), heads["keep.git"]) {
		t.Errorf("the repository that survived is missing its refs, show-ref says %q", out)
	}
}

// gc packs loose objects and deletes them, detached, while the flush a push
// triggered is still uploading. Skipping what it took would write refs naming
// objects the bucket never received, so the flush has to walk again.
func TestFlushWhileGitRepacks(t *testing.T) {
	gitIdentity(t)
	b, url := newBucket(t)

	dir := t.TempDir()
	s := newSync(newR2(url), dir)
	if err := s.restore(); err != nil {
		t.Fatal(err)
	}
	repo := filepath.Join(dir, "repos", "x.git")
	if err := gitInit(repo, true); err != nil {
		t.Fatal(err)
	}

	// Enough reachable objects that the uploads outlast the walk: sixteen are in
	// flight at a time, so the rest are still queued when repack runs.
	commitBlobs(t, repo, 60)

	blocked := b.hold()
	done := make(chan error, 1)
	go func() { done <- s.flush() }()
	<-blocked
	if out, err := git(repo, "", nil, "repack", "-a", "-d", "-q"); err != nil {
		t.Fatalf("repack: %v: %s", err, out)
	}
	b.release()
	if err := <-done; err != nil {
		t.Fatalf("flush: %v", err)
	}

	// Not what the bucket holds but whether it is a repository, which only a
	// fresh container answers.
	fresh := t.TempDir()
	f := newSync(newR2(url), fresh)
	if err := f.restore(); err != nil {
		t.Fatal(err)
	}
	if err := f.hydrate("x"); err != nil {
		t.Fatalf("hydrating what the flush published: %v", err)
	}
	out, err := git(filepath.Join(fresh, "repos", "x.git"), "", nil, "rev-list", "--objects", "--all")
	if err != nil {
		t.Fatalf("the published repository is missing objects its refs name: %v: %s", err, out)
	}
	if n := len(strings.Split(strings.TrimSpace(string(out)), "\n")); n < 61 {
		t.Errorf("only %d objects came back, want the commit, its tree and sixty blobs", n)
	}
}

// flushRepo reads the ref state before it walks the objects. A push that lands
// during the upload is left to the next flush rather than published by this
// one: refs read afterwards would name a commit the walk never saw.
func TestFlushWhileAPushLands(t *testing.T) {
	gitIdentity(t)
	b, url := newBucket(t)

	dir := t.TempDir()
	s := newSync(newR2(url), dir)
	if err := s.restore(); err != nil {
		t.Fatal(err)
	}
	repo := filepath.Join(dir, "repos", "x.git")
	if err := gitInit(repo, true); err != nil {
		t.Fatal(err)
	}
	// Enough that the uploads are still running when the commit below lands.
	head := commitBlobs(t, repo, 60)

	blocked := b.hold()
	done := make(chan error, 1)
	go func() { done <- s.flush() }()
	<-blocked

	// The walk is over by the time the first upload blocks, so this commit is
	// exactly what a push arriving mid-flush leaves behind.
	late := commitOne(t, repo)
	b.release()
	if err := <-done; err != nil {
		t.Fatalf("flush: %v", err)
	}

	refs := b.object("repos/x.git/REFS")
	if strings.Contains(refs, late) {
		t.Fatalf("the flush published a commit it did not walk, refs hold %q", refs)
	}
	if !strings.Contains(refs, head) {
		t.Fatalf("the flush published no refs at all, the bucket holds %q", refs)
	}

	// Whether the published state is servable, which only a fresh container
	// answers.
	fresh := t.TempDir()
	f := newSync(newR2(url), fresh)
	if err := f.restore(); err != nil {
		t.Fatal(err)
	}
	if err := f.hydrate("x"); err != nil {
		t.Fatalf("hydrating what the flush published: %v", err)
	}
	out, err := git(filepath.Join(fresh, "repos", "x.git"), "", nil, "rev-list", "--objects", head)
	if err != nil {
		t.Fatalf("the published repository is missing objects its refs name: %v: %s", err, out)
	}
}

// The other half of a detached gc: it removes the fanout directories it
// emptied, and a walk inside one when it goes stops there. Treating that as a
// finished walk publishes refs naming objects the bucket never received.
//
// The window is small, so this takes several rounds. A flush that reports the
// walk was short is a correct answer; success over a bucket that cannot serve
// what it published is not.
func TestFlushWhileGitPrunes(t *testing.T) {
	gitIdentity(t)

	published := 0
	for round := range 8 {
		_, url := newBucket(t)
		dir := t.TempDir()
		s := newSync(newR2(url), dir)
		if err := s.restore(); err != nil {
			t.Fatal(err)
		}
		repo := filepath.Join(dir, "repos", "x.git")
		if err := gitInit(repo, true); err != nil {
			t.Fatal(err)
		}
		head := commitBlobs(t, repo, 600)

		// Where gc is the moment before it prunes: everything reachable is in a
		// pack, and the loose copies it replaced are still on disk.
		if out, err := git(repo, "", nil, "repack", "-a", "-q"); err != nil {
			t.Fatalf("repack: %v: %s", err, out)
		}

		// prune-packed alongside the flush: the listing the flush does first is
		// long enough for this to be well under way.
		done := make(chan struct{})
		go func() {
			defer close(done)
			root := filepath.Join(repo, "objects")
			fanout, err := os.ReadDir(root)
			if err != nil {
				return
			}
			for _, d := range fanout {
				if d.IsDir() && len(d.Name()) == 2 {
					_ = os.RemoveAll(filepath.Join(root, d.Name()))
					time.Sleep(20 * time.Microsecond)
				}
			}
		}()
		err := s.flush()
		<-done
		if err != nil {
			continue
		}
		published++

		fresh := t.TempDir()
		f := newSync(newR2(url), fresh)
		if err := f.restore(); err != nil {
			t.Fatal(err)
		}
		if err := f.hydrate("x"); err != nil {
			t.Fatalf("round %d: the flush published refs the bucket cannot serve: %v", round, err)
		}
		out, err := git(filepath.Join(fresh, "repos", "x.git"), "", nil, "rev-list", "--objects", head)
		if err != nil {
			t.Fatalf("round %d: the published repository is missing objects its refs name: %v: %s", round, err, out)
		}
	}
	// Every round refusing is a pass that checked nothing, and a flush that
	// always refuses is its own bug.
	if published == 0 {
		t.Error("no round got as far as publishing, so nothing above was checked")
	}
}

// Deleting a repository must not fail the flush of everything else. DeleteRepo
// removes the directory, and the section with it when it held the last one, so
// a pass can list one and find it gone by the time it reads it.
//
// The directories are renamed away rather than removed, so each stops existing
// at an instant: that is the moment the pass has to survive. A removal caught
// half done leaves something git will not read, which flush still fails on.
//
// The deletions start when the bucket sees the first upload, so they land while
// the pass is inside the repositories.
func TestFlushWhileRepositoriesAreDeleted(t *testing.T) {
	gitIdentity(t)

	for round := range 20 {
		b := &bucket{objs: map[string][]byte{}, blocked: make(chan struct{})}
		dir := t.TempDir()
		away := t.TempDir()

		var doomed []string
		var once sync.Once
		upload, deleted := make(chan struct{}), make(chan struct{})
		release := func() { once.Do(func() { close(upload) }) }
		go func() {
			defer close(deleted)
			<-upload
			for i, p := range doomed {
				_ = os.Rename(p, filepath.Join(away, strconv.Itoa(i)))
				// The section goes with the last repository in it.
				_ = os.Remove(filepath.Dir(p))
				time.Sleep(time.Millisecond)
			}
		}()
		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/repos/") {
				release()
			}
			b.ServeHTTP(w, r)
		}))

		s := newSync(newR2(srv.URL), dir)
		if err := s.restore(); err != nil {
			t.Fatal(err)
		}
		keep := filepath.Join(dir, "repos", "keep.git")
		if err := gitInit(keep, true); err != nil {
			t.Fatal(err)
		}
		head := commitOne(t, keep)
		// Sorted after keep.git, so the pass is still working when they go.
		for i := range 12 {
			p := filepath.Join(dir, "repos", fmt.Sprintf("team%d", i), "api.git")
			if err := gitInit(p, true); err != nil {
				t.Fatal(err)
			}
			commitOne(t, p)
			doomed = append(doomed, p)
		}

		err := s.flush()
		// The deleter has to be done with the round's directories before
		// t.TempDir takes them away underneath it.
		srv.Close()
		release()
		<-deleted
		if err != nil {
			t.Fatalf("round %d: a deletion elsewhere failed the flush: %v", round, err)
		}
		if refs := b.object("repos/keep.git/REFS"); !strings.Contains(refs, head) {
			t.Fatalf("round %d: the repository that stayed is missing its refs, the bucket holds %q", round, refs)
		}
	}
}

// A repository being hydrated must be invisible to a flush: an objects
// directory that git init has just made holds no refs yet, and a flush that saw
// one would write that emptiness over the real ref state.
func TestFlushDuringHydration(t *testing.T) {
	gitIdentity(t)
	b, url := newBucket(t)
	head := seed(t, b, url)

	dir := t.TempDir()
	s := newSync(newR2(url), dir)
	if err := s.restore(); err != nil {
		t.Fatal(err)
	}

	blocked := b.hold()
	done := make(chan error, 1)
	go func() { done <- s.hydrate("x") }()
	<-blocked

	if err := s.flush(); err != nil {
		t.Fatal(err)
	}
	b.release()
	if err := <-done; err != nil {
		t.Fatal(err)
	}

	if refs := b.object("repos/x.git/REFS"); !strings.Contains(refs, head) {
		t.Errorf("the flush overwrote the stored refs, the bucket holds %q", refs)
	}
	out, _ := git(filepath.Join(dir, "repos", "x.git"), "", nil, "show-ref")
	if !strings.Contains(string(out), head) {
		t.Errorf("the hydrated repository has no refs, show-ref says %q", out)
	}
}

// A move into the placeholder that failed part way must not wedge every attempt
// after it: rename refuses an existing directory, so the debris of one failure
// would otherwise be permanent for the life of the container.
func TestHydrationAfterPartialMove(t *testing.T) {
	gitIdentity(t)
	b, url := newBucket(t)
	head := seed(t, b, url)

	dir := t.TempDir()
	s := newSync(newR2(url), dir)
	if err := s.restore(); err != nil {
		t.Fatal(err)
	}
	// What a rename that died in the middle of the loop leaves: part of a
	// repository, no objects directory, so still a placeholder.
	for _, d := range []string{"refs/heads", "info"} {
		if err := os.MkdirAll(filepath.Join(dir, "repos", "x.git", d), 0700); err != nil {
			t.Fatal(err)
		}
	}
	headfile := filepath.Join(dir, "repos", "x.git", "HEAD")
	if err := os.WriteFile(headfile, []byte("ref: refs/heads/master\n"), 0600); err != nil {
		t.Fatal(err)
	}

	if err := s.hydrate("x"); err != nil {
		t.Fatalf("hydrating over the debris: %v", err)
	}
	out, _ := git(filepath.Join(dir, "repos", "x.git"), "", nil, "show-ref")
	if !strings.Contains(string(out), head) {
		t.Errorf("the repository was not hydrated, show-ref says %q", out)
	}
}

// A manifest write that failed must be carried up by the next flush: recording
// it before the put would make one refused write permanent.
func TestManifestRetriedAfterFailure(t *testing.T) {
	gitIdentity(t)
	b, url := newBucket(t)

	dir := t.TempDir()
	s := newSync(newR2(url), dir)
	if err := s.restore(); err != nil {
		t.Fatal(err)
	}
	if err := gitInit(filepath.Join(dir, "repos", "x.git"), true); err != nil {
		t.Fatal(err)
	}

	b.fail(func(method, key string) bool {
		return method == http.MethodPut && key == manifestKey
	})
	if err := s.flush(); err == nil {
		t.Fatal("a flush whose manifest the bucket refused reported success")
	}
	b.fail(nil)

	if err := s.flush(); err != nil {
		t.Fatal(err)
	}
	if got := b.object(manifestKey); !strings.Contains(got, "x.git") {
		t.Errorf("the repository is missing from the manifest, which holds %q", got)
	}
}

// A forge with no repositories writes no manifest, because the directory set
// never moved. A restore has to tell that apart from an empty bucket, or the
// next container comes up with no accounts and flushes that over the real ones.
func TestConfigRestoredWithoutManifest(t *testing.T) {
	gitIdentity(t)
	b, url := newBucket(t)

	dir := t.TempDir()
	s := newSync(newR2(url), dir)
	if err := s.restore(); err != nil {
		t.Fatal(err)
	}
	if err := gitInit(filepath.Join(dir, "config"), false); err != nil {
		t.Fatal(err)
	}
	users := `{"alice":{}}`
	if err := os.WriteFile(filepath.Join(dir, "config", "users.json"), []byte(users), 0600); err != nil {
		t.Fatal(err)
	}
	if err := s.flush(); err != nil {
		t.Fatal(err)
	}
	if got := b.object(manifestKey); got != "" {
		t.Fatalf("a forge with no repositories wrote a manifest holding %q", got)
	}

	fresh := t.TempDir()
	f := newSync(newR2(url), fresh)
	if err := f.restore(); err != nil {
		t.Fatal(err)
	}
	got, err := os.ReadFile(filepath.Join(fresh, "config", "users.json"))
	if err != nil {
		t.Fatalf("the account table did not come back: %v", err)
	}
	if string(got) != users {
		t.Fatalf("the account table came back as %q", got)
	}
	if err := f.flush(); err != nil {
		t.Fatal(err)
	}
	if got := b.object("config/users.json"); got != users {
		t.Fatalf("the flush after the restore left %q in the bucket", got)
	}
}

// An absent manifest means no repositories, which is true of a forge that has
// none and a lie about a bucket that has lost one. Believing the lie hands the
// sweep every repository in the bucket as one nothing lists.
func TestRestoreRefusesAMissingManifest(t *testing.T) {
	gitIdentity(t)
	b, url := newBucket(t)
	seed(t, b, url)

	b.mu.Lock()
	delete(b.objs, manifestKey)
	b.mu.Unlock()

	f := newSync(newR2(url), t.TempDir())
	if err := f.restore(); err == nil {
		t.Fatal("a restore believed a bucket that holds repositories and no manifest")
	}
	if keys := b.under("repos/x.git/"); len(keys) == 0 {
		t.Error("the repository was pruned anyway")
	}

	// And the case it has to stay true for: a forge that never had one.
	empty, emptyURL := newBucket(t)
	empty.mu.Lock()
	empty.objs["config/users.json"] = []byte(`{"alice":{}}`)
	empty.mu.Unlock()
	if err := newSync(newR2(emptyURL), t.TempDir()).restore(); err != nil {
		t.Errorf("a forge with no repositories was refused: %v", err)
	}
}

// What a restore knows the bucket holds has to come from the keys it fetched.
// Taking it from the directory instead would mean a file the bucket does not
// hold is remembered as stored, and the flush after it leaves it on a disk that
// does not survive.
func TestConfigOnDiskOnlyIsStillMirrored(t *testing.T) {
	gitIdentity(t)
	b, url := newBucket(t)

	seedDir := t.TempDir()
	s := newSync(newR2(url), seedDir)
	if err := s.restore(); err != nil {
		t.Fatal(err)
	}
	if err := gitInit(filepath.Join(seedDir, "config"), false); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(seedDir, "config", "users.json"), []byte(`{"alice":{}}`), 0600); err != nil {
		t.Fatal(err)
	}
	if err := s.flush(); err != nil {
		t.Fatal(err)
	}

	// A datadir that outlived the syncd that wrote it, holding a file whose
	// flush never succeeded.
	dirty := t.TempDir()
	if err := os.MkdirAll(filepath.Join(dirty, "config"), 0700); err != nil {
		t.Fatal(err)
	}
	repos := `{"notes":{"visibility":"public"}}`
	if err := os.WriteFile(filepath.Join(dirty, "config", "repos.json"), []byte(repos), 0600); err != nil {
		t.Fatal(err)
	}

	f := newSync(newR2(url), dirty)
	if err := f.restore(); err != nil {
		t.Fatal(err)
	}
	if err := f.flush(); err != nil {
		t.Fatal(err)
	}
	if got := b.object("config/repos.json"); got != repos {
		t.Errorf("the bucket holds %q for a file only disk had", got)
	}
}

// git-forge writes a config file by creating {name}.tmp{n} beside it and
// renaming over the top, so a pass can catch one between the two. Mirroring it
// stores a half written account table under a key nothing removes.
func TestConfigTempFilesAreNotMirrored(t *testing.T) {
	b, url := newBucket(t)

	dir := t.TempDir()
	s := newSync(newR2(url), dir)
	if err := s.restore(); err != nil {
		t.Fatal(err)
	}
	users := `{"alice":{}}`
	if err := os.WriteFile(filepath.Join(dir, "config", "users.json"), []byte(users), 0600); err != nil {
		t.Fatal(err)
	}
	// What git-forge's writeFile leaves beside it for the length of an fsync.
	temp := "users.json.tmp3877658021"
	if err := os.WriteFile(filepath.Join(dir, "config", temp), []byte(`{"alice":{"par`), 0600); err != nil {
		t.Fatal(err)
	}
	if err := s.flush(); err != nil {
		t.Fatal(err)
	}
	if got := b.object("config/" + temp); got != "" {
		t.Errorf("the bucket holds a file git-forge was still writing: %q", got)
	}
	if got := b.object("config/users.json"); got != users {
		t.Errorf("the account table is not in the bucket, which holds %q", got)
	}

	// And one already there is left where it is rather than restored, because
	// putting it back on disk is what makes it permanent.
	b.mu.Lock()
	b.objs["config/"+temp] = []byte(`{"alice":{"par`)
	b.mu.Unlock()

	fresh := t.TempDir()
	f := newSync(newR2(url), fresh)
	if err := f.restore(); err != nil {
		t.Fatal(err)
	}
	if _, err := os.Stat(filepath.Join(fresh, "config", temp)); err == nil {
		t.Error("a restore wrote a half written config file back to disk")
	}
	if got, err := os.ReadFile(filepath.Join(fresh, "config", "users.json")); err != nil || string(got) != users {
		t.Errorf("the account table did not come back: %q: %v", got, err)
	}
}

// Only syncd writes this bucket, so a key of a shape it never writes means the
// bucket has been tampered with.
func TestRestoreRefusesAKeyItDidNotWrite(t *testing.T) {
	for _, key := range []string{
		"config/.git/config",
		"config/.git/hooks/pre-commit",
		"config/git/hooks/pre-commit",
		"config/sub/dir/users.json",
	} {
		b, url := newBucket(t)
		b.mu.Lock()
		b.objs["config/users.json"] = []byte(`{}`)
		b.objs[key] = []byte("#!/bin/sh\nid\n")
		b.mu.Unlock()

		dir := t.TempDir()
		s := newSync(newR2(url), dir)
		if err := s.restore(); err == nil {
			t.Errorf("a restore believed %s", key)
		}
	}
}

// git-forge serves without an audit log when git will not initialise. The
// working tree still has to be mirrored: it is what a restore reads.
func TestConfigFlushedWithoutAuditLog(t *testing.T) {
	b, url := newBucket(t)

	dir := t.TempDir()
	s := newSync(newR2(url), dir)
	if err := s.restore(); err != nil {
		t.Fatal(err)
	}
	users := `{"alice":{}}`
	if err := os.WriteFile(filepath.Join(dir, "config", "users.json"), []byte(users), 0600); err != nil {
		t.Fatal(err)
	}
	if err := s.flush(); err != nil {
		t.Fatal(err)
	}
	if got := b.object("config/users.json"); got != users {
		t.Errorf("the account table is not in the bucket, which holds %q", got)
	}
}

// A hydration that fails part way must leave the placeholder alone, so the next
// request starts over. An objects directory left behind counts as hydrated.
func TestHydrationRetriedAfterFailure(t *testing.T) {
	gitIdentity(t)
	b, url := newBucket(t)
	head := seed(t, b, url)

	dir := t.TempDir()
	s := newSync(newR2(url), dir)
	if err := s.restore(); err != nil {
		t.Fatal(err)
	}

	b.fail(func(method, key string) bool {
		return method == http.MethodGet && strings.Contains(key, "/objects/")
	})
	if err := s.hydrate("x"); err == nil {
		t.Fatal("a hydration that could not read the objects reported success")
	}
	b.fail(nil)

	if err := s.hydrate("x"); err != nil {
		t.Fatalf("second attempt: %v", err)
	}
	out, _ := git(filepath.Join(dir, "repos", "x.git"), "", nil, "show-ref")
	if !strings.Contains(string(out), head) {
		t.Errorf("the second request did not hydrate it, show-ref says %q", out)
	}
	if err := s.flush(); err != nil {
		t.Fatal(err)
	}
	if refs := b.object("repos/x.git/REFS"); !strings.Contains(refs, head) {
		t.Errorf("the flush that followed overwrote the stored refs with %q", refs)
	}
}