From 3284b1c4906b90ca2e3864937c66d0fa30772ebf Mon Sep 17 00:00:00 2001 From: Lena Date: Sat, 1 Aug 2026 00:00:00 +0000 Subject: Enter git-forge-cloudflare Runs git-forge on Cloudflare Containers with R2 for durable storage. A Worker routes every request to one container through a Durable Object. Inside it, syncd mirrors the data directory to the bucket and restores it at boot, holding a write until it is durable. git-forge is neither modified nor vendored: the image builds it from its own repository at the commit named in forge.rev. --- sync_test.go | 1113 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1113 insertions(+) create mode 100644 sync_test.go (limited to 'sync_test.go') diff --git a/sync_test.go b/sync_test.go new file mode 100644 index 0000000..2158626 --- /dev/null +++ b/sync_test.go @@ -0,0 +1,1113 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +// TestMirrorable pins which files under objects/ belong in the bucket. The +// interesting cases are all state git is in the middle of writing. +func TestMirrorable(t *testing.T) { + root := t.TempDir() + + // mirrorable stats the index beside a pack, so these must be real files. + write := func(rel string) { + path := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("x"), 0600); err != nil { + t.Fatal(err) + } + } + write("ab/cdef0123456789") + write("pack/pack-done.pack") + write("pack/pack-done.idx") + write("pack/pack-half.pack") // no index yet + write("pack/pack-held.pack") + write("pack/pack-held.idx") + write("pack/pack-held.keep") // receive-pack still has it open + write("pack/tmp_pack_writing") + write("info/commit-graph") + // receive-pack's quarantine: a temporary directory holding files whose own + // names say nothing about being temporary. + write("tmp_objdir-incoming-HBbblc/pack/pack-incoming.idx") + write("tmp_objdir-incoming-HBbblc/pack/pack-incoming.pack") + write("tmp_objdir-incoming-HBbblc/pack/pack-incoming.keep") + + cases := []struct { + rel string + want bool + }{ + {"ab/cdef0123456789", true}, + {"pack/pack-done.pack", true}, + {"pack/pack-done.idx", true}, + {"pack/pack-half.pack", false}, + {"pack/pack-held.pack", true}, + {"pack/pack-held.keep", false}, + {"pack/tmp_pack_writing", false}, + {"info/commit-graph", false}, + {"tmp_objdir-incoming-HBbblc/pack/pack-incoming.idx", false}, + {"tmp_objdir-incoming-HBbblc/pack/pack-incoming.pack", false}, + {"tmp_objdir-incoming-HBbblc/pack/pack-incoming.keep", false}, + } + for _, c := range cases { + if got := mirrorable(filepath.FromSlash(c.rel), root); got != c.want { + t.Errorf("mirrorable(%q) = %v, want %v", c.rel, got, c.want) + } + } +} + +// TestScanRepos pins what the manifest holds, and so what a restore brings +// back: repositories at the root, one section deep, and the issue and wiki +// siblings. +func TestScanRepos(t *testing.T) { + root := t.TempDir() + for _, d := range []string{ + "notes.git", "notes.issues.git", "notes.wiki.git", + "team/api.git", "team/deeper/nested.git", "empty", + } { + if err := os.MkdirAll(filepath.Join(root, filepath.FromSlash(d)), 0700); err != nil { + t.Fatal(err) + } + } + // Named like a repository and not a directory, so not one. + if err := os.WriteFile(filepath.Join(root, "afile.git"), []byte("x"), 0600); err != nil { + t.Fatal(err) + } + + got, err := scanRepos(root) + if err != nil { + t.Fatal(err) + } + // team/deeper/nested.git is absent on purpose: a section is one level deep, + // which is the rule git-forge scans by. + want := "notes.git notes.issues.git notes.wiki.git team/api.git" + if strings.Join(got, " ") != want { + t.Errorf("scanRepos = %q, want %q", strings.Join(got, " "), want) + } +} + +// TestListFollowsTheCursor pins that a caller gets every key rather than the +// first page. A repository with more than a thousand loose objects is ordinary, +// and a listing that stops short is the one input that turns a hydration into a +// deletion: what it did not fetch reads afterwards as an object gc replaced. +func TestListFollowsTheCursor(t *testing.T) { + b, url := newBucket(t) + const prefix = "repos/x.git/objects/" + b.mu.Lock() + for i := range bucketPage*3 + 1 { + b.objs[fmt.Sprintf("%s%02d/object", prefix, i)] = []byte("x") + } + b.mu.Unlock() + + keys, err := newR2(url).list(prefix) + if err != nil { + t.Fatal(err) + } + if len(keys) != bucketPage*3+1 { + t.Errorf("a listing of %d keys came back with %d", bucketPage*3+1, len(keys)) + } +} + +// TestCheckKeys pins the one thing standing between a tampered bucket and a +// file written outside the datadir. +func TestCheckKeys(t *testing.T) { + const prefix = "repos/x.git/objects/" + cases := []struct { + key string + want bool + }{ + {prefix + "ab/cdef0123456789", true}, + {prefix + "pack/pack-1.pack", true}, + {prefix + "../../../../etc/passwd", false}, + {prefix + "/etc/passwd", false}, + {prefix, false}, + } + for _, c := range cases { + if got := checkKeys([]string{c.key}, prefix) == nil; got != c.want { + t.Errorf("checkKeys(%q) accepted = %v, want %v", c.key, got, c.want) + } + } +} + +// How many keys a listing answers at once. See the list case below. +const bucketPage = 25 + +// bucket is the object handler from worker.js, in memory. Object gets can be +// held open and any request made to fail, which turns races into tests. +type bucket struct { + mu sync.Mutex + objs map[string][]byte + failing func(method, key string) bool + gate chan struct{} // object gets wait on this while it is not nil + blocked chan struct{} // closed once one has + once sync.Once +} + +func newBucket(t *testing.T) (*bucket, string) { + t.Helper() + b := &bucket{objs: map[string][]byte{}, blocked: make(chan struct{})} + srv := httptest.NewServer(b) + t.Cleanup(srv.Close) + return b, srv.URL +} + +func (b *bucket) ServeHTTP(w http.ResponseWriter, r *http.Request) { + key := strings.TrimPrefix(r.URL.Path, "/") + + b.mu.Lock() + failing, gate := b.failing, b.gate + b.mu.Unlock() + if failing != nil && failing(r.Method, key) { + http.Error(w, "the bucket is having a moment", http.StatusInternalServerError) + return + } + if gate != nil && strings.Contains(key, "/objects/") { + b.once.Do(func() { close(b.blocked) }) + <-gate + } + + switch { + case r.Method == http.MethodGet && key == "": + prefix := r.URL.Query().Get("prefix") + keys := []string{} + b.mu.Lock() + for k := range b.objs { + if strings.HasPrefix(k, prefix) { + keys = append(keys, k) + } + } + b.mu.Unlock() + // Paged, because r2 pages at a thousand and a listing that stops short + // hydrates part of a repository, which the pass after it reads as + // objects gc replaced and deletes. Small, so every test that lists + // anything with a repository in it follows a cursor. + slices.Sort(keys) + from := 0 + if c := r.URL.Query().Get("cursor"); c != "" { + from, _ = strconv.Atoi(c) + } + to := min(from+bucketPage, len(keys)) + next := "" + if to < len(keys) { + next = strconv.Itoa(to) + } + _ = json.NewEncoder(w).Encode(map[string]any{"keys": keys[from:to], "cursor": next}) + case r.Method == http.MethodGet: + b.mu.Lock() + data, found := b.objs[key] + b.mu.Unlock() + if !found { + http.Error(w, "not found", http.StatusNotFound) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + _, _ = w.Write(data) + case r.Method == http.MethodPut: + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + b.mu.Lock() + b.objs[key] = data + b.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodDelete: + b.mu.Lock() + delete(b.objs, key) + b.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + default: + http.Error(w, "bad method", http.StatusMethodNotAllowed) + } +} + +// hold blocks every object get until release, and reports when the first one is +// waiting, which is a repository part way through being hydrated. +func (b *bucket) hold() <-chan struct{} { + b.mu.Lock() + defer b.mu.Unlock() + b.gate = make(chan struct{}) + return b.blocked +} + +func (b *bucket) release() { + b.mu.Lock() + gate := b.gate + b.gate = nil + b.mu.Unlock() + close(gate) +} + +func (b *bucket) fail(when func(method, key string) bool) { + b.mu.Lock() + b.failing = when + b.mu.Unlock() +} + +func (b *bucket) object(key string) string { + b.mu.Lock() + defer b.mu.Unlock() + return string(b.objs[key]) +} + +func (b *bucket) under(prefix string) []string { + b.mu.Lock() + defer b.mu.Unlock() + var keys []string + for k := range b.objs { + if strings.HasPrefix(k, prefix) { + keys = append(keys, k) + } + } + return keys +} + +// gitIdentity keeps these tests independent of whoever runs them: git refuses +// to write a commit without one. +func gitIdentity(t *testing.T) { + t.Helper() + for _, v := range []string{"GIT_AUTHOR_NAME", "GIT_COMMITTER_NAME"} { + t.Setenv(v, "syncd test") + } + for _, v := range []string{"GIT_AUTHOR_EMAIL", "GIT_COMMITTER_EMAIL"} { + t.Setenv(v, "test@example.com") + } +} + +// seed leaves the bucket holding what a forge that had taken one push would +// have, through the same flush, and returns the commit master points at. +func seed(t *testing.T, b *bucket, url string) string { + t.Helper() + dir := t.TempDir() + s := newSync(newR2(url), dir) + if err := s.restore(); err != nil { + t.Fatal(err) + } + repo := filepath.Join(dir, "repos", "x.git") + if err := gitInit(repo, true); err != nil { + t.Fatal(err) + } + head := commitOne(t, repo) + if err := s.flush(); err != nil { + t.Fatal(err) + } + if refs := b.object("repos/x.git/REFS"); !strings.Contains(refs, head) { + t.Fatalf("the seed did not store the refs, the bucket holds %q", refs) + } + return head +} + +// commitOne puts one empty commit on master and returns it. +func commitOne(t *testing.T, repo string) string { + t.Helper() + tree, err := git(repo, "", []byte{}, "mktree") + if err != nil { + t.Fatalf("mktree: %v: %s", err, tree) + } + commit, err := git(repo, "", nil, "commit-tree", strings.TrimSpace(string(tree)), "-m", "first") + if err != nil { + t.Fatalf("commit-tree: %v: %s", err, commit) + } + head := strings.TrimSpace(string(commit)) + if out, err := git(repo, "", nil, "update-ref", "refs/heads/master", head); err != nil { + t.Fatalf("update-ref: %v: %s", err, out) + } + return head +} + +// commitBlobs puts one commit holding n blobs on master. Enough of them land in +// most of the fanout directories, which a gc racing a walk needs. +func commitBlobs(t *testing.T, repo string, n int) string { + t.Helper() + var paths, entries strings.Builder + dir := t.TempDir() + for i := range n { + name := filepath.Join(dir, fmt.Sprintf("f%d", i)) + if err := os.WriteFile(name, fmt.Appendf(nil, "content %d\n", i), 0600); err != nil { + t.Fatal(err) + } + paths.WriteString(name + "\n") + } + blobs, err := git(repo, "", []byte(paths.String()), "hash-object", "-w", "--stdin-paths") + if err != nil { + t.Fatalf("hash-object: %v: %s", err, blobs) + } + for i, sha := range strings.Fields(string(blobs)) { + fmt.Fprintf(&entries, "100644 blob %s\tf%d\n", sha, i) + } + tree, err := git(repo, "", []byte(entries.String()), "mktree") + if err != nil { + t.Fatalf("mktree: %v: %s", err, tree) + } + commit, err := git(repo, "", nil, "commit-tree", strings.TrimSpace(string(tree)), "-m", "many") + if err != nil { + t.Fatalf("commit-tree: %v: %s", err, commit) + } + head := strings.TrimSpace(string(commit)) + if out, err := git(repo, "", nil, "update-ref", "refs/heads/master", head); err != nil { + t.Fatalf("update-ref: %v: %s", err, out) + } + return head +} + +// Pruning waits a pass: deleting on the pass that first notices an object gone +// can take one whose replacement is still on disk alone. +func TestPruneWaitsForAPass(t *testing.T) { + gitIdentity(t) + b, url := newBucket(t) + + dir := t.TempDir() + s := newSync(newR2(url), dir) + if err := s.restore(); err != nil { + t.Fatal(err) + } + repo := filepath.Join(dir, "repos", "x.git") + if err := gitInit(repo, true); err != nil { + t.Fatal(err) + } + head := commitOne(t, repo) + if err := s.flush(); err != nil { + t.Fatal(err) + } + out, err := git(repo, "", nil, "rev-parse", head+"^{tree}") + if err != nil { + t.Fatalf("rev-parse: %v: %s", err, out) + } + tree := strings.TrimSpace(string(out)) + key := "repos/x.git/objects/" + tree[:2] + "/" + tree[2:] + if b.object(key) == "" { + t.Fatalf("the flush did not store the tree at %s", key) + } + + // What gc does to a loose object the moment it has packed it. + if err := os.Remove(filepath.Join(repo, "objects", tree[:2], tree[2:])); err != nil { + t.Fatal(err) + } + if err := s.flush(); err != nil { + t.Fatal(err) + } + if b.object(key) == "" { + t.Fatal("the pass that first found the object gone deleted it") + } + if err := s.flush(); err != nil { + t.Fatal(err) + } + if b.object(key) != "" { + t.Error("the pass after that did not delete it") + } +} + +// A repository the manifest no longer lists leaves the bucket after the grace, +// not at the first boot that notices: a manifest restored from a backup, or +// written short, would otherwise take every repository it omits with it. +func TestPruneWaitsOutTheGrace(t *testing.T) { + gitIdentity(t) + b, url := newBucket(t) + + dir := t.TempDir() + s := newSync(newR2(url), dir) + if err := s.restore(); err != nil { + t.Fatal(err) + } + heads := map[string]string{} + for _, name := range []string{"keep.git", "drop.git"} { + repo := filepath.Join(dir, "repos", name) + if err := gitInit(repo, true); err != nil { + t.Fatal(err) + } + heads[name] = commitOne(t, repo) + } + if err := s.flush(); err != nil { + t.Fatal(err) + } + + // What DeleteRepo leaves behind: the directory gone and the manifest + // rewritten without it, while everything it named is still in the bucket. + if err := os.RemoveAll(filepath.Join(dir, "repos", "drop.git")); err != nil { + t.Fatal(err) + } + if err := s.flush(); err != nil { + t.Fatal(err) + } + if n := len(b.under("repos/drop.git/")); n == 0 { + t.Fatal("the flush that rewrote the manifest already removed the objects") + } + + fresh := t.TempDir() + f := newSync(newR2(url), fresh) + if err := f.restore(); err != nil { + t.Fatal(err) + } + if err := f.pruneDeleted(); err != nil { + t.Fatal(err) + } + if len(b.under("repos/drop.git/")) == 0 { + t.Fatal("the first boot to find it unlisted removed it") + } + if !strings.Contains(b.object(staleKey), "drop.git") { + t.Fatalf("the bucket does not record when it went unlisted: %q", b.object(staleKey)) + } + + // Listed again, by a manifest put back: the record goes and nothing is + // removed. + f.setManifest([]string{"keep.git", "drop.git"}) + if err := f.pruneDeleted(); err != nil { + t.Fatal(err) + } + if got := b.object(staleKey); got != "" { + t.Errorf("a repository listed again is still recorded as stale: %q", got) + } + if len(b.under("repos/drop.git/")) == 0 { + t.Fatal("a repository listed again was removed") + } + + // Unlisted for longer than the grace. + f.setManifest([]string{"keep.git"}) + b.mu.Lock() + b.objs[staleKey] = []byte(`{"drop.git":"2020-01-01T00:00:00Z"}`) + b.mu.Unlock() + if err := f.pruneDeleted(); err != nil { + t.Fatal(err) + } + if keys := b.under("repos/drop.git/"); len(keys) > 0 { + t.Errorf("the bucket still holds %d keys for a repository unlisted past the grace: %v", len(keys), keys) + } + if got := b.object(staleKey); got != "" { + t.Errorf("the record outlived the deletion: %q", got) + } + + // The question worth asking of anything that deletes: whether it took + // something still reachable. + if keys := b.under("repos/keep.git/"); len(keys) == 0 { + t.Fatal("a repository the manifest lists was pruned too") + } + if err := f.hydrate("keep"); err != nil { + t.Fatalf("hydrating what the prune left: %v", err) + } + out, _ := git(filepath.Join(fresh, "repos", "keep.git"), "", nil, "show-ref") + if !strings.Contains(string(out), heads["keep.git"]) { + t.Errorf("the repository that survived is missing its refs, show-ref says %q", out) + } +} + +// gc packs loose objects and deletes them, detached, while the flush a push +// triggered is still uploading. Skipping what it took would write refs naming +// objects the bucket never received, so the flush has to walk again. +func TestFlushWhileGitRepacks(t *testing.T) { + gitIdentity(t) + b, url := newBucket(t) + + dir := t.TempDir() + s := newSync(newR2(url), dir) + if err := s.restore(); err != nil { + t.Fatal(err) + } + repo := filepath.Join(dir, "repos", "x.git") + if err := gitInit(repo, true); err != nil { + t.Fatal(err) + } + + // Enough reachable objects that the uploads outlast the walk: sixteen are in + // flight at a time, so the rest are still queued when repack runs. + commitBlobs(t, repo, 60) + + blocked := b.hold() + done := make(chan error, 1) + go func() { done <- s.flush() }() + <-blocked + if out, err := git(repo, "", nil, "repack", "-a", "-d", "-q"); err != nil { + t.Fatalf("repack: %v: %s", err, out) + } + b.release() + if err := <-done; err != nil { + t.Fatalf("flush: %v", err) + } + + // Not what the bucket holds but whether it is a repository, which only a + // fresh container answers. + fresh := t.TempDir() + f := newSync(newR2(url), fresh) + if err := f.restore(); err != nil { + t.Fatal(err) + } + if err := f.hydrate("x"); err != nil { + t.Fatalf("hydrating what the flush published: %v", err) + } + out, err := git(filepath.Join(fresh, "repos", "x.git"), "", nil, "rev-list", "--objects", "--all") + if err != nil { + t.Fatalf("the published repository is missing objects its refs name: %v: %s", err, out) + } + if n := len(strings.Split(strings.TrimSpace(string(out)), "\n")); n < 61 { + t.Errorf("only %d objects came back, want the commit, its tree and sixty blobs", n) + } +} + +// flushRepo reads the ref state before it walks the objects. A push that lands +// during the upload is left to the next flush rather than published by this +// one: refs read afterwards would name a commit the walk never saw. +func TestFlushWhileAPushLands(t *testing.T) { + gitIdentity(t) + b, url := newBucket(t) + + dir := t.TempDir() + s := newSync(newR2(url), dir) + if err := s.restore(); err != nil { + t.Fatal(err) + } + repo := filepath.Join(dir, "repos", "x.git") + if err := gitInit(repo, true); err != nil { + t.Fatal(err) + } + // Enough that the uploads are still running when the commit below lands. + head := commitBlobs(t, repo, 60) + + blocked := b.hold() + done := make(chan error, 1) + go func() { done <- s.flush() }() + <-blocked + + // The walk is over by the time the first upload blocks, so this commit is + // exactly what a push arriving mid-flush leaves behind. + late := commitOne(t, repo) + b.release() + if err := <-done; err != nil { + t.Fatalf("flush: %v", err) + } + + refs := b.object("repos/x.git/REFS") + if strings.Contains(refs, late) { + t.Fatalf("the flush published a commit it did not walk, refs hold %q", refs) + } + if !strings.Contains(refs, head) { + t.Fatalf("the flush published no refs at all, the bucket holds %q", refs) + } + + // Whether the published state is servable, which only a fresh container + // answers. + fresh := t.TempDir() + f := newSync(newR2(url), fresh) + if err := f.restore(); err != nil { + t.Fatal(err) + } + if err := f.hydrate("x"); err != nil { + t.Fatalf("hydrating what the flush published: %v", err) + } + out, err := git(filepath.Join(fresh, "repos", "x.git"), "", nil, "rev-list", "--objects", head) + if err != nil { + t.Fatalf("the published repository is missing objects its refs name: %v: %s", err, out) + } +} + +// The other half of a detached gc: it removes the fanout directories it +// emptied, and a walk inside one when it goes stops there. Treating that as a +// finished walk publishes refs naming objects the bucket never received. +// +// The window is small, so this takes several rounds. A flush that reports the +// walk was short is a correct answer; success over a bucket that cannot serve +// what it published is not. +func TestFlushWhileGitPrunes(t *testing.T) { + gitIdentity(t) + + published := 0 + for round := range 8 { + _, url := newBucket(t) + dir := t.TempDir() + s := newSync(newR2(url), dir) + if err := s.restore(); err != nil { + t.Fatal(err) + } + repo := filepath.Join(dir, "repos", "x.git") + if err := gitInit(repo, true); err != nil { + t.Fatal(err) + } + head := commitBlobs(t, repo, 600) + + // Where gc is the moment before it prunes: everything reachable is in a + // pack, and the loose copies it replaced are still on disk. + if out, err := git(repo, "", nil, "repack", "-a", "-q"); err != nil { + t.Fatalf("repack: %v: %s", err, out) + } + + // prune-packed alongside the flush: the listing the flush does first is + // long enough for this to be well under way. + done := make(chan struct{}) + go func() { + defer close(done) + root := filepath.Join(repo, "objects") + fanout, err := os.ReadDir(root) + if err != nil { + return + } + for _, d := range fanout { + if d.IsDir() && len(d.Name()) == 2 { + _ = os.RemoveAll(filepath.Join(root, d.Name())) + time.Sleep(20 * time.Microsecond) + } + } + }() + err := s.flush() + <-done + if err != nil { + continue + } + published++ + + fresh := t.TempDir() + f := newSync(newR2(url), fresh) + if err := f.restore(); err != nil { + t.Fatal(err) + } + if err := f.hydrate("x"); err != nil { + t.Fatalf("round %d: the flush published refs the bucket cannot serve: %v", round, err) + } + out, err := git(filepath.Join(fresh, "repos", "x.git"), "", nil, "rev-list", "--objects", head) + if err != nil { + t.Fatalf("round %d: the published repository is missing objects its refs name: %v: %s", round, err, out) + } + } + // Every round refusing is a pass that checked nothing, and a flush that + // always refuses is its own bug. + if published == 0 { + t.Error("no round got as far as publishing, so nothing above was checked") + } +} + +// Deleting a repository must not fail the flush of everything else. DeleteRepo +// removes the directory, and the section with it when it held the last one, so +// a pass can list one and find it gone by the time it reads it. +// +// The directories are renamed away rather than removed, so each stops existing +// at an instant: that is the moment the pass has to survive. A removal caught +// half done leaves something git will not read, which flush still fails on. +// +// The deletions start when the bucket sees the first upload, so they land while +// the pass is inside the repositories. +func TestFlushWhileRepositoriesAreDeleted(t *testing.T) { + gitIdentity(t) + + for round := range 20 { + b := &bucket{objs: map[string][]byte{}, blocked: make(chan struct{})} + dir := t.TempDir() + away := t.TempDir() + + var doomed []string + var once sync.Once + upload, deleted := make(chan struct{}), make(chan struct{}) + release := func() { once.Do(func() { close(upload) }) } + go func() { + defer close(deleted) + <-upload + for i, p := range doomed { + _ = os.Rename(p, filepath.Join(away, strconv.Itoa(i))) + // The section goes with the last repository in it. + _ = os.Remove(filepath.Dir(p)) + time.Sleep(time.Millisecond) + } + }() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/repos/") { + release() + } + b.ServeHTTP(w, r) + })) + + s := newSync(newR2(srv.URL), dir) + if err := s.restore(); err != nil { + t.Fatal(err) + } + keep := filepath.Join(dir, "repos", "keep.git") + if err := gitInit(keep, true); err != nil { + t.Fatal(err) + } + head := commitOne(t, keep) + // Sorted after keep.git, so the pass is still working when they go. + for i := range 12 { + p := filepath.Join(dir, "repos", fmt.Sprintf("team%d", i), "api.git") + if err := gitInit(p, true); err != nil { + t.Fatal(err) + } + commitOne(t, p) + doomed = append(doomed, p) + } + + err := s.flush() + // The deleter has to be done with the round's directories before + // t.TempDir takes them away underneath it. + srv.Close() + release() + <-deleted + if err != nil { + t.Fatalf("round %d: a deletion elsewhere failed the flush: %v", round, err) + } + if refs := b.object("repos/keep.git/REFS"); !strings.Contains(refs, head) { + t.Fatalf("round %d: the repository that stayed is missing its refs, the bucket holds %q", round, refs) + } + } +} + +// A repository being hydrated must be invisible to a flush: an objects +// directory that git init has just made holds no refs yet, and a flush that saw +// one would write that emptiness over the real ref state. +func TestFlushDuringHydration(t *testing.T) { + gitIdentity(t) + b, url := newBucket(t) + head := seed(t, b, url) + + dir := t.TempDir() + s := newSync(newR2(url), dir) + if err := s.restore(); err != nil { + t.Fatal(err) + } + + blocked := b.hold() + done := make(chan error, 1) + go func() { done <- s.hydrate("x") }() + <-blocked + + if err := s.flush(); err != nil { + t.Fatal(err) + } + b.release() + if err := <-done; err != nil { + t.Fatal(err) + } + + if refs := b.object("repos/x.git/REFS"); !strings.Contains(refs, head) { + t.Errorf("the flush overwrote the stored refs, the bucket holds %q", refs) + } + out, _ := git(filepath.Join(dir, "repos", "x.git"), "", nil, "show-ref") + if !strings.Contains(string(out), head) { + t.Errorf("the hydrated repository has no refs, show-ref says %q", out) + } +} + +// A move into the placeholder that failed part way must not wedge every attempt +// after it: rename refuses an existing directory, so the debris of one failure +// would otherwise be permanent for the life of the container. +func TestHydrationAfterPartialMove(t *testing.T) { + gitIdentity(t) + b, url := newBucket(t) + head := seed(t, b, url) + + dir := t.TempDir() + s := newSync(newR2(url), dir) + if err := s.restore(); err != nil { + t.Fatal(err) + } + // What a rename that died in the middle of the loop leaves: part of a + // repository, no objects directory, so still a placeholder. + for _, d := range []string{"refs/heads", "info"} { + if err := os.MkdirAll(filepath.Join(dir, "repos", "x.git", d), 0700); err != nil { + t.Fatal(err) + } + } + headfile := filepath.Join(dir, "repos", "x.git", "HEAD") + if err := os.WriteFile(headfile, []byte("ref: refs/heads/master\n"), 0600); err != nil { + t.Fatal(err) + } + + if err := s.hydrate("x"); err != nil { + t.Fatalf("hydrating over the debris: %v", err) + } + out, _ := git(filepath.Join(dir, "repos", "x.git"), "", nil, "show-ref") + if !strings.Contains(string(out), head) { + t.Errorf("the repository was not hydrated, show-ref says %q", out) + } +} + +// A manifest write that failed must be carried up by the next flush: recording +// it before the put would make one refused write permanent. +func TestManifestRetriedAfterFailure(t *testing.T) { + gitIdentity(t) + b, url := newBucket(t) + + dir := t.TempDir() + s := newSync(newR2(url), dir) + if err := s.restore(); err != nil { + t.Fatal(err) + } + if err := gitInit(filepath.Join(dir, "repos", "x.git"), true); err != nil { + t.Fatal(err) + } + + b.fail(func(method, key string) bool { + return method == http.MethodPut && key == manifestKey + }) + if err := s.flush(); err == nil { + t.Fatal("a flush whose manifest the bucket refused reported success") + } + b.fail(nil) + + if err := s.flush(); err != nil { + t.Fatal(err) + } + if got := b.object(manifestKey); !strings.Contains(got, "x.git") { + t.Errorf("the repository is missing from the manifest, which holds %q", got) + } +} + +// A forge with no repositories writes no manifest, because the directory set +// never moved. A restore has to tell that apart from an empty bucket, or the +// next container comes up with no accounts and flushes that over the real ones. +func TestConfigRestoredWithoutManifest(t *testing.T) { + gitIdentity(t) + b, url := newBucket(t) + + dir := t.TempDir() + s := newSync(newR2(url), dir) + if err := s.restore(); err != nil { + t.Fatal(err) + } + if err := gitInit(filepath.Join(dir, "config"), false); err != nil { + t.Fatal(err) + } + users := `{"alice":{}}` + if err := os.WriteFile(filepath.Join(dir, "config", "users.json"), []byte(users), 0600); err != nil { + t.Fatal(err) + } + if err := s.flush(); err != nil { + t.Fatal(err) + } + if got := b.object(manifestKey); got != "" { + t.Fatalf("a forge with no repositories wrote a manifest holding %q", got) + } + + fresh := t.TempDir() + f := newSync(newR2(url), fresh) + if err := f.restore(); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(filepath.Join(fresh, "config", "users.json")) + if err != nil { + t.Fatalf("the account table did not come back: %v", err) + } + if string(got) != users { + t.Fatalf("the account table came back as %q", got) + } + if err := f.flush(); err != nil { + t.Fatal(err) + } + if got := b.object("config/users.json"); got != users { + t.Fatalf("the flush after the restore left %q in the bucket", got) + } +} + +// An absent manifest means no repositories, which is true of a forge that has +// none and a lie about a bucket that has lost one. Believing the lie hands the +// sweep every repository in the bucket as one nothing lists. +func TestRestoreRefusesAMissingManifest(t *testing.T) { + gitIdentity(t) + b, url := newBucket(t) + seed(t, b, url) + + b.mu.Lock() + delete(b.objs, manifestKey) + b.mu.Unlock() + + f := newSync(newR2(url), t.TempDir()) + if err := f.restore(); err == nil { + t.Fatal("a restore believed a bucket that holds repositories and no manifest") + } + if keys := b.under("repos/x.git/"); len(keys) == 0 { + t.Error("the repository was pruned anyway") + } + + // And the case it has to stay true for: a forge that never had one. + empty, emptyURL := newBucket(t) + empty.mu.Lock() + empty.objs["config/users.json"] = []byte(`{"alice":{}}`) + empty.mu.Unlock() + if err := newSync(newR2(emptyURL), t.TempDir()).restore(); err != nil { + t.Errorf("a forge with no repositories was refused: %v", err) + } +} + +// What a restore knows the bucket holds has to come from the keys it fetched. +// Taking it from the directory instead would mean a file the bucket does not +// hold is remembered as stored, and the flush after it leaves it on a disk that +// does not survive. +func TestConfigOnDiskOnlyIsStillMirrored(t *testing.T) { + gitIdentity(t) + b, url := newBucket(t) + + seedDir := t.TempDir() + s := newSync(newR2(url), seedDir) + if err := s.restore(); err != nil { + t.Fatal(err) + } + if err := gitInit(filepath.Join(seedDir, "config"), false); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(seedDir, "config", "users.json"), []byte(`{"alice":{}}`), 0600); err != nil { + t.Fatal(err) + } + if err := s.flush(); err != nil { + t.Fatal(err) + } + + // A datadir that outlived the syncd that wrote it, holding a file whose + // flush never succeeded. + dirty := t.TempDir() + if err := os.MkdirAll(filepath.Join(dirty, "config"), 0700); err != nil { + t.Fatal(err) + } + repos := `{"notes":{"visibility":"public"}}` + if err := os.WriteFile(filepath.Join(dirty, "config", "repos.json"), []byte(repos), 0600); err != nil { + t.Fatal(err) + } + + f := newSync(newR2(url), dirty) + if err := f.restore(); err != nil { + t.Fatal(err) + } + if err := f.flush(); err != nil { + t.Fatal(err) + } + if got := b.object("config/repos.json"); got != repos { + t.Errorf("the bucket holds %q for a file only disk had", got) + } +} + +// git-forge writes a config file by creating {name}.tmp{n} beside it and +// renaming over the top, so a pass can catch one between the two. Mirroring it +// stores a half written account table under a key nothing removes. +func TestConfigTempFilesAreNotMirrored(t *testing.T) { + b, url := newBucket(t) + + dir := t.TempDir() + s := newSync(newR2(url), dir) + if err := s.restore(); err != nil { + t.Fatal(err) + } + users := `{"alice":{}}` + if err := os.WriteFile(filepath.Join(dir, "config", "users.json"), []byte(users), 0600); err != nil { + t.Fatal(err) + } + // What git-forge's writeFile leaves beside it for the length of an fsync. + temp := "users.json.tmp3877658021" + if err := os.WriteFile(filepath.Join(dir, "config", temp), []byte(`{"alice":{"par`), 0600); err != nil { + t.Fatal(err) + } + if err := s.flush(); err != nil { + t.Fatal(err) + } + if got := b.object("config/" + temp); got != "" { + t.Errorf("the bucket holds a file git-forge was still writing: %q", got) + } + if got := b.object("config/users.json"); got != users { + t.Errorf("the account table is not in the bucket, which holds %q", got) + } + + // And one already there is left where it is rather than restored, because + // putting it back on disk is what makes it permanent. + b.mu.Lock() + b.objs["config/"+temp] = []byte(`{"alice":{"par`) + b.mu.Unlock() + + fresh := t.TempDir() + f := newSync(newR2(url), fresh) + if err := f.restore(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(fresh, "config", temp)); err == nil { + t.Error("a restore wrote a half written config file back to disk") + } + if got, err := os.ReadFile(filepath.Join(fresh, "config", "users.json")); err != nil || string(got) != users { + t.Errorf("the account table did not come back: %q: %v", got, err) + } +} + +// Only syncd writes this bucket, so a key of a shape it never writes means the +// bucket has been tampered with. +func TestRestoreRefusesAKeyItDidNotWrite(t *testing.T) { + for _, key := range []string{ + "config/.git/config", + "config/.git/hooks/pre-commit", + "config/git/hooks/pre-commit", + "config/sub/dir/users.json", + } { + b, url := newBucket(t) + b.mu.Lock() + b.objs["config/users.json"] = []byte(`{}`) + b.objs[key] = []byte("#!/bin/sh\nid\n") + b.mu.Unlock() + + dir := t.TempDir() + s := newSync(newR2(url), dir) + if err := s.restore(); err == nil { + t.Errorf("a restore believed %s", key) + } + } +} + +// git-forge serves without an audit log when git will not initialise. The +// working tree still has to be mirrored: it is what a restore reads. +func TestConfigFlushedWithoutAuditLog(t *testing.T) { + b, url := newBucket(t) + + dir := t.TempDir() + s := newSync(newR2(url), dir) + if err := s.restore(); err != nil { + t.Fatal(err) + } + users := `{"alice":{}}` + if err := os.WriteFile(filepath.Join(dir, "config", "users.json"), []byte(users), 0600); err != nil { + t.Fatal(err) + } + if err := s.flush(); err != nil { + t.Fatal(err) + } + if got := b.object("config/users.json"); got != users { + t.Errorf("the account table is not in the bucket, which holds %q", got) + } +} + +// A hydration that fails part way must leave the placeholder alone, so the next +// request starts over. An objects directory left behind counts as hydrated. +func TestHydrationRetriedAfterFailure(t *testing.T) { + gitIdentity(t) + b, url := newBucket(t) + head := seed(t, b, url) + + dir := t.TempDir() + s := newSync(newR2(url), dir) + if err := s.restore(); err != nil { + t.Fatal(err) + } + + b.fail(func(method, key string) bool { + return method == http.MethodGet && strings.Contains(key, "/objects/") + }) + if err := s.hydrate("x"); err == nil { + t.Fatal("a hydration that could not read the objects reported success") + } + b.fail(nil) + + if err := s.hydrate("x"); err != nil { + t.Fatalf("second attempt: %v", err) + } + out, _ := git(filepath.Join(dir, "repos", "x.git"), "", nil, "show-ref") + if !strings.Contains(string(out), head) { + t.Errorf("the second request did not hydrate it, show-ref says %q", out) + } + if err := s.flush(); err != nil { + t.Fatal(err) + } + if refs := b.object("repos/x.git/REFS"); !strings.Contains(refs, head) { + t.Errorf("the flush that followed overwrote the stored refs with %q", refs) + } +} -- cgit v1.2.3