diff options
| author | Lena <lena@omega> | 2026-08-01 00:00:00 +0000 |
|---|---|---|
| committer | Lena <lena@omega> | 2026-08-01 00:00:00 +0000 |
| commit | bb6bc2387d58c737dbf92e73f8544bf39319dbd8 (patch) | |
| tree | 6d5eaf5dac213074cd832da50f64260fce728fed /store.go | |
| download | git-forge-bb6bc2387d58c737dbf92e73f8544bf39319dbd8.tar.gz | |
A small self-hosted git forge: HTTP push and pull, cgit browsing,
local and OIDC accounts, TOTP, scoped push tokens, per-repo
permissions, issues, a wiki, and outbound mail.
It does not reimplement git. git-http-backend and cgit run as CGI
behind its own listener, which authorises every request before
exec'ing either. One binary, one data directory, one listener: no
database, no JavaScript, and state is JSON in a git repo with a
working tree.
Diffstat (limited to 'store.go')
| -rw-r--r-- | store.go | 761 |
1 files changed, 761 insertions, 0 deletions
diff --git a/store.go b/store.go new file mode 100644 index 0000000..7ba9ac7 --- /dev/null +++ b/store.go @@ -0,0 +1,761 @@ +package main + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "log" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "syscall" +) + +// Store is the data directory, which holds four entries and nothing else: +// +// config/ git repo with a working tree: forge.json, users.json, repos.json +// cookie.key generated secret, deliberately outside the config repo +// mailq/ spooled outbound mail, transient +// repos/ bare git repos, holding nothing the forge wrote +type Store struct { + cfg *Config + dir string + configDir string + reposDir string + mailqDir string + cookieKey []byte + + // mu guards the tables, the git commands, and the directories: one lock, + // so there is no order to get wrong. Tables are replaced whole and handed + // out as copies. + mu sync.Mutex + users map[string]User + repos map[string]Repo // repos.json + dirs map[string]bool // what the tree actually holds, from the startup scan +} + +// RepoEntry is one repository as the tree has it. Registered is false for a +// directory with no entry in repos.json. +type RepoEntry struct { + Path string + Repo Repo + Registered bool +} + +// Author is a git identity in the config repo's audit log. +type Author struct{ Name, Email string } + +// User is one record in users.json, for either origin. +type User struct { + Origin string `json:"origin"` + Iss string `json:"iss,omitempty"` // external accounts: identity is + Sub string `json:"sub,omitempty"` // iss plus sub, and nothing else + Admin bool `json:"admin"` + Email string `json:"email,omitempty"` + EmailConfirmed bool `json:"email_confirmed"` + Notify bool `json:"notify"` + Epoch int `json:"epoch"` + Password string `json:"password,omitempty"` // argon2id, local accounts + TOTP *TOTP `json:"totp,omitempty"` + Tokens []Token `json:"tokens,omitempty"` +} + +// TOTP is the second factor. Recovery codes are kept as digests, like push +// tokens. Used is the last time step a code was accepted at, so a code cannot +// be replayed within its window. +type TOTP struct { + Secret string `json:"secret"` + Recovery []string `json:"recovery"` + Used int64 `json:"used,omitempty"` +} + +// Token is one scoped push token. Dates are days: an instant would mean a +// write per request. +type Token struct { + ID string `json:"id"` + Repo string `json:"repo"` + Hash string `json:"hash"` + Created string `json:"created,omitempty"` + Used string `json:"used,omitempty"` +} + +// The admin flag does nothing until the account holds a second factor. +func (u User) adminInEffect() bool { return u.Admin && u.TOTP != nil } + +func (u User) clone() User { + u.Tokens = append([]Token(nil), u.Tokens...) + if u.TOTP != nil { + totp := *u.TOTP + totp.Recovery = append([]string(nil), totp.Recovery...) + u.TOTP = &totp + } + return u +} + +const ( + originLocal = "local" + originOIDC = "oidc" +) + +// An account name is what grants point at, and is immutable. An external one +// is prefixed by its provider's key, as in sso:carol. +var accountNameRE = regexp.MustCompile(`^([a-z][a-z0-9]{0,15}:)?[a-z0-9][a-z0-9._-]{0,31}$`) + +// A local name has no provider prefix, so it cannot collide with a minted one. +var localNameRE = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,31}$`) + +// accountName forgives the shift key: names are lowercase by construction. +func accountName(typed string) string { return strings.ToLower(strings.TrimSpace(typed)) } + +// lockDatadir refuses a second process, which would lose writes. On the +// directory itself, so the kernel drops it however the process exits. +func lockDatadir(dir string) (func(), error) { + if err := os.MkdirAll(dir, 0700); err != nil { + return nil, fmt.Errorf("datadir %s: %v", dir, err) + } + f, err := os.Open(dir) + if err != nil { + return nil, fmt.Errorf("datadir %s: %v", dir, err) + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + f.Close() + return nil, fmt.Errorf("datadir %s is already being served by another git-forge: %v", dir, err) + } + return func() { f.Close() }, nil +} + +// openStore creates whatever is missing, idempotently, and reads the config. +func openStore(dir string) (*Store, error) { + s := &Store{ + dir: dir, + configDir: filepath.Join(dir, "config"), + reposDir: filepath.Join(dir, "repos"), + mailqDir: filepath.Join(dir, "mailq"), + } + // 0700 throughout: the whole datadir is credential material. + for _, d := range []string{s.dir, s.configDir, s.reposDir, s.mailqDir} { + if err := os.MkdirAll(d, 0700); err != nil { + return nil, err + } + } + + // git is only the audit log; one that will not initialise must not stop + // the forge. + if err := s.ensureConfigRepo(); err != nil { + log.Printf("config: no audit log: %v", err) + } + + forge, err := marshalJSON(defaultConfig()) + if err != nil { + return nil, err + } + defaults := []struct { + name string + data []byte + }{ + {"forge.json", forge}, + {"users.json", []byte("{}\n")}, + {"repos.json", []byte("{}\n")}, + } + var made []string + for _, d := range defaults { + created, err := ensureFile(s.File(d.name), d.data) + if err != nil { + return nil, err + } + if created { + log.Printf("config: created %s", d.name) + made = append(made, d.name) + } + } + + // Before the first commit: the committer's address needs the host. + cfg, err := loadConfig(s.File("forge.json")) + if err != nil { + return nil, fmt.Errorf("%s: %v", s.File("forge.json"), err) + } + s.cfg = cfg + if len(made) > 0 { + s.Commit("config: add "+strings.Join(made, ", "), s.forgeAuthor()) + } + + if err := s.loadUsers(); err != nil { + return nil, err + } + if err := s.loadRepos(); err != nil { + return nil, err + } + if err := s.scanRepos(); err != nil { + return nil, err + } + + key, err := cookieKey(filepath.Join(dir, "cookie.key")) + if err != nil { + return nil, err + } + s.cookieKey = key + return s, nil +} + +func (s *Store) loadUsers() error { + data, err := os.ReadFile(s.File("users.json")) + if err != nil { + return err + } + users := map[string]User{} + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(&users); err != nil { + return fmt.Errorf("users.json: %v", err) + } + for name, u := range users { + if !accountNameRE.MatchString(name) { + return fmt.Errorf("users.json: %q is not a usable account name", name) + } + if u.Origin != originLocal && u.Origin != originOIDC { + return fmt.Errorf("users.json: %s: origin %q is neither local nor oidc", name, u.Origin) + } + } + s.users = users + return nil +} + +func (s *Store) User(name string) (User, bool) { + s.mu.Lock() + defer s.mu.Unlock() + u, ok := s.users[name] + return u.clone(), ok +} + +func (s *Store) Users() map[string]User { + s.mu.Lock() + defer s.mu.Unlock() + out := make(map[string]User, len(s.users)) + for name, u := range s.users { + out[name] = u.clone() + } + return out +} + +// UpdateUsers edits under the lock, writes, and commits. fn is handed a copy; +// returning an error changes nothing. +func (s *Store) UpdateUsers(subject string, author Author, fn func(map[string]User) error) error { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.writeUsersLocked(fn); err != nil { + return err + } + s.Commit(subject, author) + return nil +} + +// UpdateUser edits one account. +func (s *Store) UpdateUser(name, subject string, author Author, fn func(*User) error) error { + return s.UpdateUsers(subject, author, editUser(name, fn)) +} + +// editUser leaves an account that is gone alone: writing it back would +// recreate it with an origin the next start refuses. +func editUser(name string, fn func(*User) error) func(map[string]User) error { + return func(m map[string]User) error { + acct, ok := m[name] + if !ok { + return nil + } + if err := fn(&acct); err != nil { + return err + } + m[name] = acct + return nil + } +} + +// WriteConfig rewrites forge.json. Read without the environment on top, so an +// override never becomes the file's own value; the name is the only edited +// field with a rule. +func (s *Store) WriteConfig(author Author, apply func(*Config)) error { + s.mu.Lock() + defer s.mu.Unlock() + + cfg, err := readConfig(s.File("forge.json")) + if err != nil { + return err + } + apply(cfg) + if !validSiteName(cfg.SiteName) { + return errSiteName + } + data, err := marshalJSON(cfg) + if err != nil { + return err + } + if err := writeFile(s.File("forge.json"), data, 0600); err != nil { + return err + } + s.Commit("config: update the site", author) + return nil +} + +// touchUser writes without committing: telemetry would bury the audit log. The +// next commit sweeps it in. +func (s *Store) touchUser(name string, fn func(*User) error) error { + s.mu.Lock() + defer s.mu.Unlock() + return s.writeUsersLocked(editUser(name, fn)) +} + +func (s *Store) writeUsersLocked(fn func(map[string]User) error) error { + next := make(map[string]User, len(s.users)) + for name, u := range s.users { + next[name] = u.clone() + } + if err := fn(next); err != nil { + return err + } + data, err := marshalJSON(next) + if err != nil { + return err + } + if err := writeFile(s.File("users.json"), data, 0600); err != nil { + return err + } + s.users = next + return nil +} + +func (s *Store) loadRepos() error { + data, err := os.ReadFile(s.File("repos.json")) + if err != nil { + return err + } + repos := map[string]Repo{} + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(&repos); err != nil { + return fmt.Errorf("repos.json: %v", err) + } + var errs []error + for path, repo := range repos { + errs = append(errs, repo.validate(path)...) + } + if err := errors.Join(errs...); err != nil { + return fmt.Errorf("repos.json:\n%v", err) + } + s.repos = repos + return nil +} + +// scanRepos reconciles repos.json against the tree: a directory ending in .git +// is a repository, anything else is a section one level deep. Disagreements +// are logged, never fatal. Every name is quoted: this is the one place a name +// has been through no rule, and a newline in one would write the next log +// line. +func (s *Store) scanRepos() error { + dirs := map[string]bool{} + top, err := os.ReadDir(s.reposDir) + if err != nil { + return err + } + for _, entry := range top { + if !entry.IsDir() { + log.Printf("repos: %q is not a directory, ignoring it", entry.Name()) + continue + } + if siblingDir(entry.Name()) { + continue + } + if path, ok := strings.CutSuffix(entry.Name(), ".git"); ok { + dirs[path] = true + continue + } + section := entry.Name() + inner, err := os.ReadDir(filepath.Join(s.reposDir, section)) + if err != nil { + log.Printf("repos: %q cannot be read, ignoring it: %v", section, err) + continue + } + for _, entry := range inner { + if siblingDir(entry.Name()) { + continue + } + path, ok := strings.CutSuffix(entry.Name(), ".git") + if !entry.IsDir() || !ok { + log.Printf("repos: %q is neither a repository nor a section, ignoring it", section+"/"+entry.Name()) + continue + } + dirs[section+"/"+path] = true + } + } + + s.mu.Lock() + defer s.mu.Unlock() + for path := range dirs { + if !validRepoPath(path) { + log.Printf("repos: %q is not a usable repository path, so it is not served at all", path+".git") + delete(dirs, path) + continue + } + if _, ok := s.repos[path]; !ok { + log.Printf("repos: %q has no entry in repos.json, serving it private with issues and wiki off until an admin adopts or removes it", path) + } + } + for path := range s.repos { + if !dirs[path] { + log.Printf("repos: %q has an entry in repos.json but no directory, ignoring it", path) + } + } + s.dirs = dirs + return nil +} + +// siblingDir is a repository's issues or wiki directory, not a section. +func siblingDir(name string) bool { + path, ok := strings.CutSuffix(name, ".git") + return ok && (strings.HasSuffix(path, ".issues") || strings.HasSuffix(path, ".wiki")) +} + +// The tree decides existence; a directory with no entry is served as an +// orphan. +func (s *Store) Repo(path string) (Repo, bool) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.dirs[path] { + return Repo{}, false + } + if repo, ok := s.repos[path]; ok { + return repo.clone(), true + } + return orphanRepo(), true +} + +// Repos is sorted by path. +func (s *Store) Repos() []RepoEntry { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]RepoEntry, 0, len(s.dirs)) + for path := range s.dirs { + repo, ok := s.repos[path] + if !ok { + repo = orphanRepo() + } + out = append(out, RepoEntry{Path: path, Repo: repo.clone(), Registered: ok}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out +} + +// splitRepo resolves the repository a URL names, which of the three it is, and +// the rest. Two components are tried before one, so a section wins over a root +// repository of the same name. +func (s *Store) splitRepo(p string) (repo, kind, rest string, ok bool) { + p = strings.TrimPrefix(p, "/") + parts := strings.SplitN(p, "/", 3) + for n := min(2, len(parts)); n >= 1; n-- { + head := strings.Join(parts[:n], "/") + base, kind := head, "" + if cut, found := strings.CutSuffix(head, ".issues"); found { + base, kind = cut, "issues" + } else if cut, found := strings.CutSuffix(head, ".wiki"); found { + base, kind = cut, "wiki" + } + if _, exists := s.Repo(base); !exists { + continue + } + return base, kind, strings.TrimPrefix(strings.TrimPrefix(p, head), "/"), true + } + return "", "", "", false +} + +// splitGit resolves a /{repo}.git/... path without asking whether the +// repository exists: that is decided where the answer can be a challenge. +// kind is "" for the repository, or issues or wiki for a sibling. +func splitGit(p string) (repo, kind, rest string, ok bool) { + p = strings.TrimPrefix(p, "/") + parts := strings.SplitN(p, "/", 3) + for n := min(2, len(parts)); n >= 1; n-- { + dir := strings.Join(parts[:n], "/") + base, isGit := strings.CutSuffix(dir, ".git") + if !isGit { + continue + } + if cut, ok := strings.CutSuffix(base, ".issues"); ok { + base, kind = cut, "issues" + } else if cut, ok := strings.CutSuffix(base, ".wiki"); ok { + base, kind = cut, "wiki" + } + return base, kind, strings.TrimPrefix(strings.TrimPrefix(p, dir), "/"), true + } + return "", "", "", false +} + +// UpdateRepos edits repos.json under the same rules as UpdateUsers. +func (s *Store) UpdateRepos(subject string, author Author, fn func(map[string]Repo) error) error { + s.mu.Lock() + defer s.mu.Unlock() + return s.updateReposLocked(subject, author, fn) +} + +func (s *Store) updateReposLocked(subject string, author Author, fn func(map[string]Repo) error) error { + next := make(map[string]Repo, len(s.repos)) + for path, repo := range s.repos { + next[path] = repo.clone() + } + if err := fn(next); err != nil { + return err + } + data, err := marshalJSON(next) + if err != nil { + return err + } + if err := writeFile(s.File("repos.json"), data, 0600); err != nil { + return err + } + s.repos = next + s.Commit(subject, author) + return nil +} + +// CreateRepo initialises the repository and a sibling per enabled feature, and +// records it. +func (s *Store) CreateRepo(path string, repo Repo, author Author) error { + s.mu.Lock() + defer s.mu.Unlock() + + if s.dirs[path] { + return userError(path + " already exists") + } + if errs := repo.validate(path); len(errs) > 0 { + return userError(errors.Join(errs...).Error()) + } + for _, dir := range s.repoDirs(path, repo) { + if err := gitInit(dir, true); err != nil { + return err + } + } + err := s.updateReposLocked("repos: create "+path, author, func(m map[string]Repo) error { + m[path] = repo + return nil + }) + if err != nil { + return err + } + s.dirs[path] = true + return nil +} + +// EnsureSiblings creates what a knob turned on. One turned off keeps its +// repository; the knob governs access. +func (s *Store) EnsureSiblings(path string, repo Repo) error { + s.mu.Lock() + defer s.mu.Unlock() + for _, dir := range s.repoDirs(path, repo) { + if _, err := os.Stat(dir); err == nil { + continue + } + if err := gitInit(dir, true); err != nil { + return err + } + } + return nil +} + +// DeleteRepo removes all three directories, and an emptied section. +func (s *Store) DeleteRepo(path string, author Author) error { + s.mu.Lock() + defer s.mu.Unlock() + + for _, suffix := range []string{".git", ".issues.git", ".wiki.git"} { + if err := os.RemoveAll(filepath.Join(s.reposDir, path+suffix)); err != nil { + return err + } + } + if section, _, ok := strings.Cut(path, "/"); ok { + dir := filepath.Join(s.reposDir, section) + if entries, err := os.ReadDir(dir); err == nil && len(entries) == 0 { + if err := os.Remove(dir); err != nil { + return err + } + } + } + delete(s.dirs, path) + return s.updateReposLocked("repos: delete "+path, author, func(m map[string]Repo) error { + delete(m, path) + return nil + }) +} + +// repoDirs is what a repository has on disk, given its knobs. +func (s *Store) repoDirs(path string, repo Repo) []string { + dirs := []string{filepath.Join(s.reposDir, path+".git")} + if repo.Issues != featDisabled { + dirs = append(dirs, filepath.Join(s.reposDir, path+".issues.git")) + } + if repo.Wiki != featDisabled { + dirs = append(dirs, filepath.Join(s.reposDir, path+".wiki.git")) + } + return dirs +} + +// author is the identity a change is recorded under; git wants an address that +// parses. +func (s *Store) author(name string, u User) Author { + if u.Email != "" { + return Author{Name: name, Email: u.Email} + } + return Author{Name: name, Email: name + "@" + s.cfg.host} +} + +func (s *Store) File(name string) string { return filepath.Join(s.configDir, name) } + +func (s *Store) ensureConfigRepo() error { + _, err := os.Stat(filepath.Join(s.configDir, ".git")) + if err == nil { + return nil + } + if !errors.Is(err, fs.ErrNotExist) { + return err + } + return gitInit(s.configDir, false) +} + +// Commit records a change in the config repo. Failure is logged, never fatal: +// history must not take the forge down. add -A, so a manual edit is picked up +// rather than clobbered. +func (s *Store) Commit(subject string, author Author) { + if err := s.commit(subject, author); err != nil { + log.Printf("config: %q not committed, the working tree is still authoritative: %v", subject, err) + } +} + +func (s *Store) commit(subject string, author Author) error { + if _, err := git(s.configDir, nil, "add", "-A"); err != nil { + return err + } + out, err := git(s.configDir, nil, "status", "--porcelain") + if err != nil { + return err + } + if len(bytes.TrimSpace(out)) == 0 { + return nil + } + committer := s.forgeAuthor() + env := []string{ + "GIT_AUTHOR_NAME=" + author.Name, + "GIT_AUTHOR_EMAIL=" + author.Email, + "GIT_COMMITTER_NAME=" + committer.Name, + "GIT_COMMITTER_EMAIL=" + committer.Email, + } + _, err = git(s.configDir, env, "commit", "--quiet", "-m", subject) + return err +} + +// forgeAuthor is the committer, and the author when no account is acting. +func (s *Store) forgeAuthor() Author { + return Author{Name: "git-forge", Email: "git-forge@" + s.cfg.host} +} + +// marshalJSON is encoding/json without HTML escaping: these files are read +// with cat. +func marshalJSON(v any) ([]byte, error) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + if err := enc.Encode(v); err != nil { // Encode ends the file with a newline + return nil, err + } + return buf.Bytes(), nil +} + +// ensureFile reports whether it wrote, so only a first start commits. +func ensureFile(path string, data []byte) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return false, nil + } + if !errors.Is(err, fs.ErrNotExist) { + return false, err + } + return true, writeFile(path, data, 0600) +} + +// writeFile writes beside the target and renames, so nothing sees a partial +// file. +func writeFile(path string, data []byte, perm os.FileMode) error { + dir := filepath.Dir(path) + f, err := os.CreateTemp(dir, filepath.Base(path)+".tmp") + if err != nil { + return err + } + tmp := f.Name() + defer os.Remove(tmp) + + if _, err := f.Write(data); err != nil { + f.Close() + return err + } + if err := f.Chmod(perm); err != nil { + f.Close() + return err + } + if err := f.Sync(); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + if err := os.Rename(tmp, path); err != nil { + return err + } + return syncDir(dir) // the rename itself has to be durable +} + +func syncDir(dir string) error { + f, err := os.Open(dir) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} + +// cookieKey signs every session cookie: GIT_FORGE_COOKIE_KEY, else generated +// into {datadir}/cookie.key, outside the config repo. +func cookieKey(path string) ([]byte, error) { + if v, ok := os.LookupEnv("GIT_FORGE_COOKIE_KEY"); ok { + if len(v) < 32 { + return nil, errors.New("GIT_FORGE_COOKIE_KEY: at least 32 characters required") + } + return []byte(v), nil + } + + data, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + key := make([]byte, 32) + if _, err := rand.Read(key); err != nil { + return nil, err + } + if err := writeFile(path, []byte(hex.EncodeToString(key)+"\n"), 0600); err != nil { + return nil, err + } + log.Printf("generated %s", path) + return key, nil + } + if err != nil { + return nil, err + } + + key, err := hex.DecodeString(strings.TrimSpace(string(data))) + if err != nil || len(key) < 32 { + return nil, fmt.Errorf("%s: expected 64 hex characters; deleting it generates a new key, which signs everybody out", path) + } + return key, nil +} |