feat: shallow snapshot synchronization and storage budgets

.github/workflows/ci.yml

@@ -22,11 +22,18 @@ jobs:
       - run: bun run build:web
       - run: bun run check
       - run: go test -p 1 ./...
+      - name: Snapshot transport with minimum supported Git
+        run: |
+          curl --fail --location --retry 3 --max-time 120 https://github.com/git/git/archive/refs/tags/v2.30.2.tar.gz -o /tmp/git-2.30.2.tar.gz
+          tar -xzf /tmp/git-2.30.2.tar.gz -C /tmp
+          make -s -C /tmp/git-2.30.2 -j2 CFLAGS='-O2 -std=gnu99' prefix=/tmp/git230 NO_GETTEXT=YesPlease NO_TCLTK=YesPlease NO_CURL=YesPlease NO_EXPAT=YesPlease NO_OPENSSL=YesPlease install
+          PATH=/tmp/git230/bin:$PATH git --version
+          PATH=/tmp/git230/bin:$PATH go test -count=1 -v ./internal/gitstore -run TestShallowSnapshots
       - run: python3 scripts/audit-public.py
       - run: docker build --build-arg BUILD_REVISION=${{ github.sha }} -t sourcegraph:verify .
   databases:
-    # MongoDB refuses to start on Linux 6.19+ (SERVER-121912), which ubuntu-latest (26.04) now runs.
-    runs-on: ubuntu-24.04
+    # MongoDB 8 rejects Linux 6.19+ (SERVER-121912). The 22.04 image uses Linux 6.8.
+    runs-on: ubuntu-22.04
     services:
       postgres:
         image: postgres:17
@@ -44,8 +51,13 @@ jobs:
           go-version-file: go.mod
       - name: Start MongoDB replica set
         run: |
+          uname -r
           docker run -d --name mongo-contract -p 27017:27017 mongo:8 --replSet rs0 --bind_ip_all
           for i in $(seq 1 40); do
+            if [ "$(docker inspect -f '{{.State.Running}}' mongo-contract)" != true ]; then
+              docker logs --tail 80 mongo-contract
+              exit 1
+            fi
             if docker exec mongo-contract mongosh --quiet --eval 'db.adminCommand({ping:1})'; then break; fi
             sleep 2
           done

README.md

@@ -21,6 +21,7 @@ It is built for the workload AI coding agents generate: many small, precise `sea
 - **Reproducible answers.** Branches and tags resolve to a commit before anything is read, so a search result, the file it points to and a later diff all refer to the same snapshot.
 - **One contract, four surfaces.** Browser, HTTP, MCP and CLI share the same operations: `repositories`, `availability`, `resolve`, `search`, `read`, `list`, `diff` and `prepare`. MCP adds `read_many` for several reads in one call. See [interfaces](docs/interfaces.md).
 - **Agent-ready.** A stateless Streamable HTTP MCP endpoint works with any MCP client; anonymous access is read-only and limited to public repositories.
+- **Snapshot retention.** Shallow branch-head synchronization, on-demand pinned SHAs and per-repository storage budgets; see [snapshot configuration](docs/snapshots.md).
 - **Small footprint.** One process manages Git mirrors, the Zoekt child process and background indexing. SQLite by default; PostgreSQL and MongoDB are supported.
 - **Extensible reads.** Repositories can ship Wasm plugins (`<name>.sourcegraph.wasm`) that change how `read` and `read_many` present files. See [repository plugins](docs/ref/repository-plugins.md).
 

docs/README.md

@@ -2,6 +2,7 @@
 
 - [Authentication and demo access](authentication.md)
 - [Code sources and tokens](code-sources.md)
+- [Snapshot synchronization and retention](snapshots.md)
 - [Deployment, storage and backups](deployment.md)
 - [HTTP, MCP and CLI](interfaces.md)
 - [Repository plugins](ref/repository-plugins.md)

docs/snapshots.md

@@ -0,0 +1,17 @@
+# Snapshot synchronization and retention
+
+Synchronization fetches depth-one snapshots of the watched branch heads. The first sync does not download their parent history. Later syncs reuse existing Git objects and add newly observed heads; commits between synchronization runs are not automatically indexed.
+
+To prepare a specific baseline, use **Create index** with its full lowercase 40-character commit SHA, or the prepare API with that revision. Missing snapshots are fetched directly from the registered repository, without deepening its full history. The Git server must permit fetching that SHA. A rejected request fails without substituting another commit. The branch selector identifies the watched scope; it does not prove ancestry for an explicitly supplied SHA.
+
+Explicit SHA preparation pins that snapshot. In repository settings, **Pinned snapshots** lists the versions to keep independently of ordinary retention. Removing a SHA releases its pin; it becomes eligible for expiry at the next successful sync. Adding a pin alone does not fetch or index an absent snapshot: prepare the SHA first. A missing or failed index remains visible in task status.
+
+Ordinary snapshots are retained by time since first observation, or by the configured count of most recently observed ordinary snapshots. Pins, current branch heads and active tasks are additionally protected. There are at most 4096 retained snapshots per path group. Commit choices show up to 20 observed snapshots, not the complete Git history.
+
+After a successful sync, expired snapshots lose query eligibility. Git collection then removes objects no longer reachable from any retained snapshot, preserving the complete trees and available file objects of retained versions. Index collection removes retired generations separately after its grace period. Adding snapshots alone does not rewrite all packs; collection runs when versions expire and periodically during synchronization. No synchronization means no automatic snapshot expiry. **Clean old indexes** keeps branch-head and pinned indexes.
+
+Each repository has a **Storage budget** (default 100 GiB, configurable from 1 GiB to 10 TiB) covering Git objects, index shards and active preparation files. Preparation stops when the sampled budget is reached. This is a protective budget, not an exact filesystem quota: checks occur between operations and during long-running work, so short-lived overshoot is possible. Increase the budget or release unused snapshots and clean indexes when capacity blocks preparation. Git compaction needs temporary working space and may exceed the repository budget while reclaiming objects; the global free-space reserve remains enforced.
+
+`SOURCEGRAPH_MIN_FREE_BYTES` protects the instance's remaining disk space across repositories. Keep enough additional free capacity for replacement packs and index builds. Back up the control database, including pin and retention settings; Git snapshots and indexes reside on the instance volume.
+
+Git 2.30 or newer is required for selective file hydration in path groups. The supplied production container includes a compatible Git version.

internal/control/model.go

@@ -41,6 +41,7 @@ func (p Policy) Validate() error {
 }
 
 type Version struct {
+	ObservedAt      time.Time              `json:"observed_at,omitempty" bson:"observed_at,omitempty"`
 	LinkUnavailable bool                   `json:"-" bson:"-"`
 	LinkFormat      int                    `json:"-" bson:"link_format,omitempty"`
 	Links           *gitstore.LinkSnapshot `json:"-" bson:"-"`
@@ -110,8 +111,11 @@ type DiskUsage struct {
 }
 
 type Repository struct {
-	SyncChecks   map[string]SyncCheck `json:"sync_checks,omitempty" bson:"sync_checks,omitempty"`
-	CodeProvider string               `json:"code_provider,omitempty" bson:"code_provider,omitempty"`
+	SnapshotStorage bool                 `json:"snapshot_storage" bson:"snapshot_storage,omitempty"`
+	PinnedCommits   []string             `json:"pinned_commits" bson:"pinned_commits,omitempty"`
+	MaxStorageBytes int64                `json:"max_storage_bytes" bson:"max_storage_bytes,omitempty"`
+	SyncChecks      map[string]SyncCheck `json:"sync_checks,omitempty" bson:"sync_checks,omitempty"`
+	CodeProvider    string               `json:"code_provider,omitempty" bson:"code_provider,omitempty"`
 	// CodeHost pins the HTTPS origin chosen at registration; later source changes affect new repositories only.
 	CodeHost  string `json:"code_host,omitempty" bson:"code_host,omitempty"`
 	CreatorID string `json:"created_by,omitempty" bson:"created_by,omitempty"`

internal/gitstore/fetch.go

@@ -16,6 +16,8 @@ import (
 )
 
 type FetchOptions struct {
+	Snapshot         bool
+	Revision         string
 	CheckGrowth      func() error
 	DefaultBranch    bool
 	CredentialHelper string
@@ -54,9 +56,12 @@ func (s *Store) fetch(ctx context.Context, repo string, o FetchOptions, selected
 	if o.DefaultBranch {
 		slots++
 	}
-	if slots < 1 || slots > 8 {
+	if (slots < 1 || slots > 8) && o.Revision == "" {
 		return contract.Fail("INVALID_BRANCHES", "Choose 1..8 exact branches.", 400)
 	}
+	if o.Revision != "" && !fullSHA.MatchString(o.Revision) {
+		return contract.Fail("INVALID_REVISION", "Use a full commit SHA.", 400)
+	}
 	for _, b := range o.Branches {
 		if ValidateRevision(b) != nil || strings.HasPrefix(b, "refs/") || fullSHA.MatchString(b) {
 			return contract.Fail("INVALID_BRANCH", "Use a short branch name.", 400)
@@ -104,6 +109,11 @@ func (s *Store) fetch(ctx context.Context, repo string, o FetchOptions, selected
 		}
 	}
 	defer syscall.Flock(fd, syscall.LOCK_UN)
+	if o.CheckGrowth != nil {
+		if err = o.CheckGrowth(); err != nil {
+			return err
+		}
+	}
 	if err = checkFree(s.root, o.MinFreeBytes); err != nil {
 		return err
 	}
@@ -147,12 +157,19 @@ func (s *Store) fetch(ctx context.Context, repo string, o FetchOptions, selected
 		}
 	}
 	args = append(args, "fetch", "--no-tags", "--force")
+	if o.Snapshot {
+		args = append(args, "--depth=1")
+	}
 	if s.group != "" {
 		args = append(args, "--filter=blob:none")
 	}
 	args = append(args, remote)
-	for _, b := range o.Branches {
-		args = append(args, "+refs/heads/"+b+":refs/heads/"+b)
+	if o.Revision != "" {
+		args = append(args, "+"+o.Revision+":refs/snapshots/"+o.Revision)
+	} else {
+		for _, b := range o.Branches {
+			args = append(args, "+refs/heads/"+b+":refs/heads/"+b)
+		}
 	}
 	command := s.command(ctx, p, args...)
 	command.Env = append(command.Env, credentialEnv(remote, o)...)
@@ -170,6 +187,13 @@ func (s *Store) fetch(ctx context.Context, repo string, o FetchOptions, selected
 			case <-ctx.Done():
 				return
 			case <-t.C:
+				if o.CheckGrowth != nil {
+					if e := o.CheckGrowth(); e != nil {
+						capacity = e
+						cancel()
+						return
+					}
+				}
 				if e := checkFree(s.root, o.MinFreeBytes); e != nil {
 					capacity = e
 					cancel()
@@ -191,7 +215,12 @@ func (s *Store) fetch(ctx context.Context, repo string, o FetchOptions, selected
 	if err != nil {
 		return remoteFailure(diagnostic.String(), "GIT_FETCH_FAILED", "Git fetch failed; check access and connectivity.")
 	}
-	return nil
+	if o.CheckGrowth != nil {
+		if err = o.CheckGrowth(); err != nil {
+			return err
+		}
+	}
+	return checkFree(s.root, o.MinFreeBytes)
 }
 func checkFree(root string, min uint64) error {
 	var st syscall.Statfs_t

internal/gitstore/partial.go

@@ -134,7 +134,7 @@ func (s *Store) HydrateEntries(ctx context.Context, repo, commit string, entries
 			end = len(missing)
 		}
 		remote := remoteURL(repo, o.Host)
-		args := append(fetchConfiguration(o), "fetch", "--no-tags", "--no-write-fetch-head", "--stdin", remote)
+		args := append(fetchConfiguration(o), "-c", "fetch.negotiationAlgorithm=noop", "fetch", "--no-tags", "--no-write-fetch-head", "--stdin", remote)
 		batchCtx, batchCancel := context.WithCancel(ctx)
 		c := s.command(batchCtx, p, args...)
 		c.Env = append(c.Env, credentialEnv(remote, o)...)

internal/gitstore/snapshots.go

@@ -0,0 +1,200 @@
+package gitstore
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Snapshot reads one commit without traversing its parents.
+func (s *Store) Snapshot(ctx context.Context, repo, revision string) (Commit, error) {
+	sha, err := s.Resolve(ctx, repo, revision)
+	if err != nil {
+		return Commit{}, err
+	}
+	p, err := s.repoPath(repo)
+	if err != nil {
+		return Commit{}, err
+	}
+	data, err := s.run(ctx, p, 128, "show", "-s", "--format=%ct", sha, "--")
+	if err != nil {
+		return Commit{}, err
+	}
+	n, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
+	return Commit{SHA: sha, CommitterTime: time.Unix(n, 0).UTC()}, err
+}
+
+// RetainSnapshots cuts parent traversal at each retained commit and reclaims
+// unreachable objects. The caller must drain readers of expired snapshots and exclude writers.
+// References are installed before changing boundaries; interruption can leak
+// bytes but cannot remove a retained snapshot. Re-running finishes collection.
+func (s *Store) RetainSnapshots(ctx context.Context, repo string, commits []string, checks ...func() error) error {
+	p, err := s.repoPath(repo)
+	if err != nil {
+		return err
+	}
+	if len(commits) == 0 || len(commits) > 4096 {
+		return errors.New("invalid snapshot retention set")
+	}
+	sortedCommits := append([]string(nil), commits...)
+	sort.Strings(sortedCommits)
+	marker := []byte(strings.Join(sortedCommits, "\n") + "\n")
+	markerPath := filepath.Join(p, "snapshot-retention")
+	previous, previousErr := os.ReadFile(markerPath)
+	previousInfo, statErr := os.Stat(markerPath)
+	// Adding shallow snapshots cannot make existing packs unreachable. Avoid
+	// rewriting the entire repository on every branch update; still collect
+	// daily to recover objects left behind by interrupted fetches.
+	needsPack := previousErr != nil || statErr != nil
+	if !needsPack {
+		needsPack = time.Since(previousInfo.ModTime()) >= 24*time.Hour
+	}
+	currentSet := map[string]bool{}
+	for _, sha := range commits {
+		currentSet[sha] = true
+	}
+	for _, sha := range strings.Fields(string(previous)) {
+		if !currentSet[sha] {
+			needsPack = true
+		}
+	}
+	keep := map[string]bool{}
+	for _, sha := range commits {
+		if !fullSHA.MatchString(sha) {
+			return errors.New("invalid retained commit")
+		}
+		if _, err := s.Snapshot(ctx, repo, sha); err != nil {
+			return err
+		}
+		keep[sha] = true
+	}
+	shas := make([]string, 0, len(keep))
+	for sha := range keep {
+		if _, err = s.run(ctx, p, 1024, "update-ref", "refs/snapshots/"+sha, sha); err != nil {
+			return err
+		}
+		shas = append(shas, sha)
+	}
+	sort.Strings(shas)
+	// Every kept commit is a root. Parent commits are not needed for snapshot reads.
+	tmp := filepath.Join(p, "shallow.snapshots")
+	if err = os.WriteFile(tmp, []byte(strings.Join(shas, "\n")+"\n"), 0600); err != nil {
+		return err
+	}
+	if err = os.Rename(tmp, filepath.Join(p, "shallow")); err != nil {
+		return err
+	}
+	refs, err := s.run(ctx, p, 2<<20, "for-each-ref", "--format=%(refname) %(objectname)")
+	if err != nil {
+		return err
+	}
+	for _, line := range strings.Split(strings.TrimSpace(string(refs)), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) != 2 {
+			return errors.New("invalid reference")
+		}
+		if !keep[fields[1]] {
+			if _, err = s.run(ctx, p, 1024, "update-ref", "-d", fields[0]); err != nil {
+				return err
+			}
+		}
+	}
+	if _, err = s.run(ctx, p, 1024, "reflog", "expire", "--expire=now", "--all"); err != nil {
+		return err
+	}
+	if !needsPack {
+		if err = os.WriteFile(markerPath, marker, 0600); err != nil {
+			return err
+		}
+		return os.Chtimes(markerPath, previousInfo.ModTime(), previousInfo.ModTime())
+	}
+	for _, check := range checks {
+		if err := check(); err != nil {
+			return err
+		}
+	}
+	repackCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	// Promisor repack deliberately retains unreachable remote objects. Build a
+	// reachable-only replacement pack instead, publishing it before removing old packs.
+	oldPacks, err := filepath.Glob(filepath.Join(p, "objects", "pack", "pack-*.pack"))
+	if err != nil {
+		return err
+	}
+	command := s.command(repackCtx, p, "pack-objects", "--revs", "--missing=allow-promisor", "--non-empty", filepath.Join(p, "objects", "pack", "pack"))
+	command.Stdin = strings.NewReader(strings.Join(shas, "\n") + "\n")
+	output := &limitWriter{limit: 1024}
+	command.Stdout = output
+	command.Stderr = io.Discard
+	done := make(chan struct{})
+	finished := make(chan struct{})
+	var capacity error
+	go func() {
+		defer close(finished)
+		ticker := time.NewTicker(time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-done:
+				return
+			case <-repackCtx.Done():
+				return
+			case <-ticker.C:
+				for _, check := range checks {
+					if e := check(); e != nil {
+						capacity = e
+						cancel()
+						return
+					}
+				}
+			}
+		}
+	}()
+	err = command.Run()
+	close(done)
+	<-finished
+	if capacity != nil {
+		return capacity
+	}
+	if err != nil {
+		return err
+	}
+	packID := strings.TrimSpace(output.String())
+	if !fullSHA.MatchString(packID) {
+		return errors.New("invalid replacement pack identity")
+	}
+	base := filepath.Join(p, "objects", "pack", "pack-"+packID)
+	if s.group != "" {
+		if err = os.WriteFile(base+".promisor", nil, 0600); err != nil {
+			return err
+		}
+	}
+	// A stale multi-pack index must not refer to packs about to be removed.
+	if err = os.Remove(filepath.Join(p, "objects", "pack", "multi-pack-index")); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	for _, old := range oldPacks {
+		oldBase := strings.TrimSuffix(old, ".pack")
+		if oldBase == base {
+			continue
+		}
+		for _, ext := range []string{".pack", ".idx", ".rev", ".bitmap", ".promisor", ".mtimes"} {
+			if err = os.Remove(oldBase + ext); err != nil && !os.IsNotExist(err) {
+				return err
+			}
+		}
+	}
+	if _, err = s.run(ctx, p, 4096, "prune", "--expire=now"); err != nil {
+		return err
+	}
+	return os.WriteFile(filepath.Join(p, "snapshot-retention"), marker, 0600)
+}
+
+// CheckFree enforces the global preparation reserve during snapshot collection.
+func (s *Store) CheckFree(min uint64) error { return checkFree(s.root, min) }