Persist credentials and add configurable Git timeouts

DEVELOPMENT.md

@@ -31,18 +31,18 @@ CI runs the same verification on pull requests and `main`; see `.github/workflow
 | --- | --- |
 | `SOURCEGRAPH_DATABASE` | `sqlite`; also `postgres` or `mongodb` |
 | `DATABASE_URL` | SQLite absolute file path (default `/data/sourcegraph.db`), PostgreSQL DSN, or standard MongoDB URI |
-| `SOURCEGRAPH_DATA_ROOT` | `/data`; persistent Git, indexes, manifests and encryption key |
+| `SOURCEGRAPH_DATA_ROOT` | `/data`; persistent Git, indexes and manifests |
 | `SOURCEGRAPH_ORIGIN` | `http://localhost:8080`; public origin, without a path |
 | `SOURCEGRAPH_LISTEN` | `0.0.0.0:8080` |
 | `SOURCEGRAPH_MIN_FREE_BYTES` | `1073741824`; index/fetch free-space floor |
 | `SOURCEGRAPH_SEED_DEMO` | Set `true` to register the two public example repositories |
 | `SOURCEGRAPH_GITHUB_TOKEN` | Optional GitHub token, used until a global GitHub token is saved; the earlier name `SOURCEGRAPH_GIT_TOKEN` is still read when this is unset |
 | `SOURCEGRAPH_CUSTOM_GIT_TOKEN` | Optional token for the custom Git source, used until a global one is saved |
 | `SOURCEGRAPH_CONFIG_FILE` | Optional JSON file that overrides code sources at startup; see [code sources](docs/code-sources.md) |
-| `SOURCEGRAPH_CREDENTIAL_KEY` | Optional base64 32-byte key for stored code credentials; otherwise `code-credentials.key` is generated in the data root. Required, and identical, on all instances sharing a database |
-| `SOURCEGRAPH_API_TOKEN` | Optional service automation bearer credential, at least 32 characters |
-| `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET` | Google web OAuth client |
-| `GITHUB_CLIENT_ID`, `GITHUB_CLIENT_SECRET` | GitHub OAuth app |
+| `SOURCEGRAPH_CREDENTIAL_KEY` | First-upgrade seed: base64 32-byte key for existing code credentials. Otherwise imports the existing `code-credentials.key`; a fresh database generates its own key. The database becomes authoritative |
+| `SOURCEGRAPH_API_TOKEN` | First-start seed for the database-backed service bearer, at least 32 characters; generated if absent |
+| `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET` | Google web OAuth client, imported together into the database on first configuration |
+| `GITHUB_CLIENT_ID`, `GITHUB_CLIENT_SECRET` | GitHub OAuth app, imported together into the database on first configuration |
 
 Repository read plugins are tuned with `SOURCEGRAPH_PLUGINS_*`; see [repository plugins](docs/ref/repository-plugins.md). The CLI reads `SOURCEGRAPH_URL` and `SOURCEGRAPH_API_TOKEN`; see [interfaces](docs/interfaces.md#cli).
 
@@ -54,3 +54,9 @@ Never put credentials in deployment JSON, source code or images. Public non-secr
 - [Interfaces](docs/interfaces.md) — HTTP routes, MCP and CLI
 - [Deployment](docs/deployment.md) — Fly.io, databases, backups and recovery
 - [Repository plugins](docs/ref/repository-plugins.md) — Wasm plugin contract
+
+## Runtime Git timeout
+
+System settings stores the global Git timeout in the database (15 minutes by default). Repositories inherit it by default or override it with 15, 30, 45 or 60 minutes. A new task snapshots the effective timeout; editing it does not interrupt running work or remove indexes. This covers Git fetch and object hydration, not total indexing time.
+
+See [deployment](docs/deployment.md#application-secrets) for credential migration and backup requirements.

cmd/repo-service/main.go

@@ -9,11 +9,13 @@ import (
 	"github.com/context4ai/sourcegraph/internal/config"
 	"github.com/context4ai/sourcegraph/internal/control"
 	"github.com/context4ai/sourcegraph/internal/httpserver"
+	"github.com/context4ai/sourcegraph/internal/secretstore"
 	"github.com/context4ai/sourcegraph/internal/service"
 	"github.com/context4ai/sourcegraph/internal/sitesettings"
 	"github.com/context4ai/sourcegraph/internal/sqlstore"
 	"github.com/context4ai/sourcegraph/internal/sso"
 	"log/slog"
+	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -63,6 +65,7 @@ func run() error {
 	var repos service.Store
 	var auth sso.Store
 	var settingsDB sitesettings.Store
+	var openSecrets func() (*secretstore.Store, error)
 	driver := env("SOURCEGRAPH_DATABASE", "sqlite")
 	dsn := os.Getenv("DATABASE_URL")
 	if driver == "mongodb" {
@@ -74,6 +77,7 @@ func run() error {
 		repos = db
 		auth = &sso.MongoStore{DB: db.AuthDatabase()}
 		settingsDB = &sitesettings.MongoStore{DB: db.AuthDatabase()}
+		openSecrets = func() (*secretstore.Store, error) { return secretstore.New(ctx, db.AuthDatabase()) }
 	} else {
 		if driver == "sqlite" && dsn == "" {
 			dsn = filepath.Join(root, "sourcegraph.db")
@@ -100,6 +104,7 @@ func run() error {
 		repos = &control.SQL{DB: db}
 		auth = &sso.SQLStore{DB: db}
 		settingsDB = &sitesettings.SQLStore{DB: db}
+		openSecrets = func() (*secretstore.Store, error) { return secretstore.NewSQL(ctx, db) }
 	}
 	exe, e := os.Executable()
 	if e != nil {
@@ -111,6 +116,22 @@ func run() error {
 		return e
 	}
 	defer backend.Close()
+	vault, e := openSecrets()
+	if e != nil {
+		return e
+	}
+	token := os.Getenv("SOURCEGRAPH_API_TOKEN")
+	if token != "" && len(token) < 32 {
+		return errors.New("service API credential must contain at least 32 characters")
+	}
+	savedToken, e := vault.Ensure(ctx, "service-api-token", []byte(token))
+	if e != nil {
+		return e
+	}
+	if len(savedToken) < 32 {
+		return errors.New("persisted service API credential is invalid")
+	}
+	token = string(savedToken)
 	minimum, e := strconv.ParseUint(env("SOURCEGRAPH_MIN_FREE_BYTES", "1073741824"), 10, 64)
 	if e != nil {
 		return errors.New("invalid free disk threshold")
@@ -146,7 +167,7 @@ func run() error {
 	if e != nil {
 		return e
 	}
-	bootstrap, e := sso.NewCommunity(ctx, root, origin, auth, settings)
+	bootstrap, e := sso.NewCommunity(ctx, root, origin, auth, settings, vault)
 	if e != nil {
 		return e
 	}
@@ -165,8 +186,13 @@ func run() error {
 	defer background.Stop()
 	c := config.Defaults()
 	c.DataRoot = root
-	handler := httpserver.RuntimeWithBootstrap(c, background.Logger, backend, os.Getenv("SOURCEGRAPH_API_TOKEN"), settings, bootstrap)
-	server := &http.Server{Addr: env("SOURCEGRAPH_LISTEN", "0.0.0.0:8080"), Handler: handler, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 120 * time.Second, WriteTimeout: 125 * time.Second, IdleTimeout: 60 * time.Second}
+	listen := env("SOURCEGRAPH_LISTEN", "0.0.0.0:8080")
+	c.ListenHost, _, e = net.SplitHostPort(listen)
+	if e != nil {
+		return errors.New("invalid listen address")
+	}
+	handler := httpserver.RuntimeWithBootstrap(c, background.Logger, backend, token, settings, bootstrap)
+	server := &http.Server{Addr: listen, Handler: handler, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 120 * time.Second, WriteTimeout: 125 * time.Second, IdleTimeout: 60 * time.Second}
 	done := make(chan error, 1)
 	go func() { done <- server.ListenAndServe() }()
 	slog.Info("listening", "address", server.Addr, "origin", origin, "database", driver, "revision", buildRevision)

docs/authentication.md

@@ -39,3 +39,8 @@ Use the Fly dashboard Secrets UI, or `fly secrets import --app context4ai-source
 The sign-in screen follows the site language and theme. With no configured provider it explains the missing Google/GitHub configuration and links to this setup guide; configured providers appear as sign-in buttons. A failed provider lookup offers retry without blocking the public browsing link. Login preserves the requested local Source Graph page. Sign-in must start on the configured site address because the provider always returns there and the login cookie belongs to one host. A login started on another address, such as the Fly hostname or an alias reached without the site's proxy, is first sent to the sign-in screen on the site address; the proxy forwards the original host in `X-Forwarded-Host`.
 
 Registration denial is a separate, server-rendered HTTP 403 page with the configured bilingual notice, a project link, and a five-second return to the validated previous page. This is distinct from missing OAuth configuration and provider lookup failures.
+
+
+## Persisted provider configuration
+
+Google/GitHub client IDs and secrets are imported together into encrypted database storage when first configured. Restarting or replacing the host uses the saved configuration even without the old environment values. Database records take precedence over bootstrap environment values. Keep the same public origin and provider callback allowlist when moving a deployment. See [application secrets](deployment.md#application-secrets) before credential rotation or rollback.

docs/code-sources.md

@@ -76,3 +76,6 @@ Settings apply in this order, later non-empty fields winning:
 Token routes need a browser session, and writes are CSRF-protected; API tokens cannot manage them. GET returns `configured`, `expires_at`, `revision`, `state` and `source` per source, never the token. PUT updates one source with `{kind, revision, token?, expires_at, clear?}`: an omitted `token` keeps the current one, while an omitted or null `expires_at` clears the expiry; a stale `revision` returns `REVISION_CONFLICT`.
 
 Tokens are stored encrypted with AES-256-GCM; see [deployment](deployment.md) for the encryption key and backups.
+
+
+If a token cannot be decrypted, its state is `unreadable`. Enter a replacement or clear it; editing metadata alone cannot repair it. Status never returns plaintext. The encryption key is imported from the old deployment on upgrade and then kept in the database; new instances reuse it.

docs/deployment.md

@@ -10,7 +10,7 @@ Each instance owns its Git mirrors, Zoekt indexes and data directory. The databa
 Implementation notes for sharding:
 
 - Use PostgreSQL or MongoDB; SQLite cannot be shared between instances.
-- Set the same `SOURCEGRAPH_CREDENTIAL_KEY` on every instance. Otherwise each generates its own key on its volume and cannot decrypt credentials stored by the others.
+- Application secrets and the encryption key are stored in the shared database. When upgrading existing instances, import the original credential key before allowing a replacement instance to initialize the database; do not replace it with a new key.
 - Keep the repository → instance mapping stable. Moving a repository to another instance means it is fetched and indexed again there.
 - Route by the repository named in the request: the `repo` query parameter for HTTP, and the `repo` argument of MCP tool calls.
 
@@ -34,21 +34,21 @@ GitHub Actions runs verification on PRs and main. A successful main build deploy
 
 SQLite is the default. WAL and full synchronous commits are enabled; application writes use short transactions. The SQL schema stores BSON-encoded persistence models, including fields intentionally hidden from public JSON, with namespace/ID primary keys. PostgreSQL uses the same persistence contract with serializable transactions. MongoDB uses its native document collections; authentication admission requires transactions, so use a replica set or Atlas.
 
-Changing `SOURCEGRAPH_DATABASE` does not migrate data. Start with an empty destination or perform a separately reviewed export/import. External databases do not replace the persistent Git/index/key volume. Sharded instances share one PostgreSQL or MongoDB database; each instance still needs its own volume.
+Changing `SOURCEGRAPH_DATABASE` does not migrate data. Start with an empty destination or perform a separately reviewed export/import. External databases do not replace the persistent Git/index volume. Sharded instances share one PostgreSQL or MongoDB database; each instance still needs its own volume.
 
 ## Backups and recovery
 
-Back up the database, encryption key, runtime storage configuration and repository metadata. Keep backups outside the Machine/volume, with restricted access. Fly volume snapshots are supplementary; test restoring the application state.
+Back up the entire database, including its secrets and credential records, runtime storage configuration and repository metadata. Keep backups outside the Machine/volume, with restricted access. Fly volume snapshots are supplementary; test restoring the application state.
 
 For SQLite use the SQLite online backup API / CLI `.backup`, or stop the service before copying the database and its WAL together. Never copy only a live `sourcegraph.db`. Example while the app is running:
 
 ```sh
 sqlite3 /data/sourcegraph.db '.backup /data/sourcegraph-backup.db'
 ```
 
-Copy the resulting backup and key/config files to protected external storage. Indexes and bare Git repositories can be rebuilt, but identities, settings, credentials and their encryption key cannot be reconstructed from Git. For PostgreSQL use `pg_dump`; for MongoDB use an appropriate replica-set/Atlas backup.
+Copy the resulting backup and runtime config files to protected external storage. Retain a legacy key file only for pre-upgrade recovery. Indexes and bare Git repositories can be rebuilt, but identities, settings, credentials and their encryption key cannot be reconstructed from Git. For PostgreSQL use `pg_dump`; for MongoDB use an appropriate replica-set/Atlas backup.
 
-To restore, stop the Machine, restore database and matching key/config, then start one Machine and verify administrator access, repository metadata and index recovery. Image rollback and database restoration are separate operations; retain a known-good image and take a backup before schema changes.
+To restore, stop the Machine, restore the complete database (including secrets) and runtime config; for pre-upgrade backups also restore the matching legacy key, then start one Machine and verify administrator access, repository metadata and index recovery. Image rollback and database restoration are separate operations; retain a known-good image and take a backup before schema changes.
 
 ## Task and query diagnostics
 
@@ -57,3 +57,16 @@ Terminal queue history retains its existing 24-hour/count limits. A separate `jo
 After durable task completion, structured `repository_task_completed` logs include repository/task identity, state and error code, never raw preparation errors. Unchanged successful automatic scans are omitted. Queue history and summaries describe historical events, not current readiness.
 
 Admin query logs include optional `batch` counters (`total`, `succeeded`, `failed`, `content_items`) independently of truncated request context. One log represents one request with failures. An empty File is successful; an item with both partial File and Error counts as failed with returned content. Unfinished items equal total minus succeeded and failed. Older records without counters remain unknown. Query-log retention is unchanged.
+
+
+## Application secrets
+
+Application secrets use AES-256-GCM, with the encryption key and encrypted values stored in the same database. This removes a separate host-key dependency; it does **not** protect against disclosure of the whole database. Restrict database access and protect backups.
+
+The first upgrade imports `SOURCEGRAPH_CREDENTIAL_KEY` if set, otherwise the existing private `code-credentials.key` file. Existing encrypted code-source tokens continue using that key. Fresh databases generate a random key without creating a key file. Subsequent starts read the database and ignore obsolete host key seeds. Never initialize a replacement host against an old credentials database without importing its original key first.
+
+The service API bearer and Google/GitHub OAuth client configuration are also persisted. Existing environment values seed missing records; existing database records take precedence. OAuth is left unconfigured when no provider is supplied. Keep environment secrets and the legacy key during the first rollout so the previous image remains usable on its original volume. After verified upgrade, replacement hosts need the database and non-secret runtime configuration, plus separate Git/index storage.
+
+SQL stores secrets in the `secrets` namespace of `sourcegraph_documents`; MongoDB uses `sourcegraph_secrets`. Back up those records together with credentials and identity data. Changing an environment seed does not rotate an existing database secret. Rotation requires an authorized maintenance change to the corresponding database secret and verification of the new provider/service credential; do not delete the encryption key to rotate a token. Personal/global Git tokens remain editable through the website.
+
+If a code token is already unreadable, the UI reports `unreadable` and allows replacing or clearing it with revision checks. Other users and repository records stay intact. An unreadable personal token does not block a usable global token.

docs/interfaces.md

@@ -105,3 +105,10 @@ Carry repository, original full SHA and path together. Do not reconstruct SHAs o
 `REVISION_NOT_ELIGIBLE` means the revision is absent from this repository's currently observed versions. Verify the repository/SHA pair, monitored branches, synchronization and retention before assuming expiry; never substitute another baseline.
 
 HTTP availability includes bounded historical `job_failures` summaries; see [diagnostic retention](deployment.md#task-and-query-diagnostics). MCP availability with a specific `job_id` returns `job_failure` and `job_lookup.state=archived_failure` when the live task was pruned but its failure summary remains. It is historical evidence, not current readiness or a runnable task. Missing history does not imply success.
+
+
+## Git timeout settings
+
+`repository_defaults.git_timeout_minutes` sets the global default (15 minutes initially); valid global values are 15, 30, 45 and 60. Existing settings without this field read as 15. An older settings client that omits the field preserves its saved value.
+
+Registration and repository update accept `git_timeout_minutes`: 0 means inherit the current global value, and 15/30/45/60 set an override. Omitted registration values inherit; omitted PATCH values preserve the current setting. Repository details return the stored value. Preview/CAS and registration idempotency include the timeout. Only new tasks use edits; changing a timeout preserves running tasks and published indexes. The setting covers Git fetching and hydration and can be changed from system settings, registration, or repository settings.

internal/appruntime/runtime.go

@@ -40,6 +40,11 @@ func Start(parent context.Context, backend Backend, settings Settings, logger *s
 	if source, ok := settings.(interface{ CodeSourceOverride() codesources.Override }); ok {
 		codesources.UseSettings(source.CodeSourceOverride)
 	}
+	if source, ok := settings.(interface{ GitTimeoutMinutes() int }); ok {
+		if target, ok := backend.(interface{ UseGitTimeoutDefault(func() int) }); ok {
+			target.UseGitTimeoutDefault(source.GitTimeoutMinutes)
+		}
+	}
 	start(func() { backend.PublishObservations(ctx) })
 	start(func() { settings.RunRefresh(ctx, logger) })
 	if !options.DisableRepositoryWorkers {