diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index a049cf1..082f7e7 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -31,7 +31,7 @@ CI runs the same verification on pull requests and `main`; see `.github/workflow | --- | --- | | `SOURCEGRAPH_DATABASE` | `sqlite`; also `postgres` or `mongodb` | | `DATABASE_URL` | SQLite absolute file path (default `/data/sourcegraph.db`), PostgreSQL DSN, or standard MongoDB URI | -| `SOURCEGRAPH_DATA_ROOT` | `/data`; persistent Git, indexes, manifests and encryption key | +| `SOURCEGRAPH_DATA_ROOT` | `/data`; persistent Git, indexes and manifests | | `SOURCEGRAPH_ORIGIN` | `http://localhost:8080`; public origin, without a path | | `SOURCEGRAPH_LISTEN` | `0.0.0.0:8080` | | `SOURCEGRAPH_MIN_FREE_BYTES` | `1073741824`; index/fetch free-space floor | @@ -39,10 +39,10 @@ CI runs the same verification on pull requests and `main`; see `.github/workflow | `SOURCEGRAPH_GITHUB_TOKEN` | Optional GitHub token, used until a global GitHub token is saved; the earlier name `SOURCEGRAPH_GIT_TOKEN` is still read when this is unset | | `SOURCEGRAPH_CUSTOM_GIT_TOKEN` | Optional token for the custom Git source, used until a global one is saved | | `SOURCEGRAPH_CONFIG_FILE` | Optional JSON file that overrides code sources at startup; see [code sources](docs/code-sources.md) | -| `SOURCEGRAPH_CREDENTIAL_KEY` | Optional base64 32-byte key for stored code credentials; otherwise `code-credentials.key` is generated in the data root. Required, and identical, on all instances sharing a database | -| `SOURCEGRAPH_API_TOKEN` | Optional service automation bearer credential, at least 32 characters | -| `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET` | Google web OAuth client | -| `GITHUB_CLIENT_ID`, `GITHUB_CLIENT_SECRET` | GitHub OAuth app | +| `SOURCEGRAPH_CREDENTIAL_KEY` | First-upgrade seed: base64 32-byte key for existing code credentials. Otherwise imports the existing `code-credentials.key`; a fresh database generates its own key. The database becomes authoritative | +| `SOURCEGRAPH_API_TOKEN` | First-start seed for the database-backed service bearer, at least 32 characters; generated if absent | +| `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET` | Google web OAuth client, imported together into the database on first configuration | +| `GITHUB_CLIENT_ID`, `GITHUB_CLIENT_SECRET` | GitHub OAuth app, imported together into the database on first configuration | Repository read plugins are tuned with `SOURCEGRAPH_PLUGINS_*`; see [repository plugins](docs/ref/repository-plugins.md). The CLI reads `SOURCEGRAPH_URL` and `SOURCEGRAPH_API_TOKEN`; see [interfaces](docs/interfaces.md#cli). @@ -54,3 +54,9 @@ Never put credentials in deployment JSON, source code or images. Public non-secr - [Interfaces](docs/interfaces.md) — HTTP routes, MCP and CLI - [Deployment](docs/deployment.md) — Fly.io, databases, backups and recovery - [Repository plugins](docs/ref/repository-plugins.md) — Wasm plugin contract + +## Runtime Git timeout + +System settings stores the global Git timeout in the database (15 minutes by default). Repositories inherit it by default or override it with 15, 30, 45 or 60 minutes. A new task snapshots the effective timeout; editing it does not interrupt running work or remove indexes. This covers Git fetch and object hydration, not total indexing time. + +See [deployment](docs/deployment.md#application-secrets) for credential migration and backup requirements. diff --git a/cmd/repo-service/main.go b/cmd/repo-service/main.go index a7ddc03..2ed381f 100644 --- a/cmd/repo-service/main.go +++ b/cmd/repo-service/main.go @@ -9,11 +9,13 @@ import ( "github.com/context4ai/sourcegraph/internal/config" "github.com/context4ai/sourcegraph/internal/control" "github.com/context4ai/sourcegraph/internal/httpserver" + "github.com/context4ai/sourcegraph/internal/secretstore" "github.com/context4ai/sourcegraph/internal/service" "github.com/context4ai/sourcegraph/internal/sitesettings" "github.com/context4ai/sourcegraph/internal/sqlstore" "github.com/context4ai/sourcegraph/internal/sso" "log/slog" + "net" "net/http" "os" "os/exec" @@ -63,6 +65,7 @@ func run() error { var repos service.Store var auth sso.Store var settingsDB sitesettings.Store + var openSecrets func() (*secretstore.Store, error) driver := env("SOURCEGRAPH_DATABASE", "sqlite") dsn := os.Getenv("DATABASE_URL") if driver == "mongodb" { @@ -74,6 +77,7 @@ func run() error { repos = db auth = &sso.MongoStore{DB: db.AuthDatabase()} settingsDB = &sitesettings.MongoStore{DB: db.AuthDatabase()} + openSecrets = func() (*secretstore.Store, error) { return secretstore.New(ctx, db.AuthDatabase()) } } else { if driver == "sqlite" && dsn == "" { dsn = filepath.Join(root, "sourcegraph.db") @@ -100,6 +104,7 @@ func run() error { repos = &control.SQL{DB: db} auth = &sso.SQLStore{DB: db} settingsDB = &sitesettings.SQLStore{DB: db} + openSecrets = func() (*secretstore.Store, error) { return secretstore.NewSQL(ctx, db) } } exe, e := os.Executable() if e != nil { @@ -111,6 +116,22 @@ func run() error { return e } defer backend.Close() + vault, e := openSecrets() + if e != nil { + return e + } + token := os.Getenv("SOURCEGRAPH_API_TOKEN") + if token != "" && len(token) < 32 { + return errors.New("service API credential must contain at least 32 characters") + } + savedToken, e := vault.Ensure(ctx, "service-api-token", []byte(token)) + if e != nil { + return e + } + if len(savedToken) < 32 { + return errors.New("persisted service API credential is invalid") + } + token = string(savedToken) minimum, e := strconv.ParseUint(env("SOURCEGRAPH_MIN_FREE_BYTES", "1073741824"), 10, 64) if e != nil { return errors.New("invalid free disk threshold") @@ -146,7 +167,7 @@ func run() error { if e != nil { return e } - bootstrap, e := sso.NewCommunity(ctx, root, origin, auth, settings) + bootstrap, e := sso.NewCommunity(ctx, root, origin, auth, settings, vault) if e != nil { return e } @@ -165,8 +186,13 @@ func run() error { defer background.Stop() c := config.Defaults() c.DataRoot = root - handler := httpserver.RuntimeWithBootstrap(c, background.Logger, backend, os.Getenv("SOURCEGRAPH_API_TOKEN"), settings, bootstrap) - server := &http.Server{Addr: env("SOURCEGRAPH_LISTEN", "0.0.0.0:8080"), Handler: handler, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 120 * time.Second, WriteTimeout: 125 * time.Second, IdleTimeout: 60 * time.Second} + listen := env("SOURCEGRAPH_LISTEN", "0.0.0.0:8080") + c.ListenHost, _, e = net.SplitHostPort(listen) + if e != nil { + return errors.New("invalid listen address") + } + handler := httpserver.RuntimeWithBootstrap(c, background.Logger, backend, token, settings, bootstrap) + server := &http.Server{Addr: listen, Handler: handler, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 120 * time.Second, WriteTimeout: 125 * time.Second, IdleTimeout: 60 * time.Second} done := make(chan error, 1) go func() { done <- server.ListenAndServe() }() slog.Info("listening", "address", server.Addr, "origin", origin, "database", driver, "revision", buildRevision) diff --git a/docs/authentication.md b/docs/authentication.md index 3a9f6bf..8fa3285 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -39,3 +39,8 @@ Use the Fly dashboard Secrets UI, or `fly secrets import --app context4ai-source The sign-in screen follows the site language and theme. With no configured provider it explains the missing Google/GitHub configuration and links to this setup guide; configured providers appear as sign-in buttons. A failed provider lookup offers retry without blocking the public browsing link. Login preserves the requested local Source Graph page. Sign-in must start on the configured site address because the provider always returns there and the login cookie belongs to one host. A login started on another address, such as the Fly hostname or an alias reached without the site's proxy, is first sent to the sign-in screen on the site address; the proxy forwards the original host in `X-Forwarded-Host`. Registration denial is a separate, server-rendered HTTP 403 page with the configured bilingual notice, a project link, and a five-second return to the validated previous page. This is distinct from missing OAuth configuration and provider lookup failures. + + +## Persisted provider configuration + +Google/GitHub client IDs and secrets are imported together into encrypted database storage when first configured. Restarting or replacing the host uses the saved configuration even without the old environment values. Database records take precedence over bootstrap environment values. Keep the same public origin and provider callback allowlist when moving a deployment. See [application secrets](deployment.md#application-secrets) before credential rotation or rollback. diff --git a/docs/code-sources.md b/docs/code-sources.md index 5adf269..b61f050 100644 --- a/docs/code-sources.md +++ b/docs/code-sources.md @@ -76,3 +76,6 @@ Settings apply in this order, later non-empty fields winning: Token routes need a browser session, and writes are CSRF-protected; API tokens cannot manage them. GET returns `configured`, `expires_at`, `revision`, `state` and `source` per source, never the token. PUT updates one source with `{kind, revision, token?, expires_at, clear?}`: an omitted `token` keeps the current one, while an omitted or null `expires_at` clears the expiry; a stale `revision` returns `REVISION_CONFLICT`. Tokens are stored encrypted with AES-256-GCM; see [deployment](deployment.md) for the encryption key and backups. + + +If a token cannot be decrypted, its state is `unreadable`. Enter a replacement or clear it; editing metadata alone cannot repair it. Status never returns plaintext. The encryption key is imported from the old deployment on upgrade and then kept in the database; new instances reuse it. diff --git a/docs/deployment.md b/docs/deployment.md index 38ed6f7..78e0bbc 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -10,7 +10,7 @@ Each instance owns its Git mirrors, Zoekt indexes and data directory. The databa Implementation notes for sharding: - Use PostgreSQL or MongoDB; SQLite cannot be shared between instances. -- Set the same `SOURCEGRAPH_CREDENTIAL_KEY` on every instance. Otherwise each generates its own key on its volume and cannot decrypt credentials stored by the others. +- Application secrets and the encryption key are stored in the shared database. When upgrading existing instances, import the original credential key before allowing a replacement instance to initialize the database; do not replace it with a new key. - Keep the repository → instance mapping stable. Moving a repository to another instance means it is fetched and indexed again there. - Route by the repository named in the request: the `repo` query parameter for HTTP, and the `repo` argument of MCP tool calls. @@ -34,11 +34,11 @@ GitHub Actions runs verification on PRs and main. A successful main build deploy SQLite is the default. WAL and full synchronous commits are enabled; application writes use short transactions. The SQL schema stores BSON-encoded persistence models, including fields intentionally hidden from public JSON, with namespace/ID primary keys. PostgreSQL uses the same persistence contract with serializable transactions. MongoDB uses its native document collections; authentication admission requires transactions, so use a replica set or Atlas. -Changing `SOURCEGRAPH_DATABASE` does not migrate data. Start with an empty destination or perform a separately reviewed export/import. External databases do not replace the persistent Git/index/key volume. Sharded instances share one PostgreSQL or MongoDB database; each instance still needs its own volume. +Changing `SOURCEGRAPH_DATABASE` does not migrate data. Start with an empty destination or perform a separately reviewed export/import. External databases do not replace the persistent Git/index volume. Sharded instances share one PostgreSQL or MongoDB database; each instance still needs its own volume. ## Backups and recovery -Back up the database, encryption key, runtime storage configuration and repository metadata. Keep backups outside the Machine/volume, with restricted access. Fly volume snapshots are supplementary; test restoring the application state. +Back up the entire database, including its secrets and credential records, runtime storage configuration and repository metadata. Keep backups outside the Machine/volume, with restricted access. Fly volume snapshots are supplementary; test restoring the application state. For SQLite use the SQLite online backup API / CLI `.backup`, or stop the service before copying the database and its WAL together. Never copy only a live `sourcegraph.db`. Example while the app is running: @@ -46,9 +46,9 @@ For SQLite use the SQLite online backup API / CLI `.backup`, or stop the service sqlite3 /data/sourcegraph.db '.backup /data/sourcegraph-backup.db' ``` -Copy the resulting backup and key/config files to protected external storage. Indexes and bare Git repositories can be rebuilt, but identities, settings, credentials and their encryption key cannot be reconstructed from Git. For PostgreSQL use `pg_dump`; for MongoDB use an appropriate replica-set/Atlas backup. +Copy the resulting backup and runtime config files to protected external storage. Retain a legacy key file only for pre-upgrade recovery. Indexes and bare Git repositories can be rebuilt, but identities, settings, credentials and their encryption key cannot be reconstructed from Git. For PostgreSQL use `pg_dump`; for MongoDB use an appropriate replica-set/Atlas backup. -To restore, stop the Machine, restore database and matching key/config, then start one Machine and verify administrator access, repository metadata and index recovery. Image rollback and database restoration are separate operations; retain a known-good image and take a backup before schema changes. +To restore, stop the Machine, restore the complete database (including secrets) and runtime config; for pre-upgrade backups also restore the matching legacy key, then start one Machine and verify administrator access, repository metadata and index recovery. Image rollback and database restoration are separate operations; retain a known-good image and take a backup before schema changes. ## Task and query diagnostics @@ -57,3 +57,16 @@ Terminal queue history retains its existing 24-hour/count limits. A separate `jo After durable task completion, structured `repository_task_completed` logs include repository/task identity, state and error code, never raw preparation errors. Unchanged successful automatic scans are omitted. Queue history and summaries describe historical events, not current readiness. Admin query logs include optional `batch` counters (`total`, `succeeded`, `failed`, `content_items`) independently of truncated request context. One log represents one request with failures. An empty File is successful; an item with both partial File and Error counts as failed with returned content. Unfinished items equal total minus succeeded and failed. Older records without counters remain unknown. Query-log retention is unchanged. + + +## Application secrets + +Application secrets use AES-256-GCM, with the encryption key and encrypted values stored in the same database. This removes a separate host-key dependency; it does **not** protect against disclosure of the whole database. Restrict database access and protect backups. + +The first upgrade imports `SOURCEGRAPH_CREDENTIAL_KEY` if set, otherwise the existing private `code-credentials.key` file. Existing encrypted code-source tokens continue using that key. Fresh databases generate a random key without creating a key file. Subsequent starts read the database and ignore obsolete host key seeds. Never initialize a replacement host against an old credentials database without importing its original key first. + +The service API bearer and Google/GitHub OAuth client configuration are also persisted. Existing environment values seed missing records; existing database records take precedence. OAuth is left unconfigured when no provider is supplied. Keep environment secrets and the legacy key during the first rollout so the previous image remains usable on its original volume. After verified upgrade, replacement hosts need the database and non-secret runtime configuration, plus separate Git/index storage. + +SQL stores secrets in the `secrets` namespace of `sourcegraph_documents`; MongoDB uses `sourcegraph_secrets`. Back up those records together with credentials and identity data. Changing an environment seed does not rotate an existing database secret. Rotation requires an authorized maintenance change to the corresponding database secret and verification of the new provider/service credential; do not delete the encryption key to rotate a token. Personal/global Git tokens remain editable through the website. + +If a code token is already unreadable, the UI reports `unreadable` and allows replacing or clearing it with revision checks. Other users and repository records stay intact. An unreadable personal token does not block a usable global token. diff --git a/docs/interfaces.md b/docs/interfaces.md index debc81a..d5f71dd 100644 --- a/docs/interfaces.md +++ b/docs/interfaces.md @@ -105,3 +105,10 @@ Carry repository, original full SHA and path together. Do not reconstruct SHAs o `REVISION_NOT_ELIGIBLE` means the revision is absent from this repository's currently observed versions. Verify the repository/SHA pair, monitored branches, synchronization and retention before assuming expiry; never substitute another baseline. HTTP availability includes bounded historical `job_failures` summaries; see [diagnostic retention](deployment.md#task-and-query-diagnostics). MCP availability with a specific `job_id` returns `job_failure` and `job_lookup.state=archived_failure` when the live task was pruned but its failure summary remains. It is historical evidence, not current readiness or a runnable task. Missing history does not imply success. + + +## Git timeout settings + +`repository_defaults.git_timeout_minutes` sets the global default (15 minutes initially); valid global values are 15, 30, 45 and 60. Existing settings without this field read as 15. An older settings client that omits the field preserves its saved value. + +Registration and repository update accept `git_timeout_minutes`: 0 means inherit the current global value, and 15/30/45/60 set an override. Omitted registration values inherit; omitted PATCH values preserve the current setting. Repository details return the stored value. Preview/CAS and registration idempotency include the timeout. Only new tasks use edits; changing a timeout preserves running tasks and published indexes. The setting covers Git fetching and hydration and can be changed from system settings, registration, or repository settings. diff --git a/internal/appruntime/runtime.go b/internal/appruntime/runtime.go index 1e7818f..5c8c7f6 100644 --- a/internal/appruntime/runtime.go +++ b/internal/appruntime/runtime.go @@ -40,6 +40,11 @@ func Start(parent context.Context, backend Backend, settings Settings, logger *s if source, ok := settings.(interface{ CodeSourceOverride() codesources.Override }); ok { codesources.UseSettings(source.CodeSourceOverride) } + if source, ok := settings.(interface{ GitTimeoutMinutes() int }); ok { + if target, ok := backend.(interface{ UseGitTimeoutDefault(func() int) }); ok { + target.UseGitTimeoutDefault(source.GitTimeoutMinutes) + } + } start(func() { backend.PublishObservations(ctx) }) start(func() { settings.RunRefresh(ctx, logger) }) if !options.DisableRepositoryWorkers { diff --git a/internal/codecredentials/manager.go b/internal/codecredentials/manager.go index 5b2d6dc..7a0d750 100644 --- a/internal/codecredentials/manager.go +++ b/internal/codecredentials/manager.go @@ -22,6 +22,8 @@ import ( const Global = "global" +var unreadable = contract.Fail("CREDENTIAL_RECONFIGURATION_REQUIRED", "Credential cannot be decrypted; enter a new token.", 409) + type Record struct { ID string `bson:"_id"` Revision int64 `bson:"revision"` @@ -61,7 +63,13 @@ type Manager struct { func New(root string, store Store) (*Manager, error) { var key []byte var err error - if raw := os.Getenv("SOURCEGRAPH_CREDENTIAL_KEY"); raw != "" { + if persistent, ok := store.(interface { + EncryptionKey(context.Context, func() ([]byte, error)) ([]byte, error) + }); ok { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + key, err = persistent.EncryptionKey(ctx, func() ([]byte, error) { return legacyKey(root) }) + } else if raw := os.Getenv("SOURCEGRAPH_CREDENTIAL_KEY"); raw != "" { key, err = base64.StdEncoding.DecodeString(raw) } else { path := filepath.Join(root, "code-credentials.key") @@ -139,11 +147,11 @@ func (m *Manager) load(ctx context.Context, owner, kind string) (Record, string, } n := m.aead.NonceSize() if len(r.Ciphertext) < n { - return r, "", unavailable + return r, "", unreadable } plain, err := m.aead.Open(nil, r.Ciphertext[:n], r.Ciphertext[n:], []byte(id)) if err != nil { - return r, "", unavailable + return r, "", unreadable } return r, string(plain), nil } @@ -172,6 +180,10 @@ func (m *Manager) Status(ctx context.Context, owner string) (map[string]Status, out := map[string]Status{} for _, kind := range codesources.Kinds() { r, t, err := m.load(ctx, owner, kind) + if errors.Is(err, unreadable) { + out[kind] = Status{Revision: r.Revision, Configured: true, Expires: r.Expires, State: "unreadable", Source: "saved", Retired: codesources.IsAlias(kind)} + continue + } if err != nil { return nil, err } @@ -188,7 +200,7 @@ func (m *Manager) Status(ctx context.Context, owner string) (map[string]Status, } func (m *Manager) Save(ctx context.Context, owner, kind string, in Input) (Status, error) { r, token, err := m.load(ctx, owner, kind) - if err != nil { + if err != nil && !(errors.Is(err, unreadable) && (in.Token != nil || in.Clear)) { return Status{}, err } if r.Revision != in.Revision { @@ -245,6 +257,9 @@ func (m *Manager) Candidates(ctx context.Context, owner string, provider ...stri for _, who := range owners { for _, kind := range []string{kind} { r, t, err := m.load(ctx, who, kind) + if errors.Is(err, unreadable) { + continue + } if err != nil { return nil, err } @@ -302,7 +317,7 @@ func (m *Manager) Warnings(ctx context.Context) []Warning { if !ok { continue } - if v.State == "expired" || v.State == "invalid" || v.State == "expiring" { + if v.State == "unreadable" || v.State == "expired" || v.State == "invalid" || v.State == "expiring" { out = append(out, Warning{kind, v.State, v.Expires}) } } @@ -310,3 +325,23 @@ func (m *Manager) Warnings(ctx context.Context) []Warning { } func (m *Manager) Reset(ctx context.Context) error { return m.store.Clear(ctx) } + +// legacyKey is only called when the database has no key yet. Existing databases +// remain authoritative even if an old host file or environment value changes. +func legacyKey(root string) ([]byte, error) { + if raw := os.Getenv("SOURCEGRAPH_CREDENTIAL_KEY"); raw != "" { + return base64.StdEncoding.DecodeString(raw) + } + path := filepath.Join(root, "code-credentials.key") + info, e := os.Lstat(path) + if os.IsNotExist(e) { + return nil, nil + } + if e != nil { + return nil, e + } + if !info.Mode().IsRegular() || info.Mode().Perm()&0077 != 0 { + return nil, errors.New("unsafe code credential key permissions") + } + return os.ReadFile(path) +} diff --git a/internal/codecredentials/manager_test.go b/internal/codecredentials/manager_test.go index 708228b..2d70e0c 100644 --- a/internal/codecredentials/manager_test.go +++ b/internal/codecredentials/manager_test.go @@ -68,3 +68,64 @@ func TestEnvironmentTokenFollowsSourceVariable(t *testing.T) { t.Fatal("custom git must not reuse the GitHub variables") } } + +func TestLostKeyCanBeReplacedWithoutResettingRecords(t *testing.T) { + ctx := context.Background() + store := &FileStore{Path: filepath.Join(t.TempDir(), "credentials.json")} + old, e := New(t.TempDir(), store) + if e != nil { + t.Fatal(e) + } + token := "old-secret-value" + saved, e := old.Save(ctx, Global, "github", Input{Token: &token}) + if e != nil { + t.Fatal(e) + } + fresh, e := New(t.TempDir(), store) + if e != nil { + t.Fatal(e) + } + states, e := fresh.Status(ctx, Global) + if e != nil || states["github"].State != "unreadable" || states["github"].Revision != saved.Revision { + t.Fatalf("status %#v: %v", states, e) + } + if _, e = fresh.Save(ctx, Global, "github", Input{Revision: saved.Revision}); !errors.Is(e, unreadable) { + t.Fatalf("metadata-only update must not discard unreadable secret: %v", e) + } + replacement := "replacement-secret-value" + if _, e = fresh.Save(ctx, Global, "github", Input{Revision: saved.Revision - 1, Token: &replacement}); !errors.Is(e, conflict) { + t.Fatalf("CAS bypassed: %v", e) + } + if _, e = fresh.Save(ctx, Global, "github", Input{Revision: saved.Revision, Token: &replacement}); e != nil { + t.Fatal(e) + } + got, e := fresh.Candidates(ctx, "", "github") + if e != nil || len(got) < 1 || got[0].Token != replacement { + t.Fatal("replacement unusable", e) + } +} + +func TestUnreadablePersonalCredentialDoesNotBlockGlobal(t *testing.T) { + ctx := context.Background() + store := &FileStore{Path: filepath.Join(t.TempDir(), "credentials.json")} + old, e := New(t.TempDir(), store) + if e != nil { + t.Fatal(e) + } + token := "personal-old-token" + if _, e = old.Save(ctx, "user-1", "github", Input{Token: &token}); e != nil { + t.Fatal(e) + } + fresh, e := New(t.TempDir(), store) + if e != nil { + t.Fatal(e) + } + token = "new-global-token" + if _, e = fresh.Save(ctx, Global, "github", Input{Token: &token}); e != nil { + t.Fatal(e) + } + got, e := fresh.Candidates(ctx, "user-1", "github") + if e != nil || len(got) < 1 || got[0].Token != token { + t.Fatal("unreadable personal credential blocked fallback", e) + } +} diff --git a/internal/codecredentials/persistence_test.go b/internal/codecredentials/persistence_test.go new file mode 100644 index 0000000..26d18d4 --- /dev/null +++ b/internal/codecredentials/persistence_test.go @@ -0,0 +1,60 @@ +package codecredentials + +import ( + "context" + "github.com/context4ai/sourcegraph/internal/sqlstore" + "os" + "path/filepath" + "testing" +) + +func TestDatabaseImportsLegacyKeyAndSurvivesHostReplacement(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + legacy := &FileStore{Path: filepath.Join(root, "credentials.json")} + old, e := New(root, legacy) + if e != nil { + t.Fatal(e) + } + token := "existing-private-token" + if _, e = old.Save(ctx, Global, "github", Input{Token: &token}); e != nil { + t.Fatal(e) + } + row, e := legacy.Get(ctx, recordID(Global, "github")) + if e != nil { + t.Fatal(e) + } + db, e := sqlstore.Open(ctx, "sqlite", filepath.Join(t.TempDir(), "db")) + if e != nil { + t.Fatal(e) + } + defer db.Close() + store := SQLStore{DB: db} + if e = store.Save(ctx, row, 0); e != nil { + t.Fatal(e) + } + migrated, e := New(root, store) + if e != nil { + t.Fatal(e) + } + got, e := migrated.Candidates(ctx, "", "github") + if e != nil || len(got) < 1 || got[0].Token != token { + t.Fatal("legacy token lost", e) + } + if e = os.Remove(filepath.Join(root, "code-credentials.key")); e != nil { + t.Fatal(e) + } + t.Setenv("SOURCEGRAPH_CREDENTIAL_KEY", "stale-invalid-bootstrap-value") + freshRoot := t.TempDir() + fresh, e := New(freshRoot, store) + if e != nil { + t.Fatal(e) + } + got, e = fresh.Candidates(ctx, "", "github") + if e != nil || len(got) < 1 || got[0].Token != token { + t.Fatal("replacement host token lost", e) + } + if _, e = os.Stat(filepath.Join(freshRoot, "code-credentials.key")); !os.IsNotExist(e) { + t.Fatal("new local key created") + } +} diff --git a/internal/codecredentials/sql.go b/internal/codecredentials/sql.go index 5a840f2..263b125 100644 --- a/internal/codecredentials/sql.go +++ b/internal/codecredentials/sql.go @@ -3,6 +3,7 @@ package codecredentials import ( "context" "errors" + "github.com/context4ai/sourcegraph/internal/secretstore" "github.com/context4ai/sourcegraph/internal/sqlstore" ) @@ -32,3 +33,11 @@ func (s SQLStore) Save(ctx context.Context, r Record, expected int64) error { func (s SQLStore) Clear(ctx context.Context) error { return s.DB.Write(ctx, func(t *sqlstore.Tx) error { return t.Clear("credentials") }) } + +func (s SQLStore) EncryptionKey(ctx context.Context, seed func() ([]byte, error)) ([]byte, error) { + v, e := secretstore.NewSQL(ctx, s.DB, secretstore.KeySeed(seed)) + if e != nil { + return nil, e + } + return v.Key(), nil +} diff --git a/internal/codecredentials/store.go b/internal/codecredentials/store.go index e98d158..f35ca3b 100644 --- a/internal/codecredentials/store.go +++ b/internal/codecredentials/store.go @@ -6,6 +6,7 @@ import ( "context" "encoding/json" "errors" + "github.com/context4ai/sourcegraph/internal/secretstore" "os" "path/filepath" "sync" @@ -142,3 +143,11 @@ func (s *FileStore) Clear(context.Context) error { } return nil } + +func (s MongoStore) EncryptionKey(ctx context.Context, seed func() ([]byte, error)) ([]byte, error) { + v, e := secretstore.New(ctx, s.Collection.Database(), secretstore.KeySeed(seed)) + if e != nil { + return nil, e + } + return v.Key(), nil +} diff --git a/internal/control/git_timeout.go b/internal/control/git_timeout.go new file mode 100644 index 0000000..7353916 --- /dev/null +++ b/internal/control/git_timeout.go @@ -0,0 +1,12 @@ +package control + +import "github.com/context4ai/sourcegraph/internal/contract" + +// Zero is persisted inheritance; old repository records therefore inherit. +func ValidateGitTimeout(minutes int) error { + switch minutes { + case 0, 15, 30, 45, 60: + return nil + } + return contract.Fail("INVALID_GIT_TIMEOUT", "Use 0 to inherit, or 15, 30, 45, 60 minutes.", 400) +} diff --git a/internal/control/model.go b/internal/control/model.go index 2f12331..b3f20aa 100644 --- a/internal/control/model.go +++ b/internal/control/model.go @@ -131,6 +131,7 @@ type Repository struct { Scopes map[string]ScopeState `json:"scopes,omitempty" bson:"scopes,omitempty"` LastIndexedAt time.Time `json:"last_indexed_at,omitempty,omitzero" bson:"last_indexed_at,omitempty"` LastIndexDurationSeconds float64 `json:"last_index_duration_seconds,omitempty" bson:"last_index_duration_seconds,omitempty"` + GitTimeoutMinutes int `json:"git_timeout_minutes" bson:"git_timeout_minutes"` SyncIntervalMinutes int `json:"sync_interval_minutes" bson:"sync_interval_minutes"` // IndexGenerated is false by default: minified JS/CSS and source maps are // left out of the text index from the next build on. diff --git a/internal/gitstore/fetch.go b/internal/gitstore/fetch.go index 2f93620..f922d8e 100644 --- a/internal/gitstore/fetch.go +++ b/internal/gitstore/fetch.go @@ -68,7 +68,7 @@ func (s *Store) fetch(ctx context.Context, repo string, o FetchOptions, selected if !strings.HasPrefix(o.Host, "https://") { return contract.Fail("CODE_SOURCE_NOT_CONFIGURED", "The repository's code source has no host.", 503) } - if o.Timeout <= 0 || o.Timeout > 30*time.Minute { + if o.Timeout <= 0 || o.Timeout > 60*time.Minute { o.Timeout = 15 * time.Minute } ctx, cancel := context.WithTimeout(ctx, o.Timeout) diff --git a/internal/gitstore/partial.go b/internal/gitstore/partial.go index 22319dd..d6c1cb8 100644 --- a/internal/gitstore/partial.go +++ b/internal/gitstore/partial.go @@ -71,7 +71,7 @@ func (s *Store) HydratePaths(ctx context.Context, repo, commit string, paths []s if !filepath.IsAbs(o.CredentialHelper) || strings.ContainsAny(o.CredentialHelper, " \t\r\n\"'\\") || (o.Token == "" && !o.Anonymous) { return contract.Fail("GIT_CREDENTIAL_UNAVAILABLE", "Git credential is not configured.", 503) } - if o.Timeout <= 0 || o.Timeout > 30*time.Minute { + if o.Timeout <= 0 || o.Timeout > 60*time.Minute { o.Timeout = 15 * time.Minute } ctx, cancel := context.WithTimeout(ctx, o.Timeout) @@ -94,7 +94,7 @@ func (s *Store) HydrateEntries(ctx context.Context, repo, commit string, entries if !strings.HasPrefix(o.Host, "https://") { return contract.Fail("CODE_SOURCE_NOT_CONFIGURED", "The repository's code source has no host.", 503) } - if o.Timeout <= 0 || o.Timeout > 30*time.Minute { + if o.Timeout <= 0 || o.Timeout > 60*time.Minute { o.Timeout = 15 * time.Minute } ctx, cancel := context.WithTimeout(ctx, o.Timeout) diff --git a/internal/httpserver/registration.go b/internal/httpserver/registration.go index 3202853..dee7098 100644 --- a/internal/httpserver/registration.go +++ b/internal/httpserver/registration.go @@ -18,6 +18,7 @@ type registrationRequest struct { Days *int `json:"days"` Count *int `json:"count"` } `json:"policy"` + GitTimeoutMinutes int `json:"git_timeout_minutes"` SyncIntervalMinutes *int `json:"sync_interval_minutes"` } diff --git a/internal/httpserver/runtime.go b/internal/httpserver/runtime.go index b7bb617..72b08a4 100644 --- a/internal/httpserver/runtime.go +++ b/internal/httpserver/runtime.go @@ -39,6 +39,7 @@ func RuntimeWithBootstrap(c config.Config, log *slog.Logger, s *service.Service, func runtime(c config.Config, log *slog.Logger, s *service.Service, token string, settings *sitesettings.Service, bootstrap *sso.Bootstrap, auth ...*sso.Handler) http.Handler { fallback := New(c, log) limits := &websiteLimits{} + mcpserver.ConfigureListener(c.ListenHost) mcpHandler := mcpserver.New(s) // Discovery routing filters tools; keep this independent invocation guard // as well. CanPrepare only reads the already-authenticated request context. @@ -328,7 +329,7 @@ func runtime(c config.Config, log *slog.Logger, s *service.Service, token string if q.SyncIntervalMinutes != nil { interval = *q.SyncIntervalMinutes } - result, err = s.RegisterWithProvider(r.Context(), q.Name, policy, interval, q.IndexMode, q.PathGroups, q.PublicRead, q.CodeProvider, r.Header.Get("Idempotency-Key")) + result, err = s.RegisterWithGitTimeout(r.Context(), q.Name, policy, interval, q.IndexMode, q.PathGroups, q.PublicRead, q.CodeProvider, q.GitTimeoutMinutes, r.Header.Get("Idempotency-Key")) } status = 202 } diff --git a/internal/mcpserver/listener_test.go b/internal/mcpserver/listener_test.go new file mode 100644 index 0000000..ad706ca --- /dev/null +++ b/internal/mcpserver/listener_test.go @@ -0,0 +1,46 @@ +package mcpserver + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestConfigureListener(t *testing.T) { + defer func() { networkListener = false }() + for host, want := range map[string]bool{"::": true, "0.0.0.0": true, "10.1.2.3": true, "127.0.0.1": false, "::1": false, "localhost": false} { + ConfigureListener(host) + if networkListener != want { + t.Errorf("ConfigureListener(%q) = %v, want %v", host, networkListener, want) + } + } +} + +func TestLoopbackRequestWithPublicHost(t *testing.T) { + defer func() { networkListener = false }() + for _, exposed := range []bool{false, true} { + if exposed { + ConfigureListener("::") + } else { + ConfigureListener("127.0.0.1") + } + srv := httptest.NewServer(New(nil)) + req, _ := http.NewRequest("POST", srv.URL, strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"tools/list"}`)) + req.Host = "context.example" + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json, text/event-stream") + resp, err := http.DefaultClient.Do(req) + srv.Close() + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if exposed && resp.StatusCode == http.StatusForbidden { + t.Fatalf("network listener must accept proxied requests with a public Host") + } + if !exposed && resp.StatusCode != http.StatusForbidden { + t.Fatalf("loopback-only listener must keep DNS rebinding protection, got %d", resp.StatusCode) + } + } +} diff --git a/internal/mcpserver/server.go b/internal/mcpserver/server.go index f3cfcaa..827e1ad 100644 --- a/internal/mcpserver/server.go +++ b/internal/mcpserver/server.go @@ -8,9 +8,23 @@ import ( "github.com/context4ai/sourcegraph/internal/gitstore" "github.com/context4ai/sourcegraph/internal/service" "github.com/modelcontextprotocol/go-sdk/mcp" + "net" "net/http" ) +// networkListener records whether the service listens beyond loopback. The +// SDK's DNS-rebinding protection rejects requests that arrive on a loopback +// address with a public Host; that guards localhost-only servers, but behind +// a reverse proxy forwarding to loopback every request +// arrives that way. A network-exposed listener gains nothing from the check. +var networkListener bool + +// ConfigureListener must be called before handlers are created. +func ConfigureListener(listenHost string) { + ip := net.ParseIP(listenHost) + networkListener = !(listenHost == "localhost" || (ip != nil && ip.IsLoopback())) +} + func New(s *service.Service) http.Handler { return newHandler(s, nil, true) } @@ -79,7 +93,7 @@ func newHandler(s *service.Service, authorize func(context.Context) error, inclu return s.PrepareRevision(ctx, in.Repo, in.Revision, in.Group, in.Key) }, includePrepare) } - return pluginConnectionHandler(mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server { return server }, &mcp.StreamableHTTPOptions{Stateless: true, JSONResponse: true})) + return pluginConnectionHandler(mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server { return server }, &mcp.StreamableHTTPOptions{Stateless: true, JSONResponse: true, DisableLocalhostProtection: networkListener})) } // register keeps the shared service result/error contract while exposing only diff --git a/internal/secretstore/sql.go b/internal/secretstore/sql.go new file mode 100644 index 0000000..21c3e7f --- /dev/null +++ b/internal/secretstore/sql.go @@ -0,0 +1,44 @@ +package secretstore + +import ( + "context" + "errors" + "github.com/context4ai/sourcegraph/internal/sqlstore" +) + +type sqlBackend struct{ db *sqlstore.DB } + +func NewSQL(ctx context.Context, db *sqlstore.DB, seed ...KeySeed) (*Store, error) { + return open(ctx, sqlBackend{db}, seed...) +} +func (s sqlBackend) get(ctx context.Context, id string) ([]byte, error) { + var r record + e := s.db.Get(ctx, "secrets", id, &r) + if errors.Is(e, sqlstore.Missing) { + return nil, ErrMissing + } + if e != nil { + return nil, unavailable + } + return r.Data, nil +} +func (s sqlBackend) insert(ctx context.Context, id string, data []byte) error { + e := s.db.Write(ctx, func(t *sqlstore.Tx) error { + var r record + e := t.Get("secrets", id, &r) + if e == nil { + return ErrExists + } + if !errors.Is(e, sqlstore.Missing) { + return e + } + return t.Put("secrets", id, record{ID: id, Data: data}) + }) + if errors.Is(e, ErrExists) { + return ErrExists + } + if e != nil { + return unavailable + } + return nil +} diff --git a/internal/secretstore/store.go b/internal/secretstore/store.go new file mode 100644 index 0000000..abdfbfc --- /dev/null +++ b/internal/secretstore/store.go @@ -0,0 +1,148 @@ +// Package secretstore persists application secrets and their AES-GCM key in +// the control database. It removes host-disk dependencies; it is not a defense +// against an attacker who can read the whole database. +package secretstore + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "errors" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +var ErrMissing = errors.New("secret not configured") +var ErrExists = errors.New("secret already configured") +var unavailable = errors.New("persistent secret storage unavailable") + +type record struct { + ID string `bson:"_id"` + Data []byte `bson:"data"` +} +type backend interface { + get(context.Context, string) ([]byte, error) + insert(context.Context, string, []byte) error +} +type mongoBackend struct{ c *mongo.Collection } + +func (m mongoBackend) get(ctx context.Context, id string) ([]byte, error) { + var r record + e := m.c.FindOne(ctx, bson.M{"_id": id}).Decode(&r) + if errors.Is(e, mongo.ErrNoDocuments) { + return nil, ErrMissing + } + if e != nil { + return nil, unavailable + } + return r.Data, nil +} +func (m mongoBackend) insert(ctx context.Context, id string, data []byte) error { + _, e := m.c.InsertOne(ctx, record{id, data}) + if mongo.IsDuplicateKeyError(e) { + return ErrExists + } + if e != nil { + return unavailable + } + return nil +} + +type Store struct { + backend backend + key []byte + aead cipher.AEAD +} + +type KeySeed func() ([]byte, error) + +func New(ctx context.Context, db *mongo.Database, seed ...KeySeed) (*Store, error) { + return open(ctx, mongoBackend{db.Collection("sourcegraph_secrets")}, seed...) +} +func open(ctx context.Context, b backend, seed ...KeySeed) (*Store, error) { + key, e := b.get(ctx, "encryption-key-v1") + if errors.Is(e, ErrMissing) { + if len(seed) > 0 && seed[0] != nil { + key, e = seed[0]() + if e != nil { + return nil, unavailable + } + } + if len(key) == 0 { + key = make([]byte, 32) + if _, e = rand.Read(key); e != nil { + return nil, unavailable + } + } + if len(key) != 32 { + return nil, unavailable + } + if e = b.insert(ctx, "encryption-key-v1", key); e != nil && !errors.Is(e, ErrExists) { + return nil, e + } + key, e = b.get(ctx, "encryption-key-v1") + } + if e != nil { + return nil, e + } + if len(key) != 32 { + return nil, unavailable + } + block, e := aes.NewCipher(key) + if e != nil { + return nil, unavailable + } + aead, e := cipher.NewGCM(block) + if e != nil { + return nil, unavailable + } + return &Store{b, key, aead}, nil +} +func (s *Store) Key() []byte { return append([]byte(nil), s.key...) } +func (s *Store) Get(ctx context.Context, id string) ([]byte, error) { + data, e := s.backend.get(ctx, "value:"+id) + if e != nil { + return nil, e + } + n := s.aead.NonceSize() + if len(data) < n { + return nil, unavailable + } + plain, e := s.aead.Open(nil, data[:n], data[n:], []byte(id)) + if e != nil { + return nil, unavailable + } + return plain, nil +} + +// Insert never replaces a winner from another process. +func (s *Store) Insert(ctx context.Context, id string, plain []byte) error { + nonce := make([]byte, s.aead.NonceSize()) + if _, e := rand.Read(nonce); e != nil { + return unavailable + } + return s.backend.insert(ctx, "value:"+id, s.aead.Seal(nonce, nonce, plain, []byte(id))) +} + +// Ensure returns the already persisted value, importing the supplied value only +// for a missing record. An empty value creates a random service credential. +func (s *Store) Ensure(ctx context.Context, id string, seed []byte) ([]byte, error) { + v, e := s.Get(ctx, id) + if !errors.Is(e, ErrMissing) { + return v, e + } + if len(seed) == 0 { + seed = make([]byte, 48) + if _, e = rand.Read(seed); e != nil { + return nil, unavailable + } + seed = []byte(base64.RawURLEncoding.EncodeToString(seed)) + } + if e = s.Insert(ctx, id, seed); e != nil && !errors.Is(e, ErrExists) { + return nil, e + } + return s.Get(ctx, id) +} diff --git a/internal/secretstore/store_test.go b/internal/secretstore/store_test.go new file mode 100644 index 0000000..f2a3c5c --- /dev/null +++ b/internal/secretstore/store_test.go @@ -0,0 +1,80 @@ +package secretstore + +import ( + "bytes" + "context" + "sync" + "testing" +) + +type memory struct { + mu sync.Mutex + rows map[string][]byte +} + +func (m *memory) get(_ context.Context, id string) ([]byte, error) { + m.mu.Lock() + defer m.mu.Unlock() + b, ok := m.rows[id] + if !ok { + return nil, ErrMissing + } + return append([]byte(nil), b...), nil +} +func (m *memory) insert(_ context.Context, id string, b []byte) error { + m.mu.Lock() + defer m.mu.Unlock() + if _, ok := m.rows[id]; ok { + return ErrExists + } + m.rows[id] = append([]byte(nil), b...) + return nil +} +func TestConcurrentInitializationAndHostReplacement(t *testing.T) { + ctx := context.Background() + m := &memory{rows: map[string][]byte{}} + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { + defer wg.Done() + s, e := open(ctx, m) + if e != nil { + t.Error(e) + return + } + if _, e = s.Ensure(ctx, "api", []byte("existing-service-token")); e != nil { + t.Error(e) + } + }() + } + wg.Wait() + fresh, e := open(ctx, m) + if e != nil { + t.Fatal(e) + } + got, e := fresh.Get(ctx, "api") + if e != nil || string(got) != "existing-service-token" { + t.Fatal("credential did not survive replacement", e) + } + if bytes.Contains(m.rows["value:api"], got) { + t.Fatal("plaintext persisted") + } + if _, e = fresh.Ensure(ctx, "api", []byte("must-not-overwrite")); e != nil { + t.Fatal(e) + } + got, _ = fresh.Get(ctx, "api") + if string(got) != "existing-service-token" { + t.Fatal("overwrote credential") + } + m.rows["value:other"] = m.rows["value:api"] + if _, e = fresh.Get(ctx, "other"); e == nil { + t.Fatal("AAD did not bind record ID") + } +} +func TestCorruptKeyFailsClosed(t *testing.T) { + m := &memory{rows: map[string][]byte{"encryption-key-v1": {1}}} + if _, e := open(context.Background(), m); e == nil { + t.Fatal("accepted corrupt key") + } +} diff --git a/internal/service/code_credentials.go b/internal/service/code_credentials.go index 15d039a..76b6883 100644 --- a/internal/service/code_credentials.go +++ b/internal/service/code_credentials.go @@ -54,5 +54,5 @@ func (s *Service) fetchOptions(r control.Repository, token string) gitstore.Fetc if host == "" { host = source.Host } - return gitstore.FetchOptions{CredentialHelper: s.Helper, Token: token, Host: host, Username: source.Username, Anonymous: source.Anonymous, Branches: r.Policy.Branches, MinFreeBytes: s.MinFree} + return gitstore.FetchOptions{Timeout: time.Duration(s.gitTimeoutMinutes(r)) * time.Minute, CredentialHelper: s.Helper, Token: token, Host: host, Username: source.Username, Anonymous: source.Anonymous, Branches: r.Policy.Branches, MinFreeBytes: s.MinFree} } diff --git a/internal/service/git_timeout.go b/internal/service/git_timeout.go new file mode 100644 index 0000000..ae2dfa9 --- /dev/null +++ b/internal/service/git_timeout.go @@ -0,0 +1,32 @@ +package service + +import ( + "github.com/context4ai/sourcegraph/internal/control" + "time" +) + +// UseGitTimeoutDefault is wired before background workers and HTTP start. +func (s *Service) UseGitTimeoutDefault(read func() int) { s.gitTimeoutDefault = read } + +func (s *Service) gitTimeoutMinutes(r control.Repository) int { + if r.GitTimeoutMinutes != 0 && control.ValidateGitTimeout(r.GitTimeoutMinutes) == nil { + return r.GitTimeoutMinutes + } + if s.gitTimeoutDefault != nil { + if n := s.gitTimeoutDefault(); n != 0 && control.ValidateGitTimeout(n) == nil { + return n + } + } + return 15 +} + +func gitJobBudget(minutes int, kind string) time.Duration { + budget := time.Duration(minutes) * time.Minute + if budget < 45*time.Minute { + budget = 45 * time.Minute + } + if kind == "prepare" { + budget *= 16 + } + return budget +} diff --git a/internal/service/git_timeout_test.go b/internal/service/git_timeout_test.go new file mode 100644 index 0000000..0be041c --- /dev/null +++ b/internal/service/git_timeout_test.go @@ -0,0 +1,141 @@ +package service + +import ( + "context" + "errors" + "fmt" + "github.com/context4ai/sourcegraph/internal/control" + "testing" + "time" +) + +func TestGitTimeoutInheritanceAndTaskSnapshot(t *testing.T) { + s := &Service{} + r := control.Repository{} + if s.fetchOptions(r, "").Timeout != 15*time.Minute { + t.Fatal("default timeout") + } + global := 30 + s.UseGitTimeoutDefault(func() int { return global }) + if s.fetchOptions(r, "").Timeout != 30*time.Minute { + t.Fatal("inherit") + } + // Worker captures the resolved value in its private repository copy. + captured := r + captured.GitTimeoutMinutes = s.gitTimeoutMinutes(r) + global = 60 + if s.fetchOptions(captured, "").Timeout != 30*time.Minute || s.fetchOptions(r, "").Timeout != 60*time.Minute { + t.Fatal("task snapshot or live inheritance") + } + for _, n := range []int{15, 30, 45, 60} { + r.GitTimeoutMinutes = n + if s.fetchOptions(r, "").Timeout != time.Duration(n)*time.Minute { + t.Fatalf("override %d", n) + } + if gitJobBudget(n, "sync") < s.fetchOptions(r, "").Timeout { + t.Fatalf("outer budget clips %d", n) + } + } +} + +func TestGitTimeoutUpdatePreservesWorkAndVersions(t *testing.T) { + policy := control.Policy{Branches: []string{"main"}, Count: 10} + db := &managementStore{repo: control.Repository{Name: "org/repo", Enabled: true, Revision: 4, PolicyRevision: 2, Policy: policy, Versions: []control.Version{{Commit: "1111111111111111111111111111111111111111", Generation: 3}}, Jobs: []control.Job{{State: "running", Kind: "sync"}}}} + s := &Service{DB: db, Root: t.TempDir()} + for _, n := range []int{30, 60, 0} { + u := Update{Policy: policy, Enabled: true, GitTimeoutMinutes: &n} + u.Preview = preview(db.repo, u) + rev := db.repo.Revision + _, err := s.Update(context.Background(), db.repo.Name, u, rev) + if err != nil { + t.Fatal(err) + } + if db.repo.GitTimeoutMinutes != n || db.repo.PolicyRevision != 2 || len(db.repo.Versions) != 1 || db.repo.Jobs[0].State != "running" { + t.Fatal("timeout revoked data or work") + } + if _, err = s.Update(context.Background(), db.repo.Name, u, rev); err == nil { + t.Fatal("stale CAS accepted") + } + } + n := 60 + u := Update{Policy: policy, Enabled: true, GitTimeoutMinutes: &n} + u.Preview = preview(db.repo, u) + if _, err := s.Update(context.Background(), db.repo.Name, u, db.repo.Revision); err != nil { + t.Fatal(err) + } + u = Update{Policy: policy, Enabled: true} + u.Preview = preview(db.repo, u) + if _, err := s.Update(context.Background(), db.repo.Name, u, db.repo.Revision); err != nil || db.repo.GitTimeoutMinutes != 60 { + t.Fatal("omission must preserve") + } + for _, n := range []int{-1, 1, 16, 61, 90} { + u.GitTimeoutMinutes = &n + if _, err := s.Preview(context.Background(), db.repo.Name, u); err == nil { + t.Fatalf("invalid %d", n) + } + } + // Invalid timeout must not be hidden by a valid sync interval. + interval, bad := 5, 61 + if validateUpdate(Update{Policy: policy, SyncIntervalMinutes: &interval, GitTimeoutMinutes: &bad}) == nil { + t.Fatal("validation bypass") + } +} + +func TestWorkerUsesConfiguredGitBudget(t *testing.T) { + for _, n := range []int{15, 30, 45, 60} { + t.Run(fmt.Sprint(n), func(t *testing.T) { + now := time.Now().UTC() + db := &managementStore{repo: control.Repository{Name: "org/repo", Enabled: true, Revision: 1, PolicyRevision: 1, Jobs: []control.Job{{ID: "job", Kind: "sync", State: "queued", Created: now, Updated: now}}}} + job, ok := db.repo.Claim("test", now) + if !ok { + t.Fatal("claim") + } + s := &Service{DB: db, Root: t.TempDir()} + global := n + s.UseGitTimeoutDefault(func() int { return global }) + called := false + s.executeClaimed(context.Background(), db.repo.Name, job, func(ctx context.Context, r control.Repository, j control.Job) error { + called = true + deadline, ok := ctx.Deadline() + want := gitJobBudget(n, "sync") + if !ok || time.Until(deadline) > want || time.Until(deadline) < want-time.Second { + t.Fatalf("wrong budget: %v", time.Until(deadline)) + } + global = 15 + if s.fetchOptions(r, "").Timeout != time.Duration(n)*time.Minute { + t.Fatal("task timeout changed after start") + } + return errors.New("test preparation stopped") + }) + if !called || db.repo.GitTimeoutMinutes != 0 || db.repo.Jobs[0].State != "failed" { + t.Fatal("task snapshot persisted or worker did not finish") + } + }) + } +} + +func TestGitTimeoutRegistrationIdempotency(t *testing.T) { + db := &sourceStore{repos: map[string]control.Repository{}} + s := &Service{DB: db, Root: t.TempDir()} + policy := control.Policy{Branches: []string{"main"}, Count: 10} + register := func(n int) (control.Repository, error) { + return s.RegisterWithGitTimeout(context.Background(), "org/repo", policy, 5, "full", nil, nil, "github", n, "timeout-registration") + } + first, err := register(30) + if err != nil || first.GitTimeoutMinutes != 30 { + t.Fatalf("register: %v", err) + } + replay, err := register(30) + if err != nil || replay.Jobs[0].ID != first.Jobs[0].ID { + t.Fatalf("replay: %v", err) + } + if _, err = register(60); err == nil { + t.Fatal("different timeout replay accepted") + } + if _, err = register(0); err == nil { + t.Fatal("inherit replay accepted") + } + if _, err = register(61); err == nil { + t.Fatal("invalid timeout accepted") + } +} diff --git a/internal/service/management.go b/internal/service/management.go index d75fe29..9075203 100644 --- a/internal/service/management.go +++ b/internal/service/management.go @@ -9,6 +9,7 @@ import ( ) type Update struct { + GitTimeoutMinutes *int `json:"git_timeout_minutes,omitempty"` PublicRead *bool `json:"public_read,omitempty"` PathGroups *[]control.PathGroup `json:"path_groups,omitempty"` SyncIntervalMinutes *int `json:"sync_interval_minutes,omitempty"` @@ -24,6 +25,11 @@ func validateUpdate(u Update) error { if err := u.Policy.Validate(); err != nil { return err } + if u.GitTimeoutMinutes != nil { + if err := control.ValidateGitTimeout(*u.GitTimeoutMinutes); err != nil { + return err + } + } if u.SyncIntervalMinutes != nil { return control.ValidateSyncInterval(*u.SyncIntervalMinutes) } @@ -97,6 +103,9 @@ func (s *Service) Update(ctx context.Context, name string, u Update, revision in if u.SyncIntervalMinutes != nil { r.SyncIntervalMinutes = *u.SyncIntervalMinutes } + if u.GitTimeoutMinutes != nil { + r.GitTimeoutMinutes = *u.GitTimeoutMinutes + } if u.PublicRead != nil { value := *u.PublicRead r.PublicRead = &value diff --git a/internal/service/prepare_revision.go b/internal/service/prepare_revision.go index 50ee4f7..9cf7a7a 100644 --- a/internal/service/prepare_revision.go +++ b/internal/service/prepare_revision.go @@ -82,6 +82,7 @@ func prepareBranch(r control.Repository, revision string, beforeSync bool) (stri } func (s *Service) prepareRevision(ctx context.Context, r control.Repository, j control.Job) error { + timeout := r.GitTimeoutMinutes groups := r.WorkGroups() if j.Group != "" { groups = []string{j.Group} @@ -108,6 +109,7 @@ func (s *Service) prepareRevision(ctx context.Context, r control.Repository, j c } var err error r, err = s.DB.Get(ctx, r.Name) + r.GitTimeoutMinutes = timeout if err != nil { return err } diff --git a/internal/service/service.go b/internal/service/service.go index 0903580..a4452ec 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -32,6 +32,7 @@ type Store interface { NextGeneration(context.Context) (uint32, error) } type Service struct { + gitTimeoutDefault func() int linkManifests linkManifestCache Plugins *wasmplugin.Host pluginSnapshots sync.RWMutex @@ -162,6 +163,13 @@ func (s *Service) RegisterWithVisibility(ctx context.Context, name string, p con } func (s *Service) RegisterWithProvider(ctx context.Context, name string, p control.Policy, interval int, mode string, groups []control.PathGroup, publicRead *bool, provider, key string) (control.Repository, error) { + return s.RegisterWithGitTimeout(ctx, name, p, interval, mode, groups, publicRead, provider, 0, key) +} + +func (s *Service) RegisterWithGitTimeout(ctx context.Context, name string, p control.Policy, interval int, mode string, groups []control.PathGroup, publicRead *bool, provider string, timeout int, key string) (control.Repository, error) { + if err := control.ValidateGitTimeout(timeout); err != nil { + return control.Repository{}, err + } if provider == "" { provider = defaultCodeProvider } @@ -192,6 +200,9 @@ func (s *Service) RegisterWithProvider(ctx context.Context, name string, p contr if provider != defaultCodeProvider { registrationDigest = digest([]any{registrationDigest, provider}) } + if timeout != 0 { + registrationDigest = digest([]any{registrationDigest, timeout}) + } if e := gitstore.ValidateRepo(name); e != nil { return control.Repository{}, e } @@ -206,7 +217,7 @@ func (s *Service) RegisterWithProvider(ctx context.Context, name string, p contr defer s.mu.Unlock() if existing, e := s.DB.Get(ctx, name); e == nil { for _, r := range existing.Receipts { - if r.Key == k && (r.Digest == registrationDigest || ((publicRead == nil || *publicRead) && mode == control.IndexModeFull && existing.EffectiveIndexMode() == mode && interval == control.DefaultSyncIntervalMinutes && existing.SyncInterval() == interval && r.Digest == digest(p))) { + if r.Key == k && (r.Digest == registrationDigest || (timeout == 0 && existing.GitTimeoutMinutes == 0 && (publicRead == nil || *publicRead) && mode == control.IndexModeFull && existing.EffectiveIndexMode() == mode && interval == control.DefaultSyncIntervalMinutes && existing.SyncInterval() == interval && r.Digest == digest(p))) { return projectRepository(s.markStorage(existing)), nil } } @@ -217,7 +228,7 @@ func (s *Service) RegisterWithProvider(ctx context.Context, name string, p contr return control.Repository{}, e } } - r := control.Repository{CodeProvider: provider, CodeHost: source.Host, CreatorID: identity.UserID(ctx), PublicRead: publicRead, IndexMode: mode, PathGroups: groups, ID: control.ID(name), Name: name, Revision: 1, PolicyRevision: 1, Enabled: true, Policy: p, SyncIntervalMinutes: interval, Updated: time.Now().UTC(), Heads: map[string]string{}, Versions: []control.Version{}} + r := control.Repository{GitTimeoutMinutes: timeout, CodeProvider: provider, CodeHost: source.Host, CreatorID: identity.UserID(ctx), PublicRead: publicRead, IndexMode: mode, PathGroups: groups, ID: control.ID(name), Name: name, Revision: 1, PolicyRevision: 1, Enabled: true, Policy: p, SyncIntervalMinutes: interval, Updated: time.Now().UTC(), Heads: map[string]string{}, Versions: []control.Version{}} for _, group := range r.WorkGroups() { groupKey := k if group != "" { diff --git a/internal/service/worker.go b/internal/service/worker.go index c5bc201..1d83ad9 100644 --- a/internal/service/worker.go +++ b/internal/service/worker.go @@ -170,9 +170,9 @@ func (s *Service) executeClaimed(parent context.Context, name string, j control. if s.StorageSettings().CleanupPending { return } - budget := 45 * time.Minute + budget := 60 * time.Minute if j.Kind == "prepare" { - budget = 16 * 45 * time.Minute + budget = 16 * 60 * time.Minute } // up to 16 sequential path groups ctx, cancel := context.WithTimeout(parent, budget) defer cancel() @@ -192,7 +192,14 @@ func (s *Service) executeClaimed(parent context.Context, name string, j control. r, e := s.DB.Get(ctx, name) if e == nil { if _, e = r.Owned(j, time.Now().UTC()); e == nil { - e = prepare(ctx, r, j) + // Resolve inheritance once. Every Git stage of this job uses this snapshot. + r.GitTimeoutMinutes = s.gitTimeoutMinutes(r) + work, stop := context.WithTimeout(ctx, gitJobBudget(r.GitTimeoutMinutes, j.Kind)) + e = prepare(work, r, j) + if work.Err() != nil { + e = work.Err() + } + stop() } } if ctx.Err() != nil { diff --git a/internal/sitesettings/git_timeout_test.go b/internal/sitesettings/git_timeout_test.go new file mode 100644 index 0000000..7992f98 --- /dev/null +++ b/internal/sitesettings/git_timeout_test.go @@ -0,0 +1,53 @@ +package sitesettings + +import ( + "context" + "fmt" + "testing" + "time" +) + +func TestGitTimeoutLegacyPersistenceValidationAndCAS(t *testing.T) { + ctx := context.Background() + store := NewMemoryStore() + old := Record{Name: RepositoryDefaultsName, Revision: 3, Values: map[string]any{"retention_mode": "days", "retention_days": 30, "retention_count": 20}, UpdatedAt: time.Now(), UpdatedBy: "admin"} + if err := store.Ensure(ctx, old); err != nil { + t.Fatal(err) + } + s, err := New(ctx, store, true) + if err != nil { + t.Fatal(err) + } + if s.GitTimeoutMinutes() != 15 { + t.Fatal("legacy default must be 15") + } + revision := int64(3) + for _, n := range []int{15, 30, 45, 60} { + body := []byte(fmt.Sprintf(`{"retention_mode":"days","retention_days":30,"retention_count":20,"git_timeout_minutes":%d}`, n)) + if _, err = s.Update(ctx, RepositoryDefaultsName, revision, body, "admin"); err != nil { + t.Fatal(err) + } + if _, err = s.Update(ctx, RepositoryDefaultsName, revision, body, "admin"); err == nil { + t.Fatal("stale revision accepted") + } + revision++ + restarted, e := New(ctx, store, true) + if e != nil || restarted.GitTimeoutMinutes() != n { + t.Fatalf("restart %d: %v", n, e) + } + } + // Older clients can edit retention without resetting the timeout. + if _, err = s.Update(ctx, RepositoryDefaultsName, revision, []byte(`{"retention_mode":"days","retention_days":20,"retention_count":20}`), "admin"); err != nil { + t.Fatal(err) + } + revision++ + if s.GitTimeoutMinutes() != 60 { + t.Fatal("omitted timeout reset existing value") + } + for _, value := range []string{"0", "-1", "16", "61", "90", "15.5", "null", `"30"`} { + body := []byte(fmt.Sprintf(`{"retention_mode":"days","retention_days":30,"retention_count":20,"git_timeout_minutes":%s}`, value)) + if _, err = s.Update(ctx, RepositoryDefaultsName, revision, body, "admin"); err == nil { + t.Fatalf("accepted %s", value) + } + } +} diff --git a/internal/sitesettings/model.go b/internal/sitesettings/model.go index 055bae8..121ff20 100644 --- a/internal/sitesettings/model.go +++ b/internal/sitesettings/model.go @@ -27,9 +27,10 @@ var ( ) type RepositoryDefaults struct { - RetentionMode string `json:"retention_mode"` - RetentionDays int `json:"retention_days"` - RetentionCount int `json:"retention_count"` + GitTimeoutMinutes int `json:"git_timeout_minutes"` + RetentionMode string `json:"retention_mode"` + RetentionDays int `json:"retention_days"` + RetentionCount int `json:"retention_count"` } type Access struct { PublicRead bool `json:"public_read"` @@ -94,11 +95,14 @@ func validSection(section string) bool { func decodeValues(section string, data []byte) (map[string]any, error) { switch section { case RepositoryDefaultsName: - var v RepositoryDefaults + v := RepositoryDefaults{GitTimeoutMinutes: 15} if contract.DecodeObject(data, &v) != nil || (v.RetentionMode != "days" && v.RetentionMode != "count") || v.RetentionDays < 1 || v.RetentionDays > 365 || v.RetentionCount < 1 || v.RetentionCount > 4096 { return nil, contract.Fail("INVALID_SETTINGS", "Use retention_mode days or count, retention_days 1..365 and retention_count 1..4096; all fields are required.", 400) } - return map[string]any{"retention_mode": v.RetentionMode, "retention_days": v.RetentionDays, "retention_count": v.RetentionCount}, nil + if v.GitTimeoutMinutes != 15 && v.GitTimeoutMinutes != 30 && v.GitTimeoutMinutes != 45 && v.GitTimeoutMinutes != 60 { + return nil, contract.Fail("INVALID_SETTINGS", "Git timeout must be 15, 30, 45 or 60 minutes.", 400) + } + return map[string]any{"git_timeout_minutes": v.GitTimeoutMinutes, "retention_mode": v.RetentionMode, "retention_days": v.RetentionDays, "retention_count": v.RetentionCount}, nil case AccessName: var input struct { PublicRead *bool `json:"public_read"` @@ -135,7 +139,7 @@ func decodeValues(section string, data []byte) (map[string]any, error) { func assign(out *Snapshot, section string, meta Metadata, values map[string]any) { switch section { case RepositoryDefaultsName: - out.RepositoryDefaults = RepositorySection{Metadata: meta, Values: RepositoryDefaults{values["retention_mode"].(string), values["retention_days"].(int), values["retention_count"].(int)}} + out.RepositoryDefaults = RepositorySection{Metadata: meta, Values: RepositoryDefaults{values["git_timeout_minutes"].(int), values["retention_mode"].(string), values["retention_days"].(int), values["retention_count"].(int)}} case AccessName: out.Access = AccessSection{Metadata: meta, Values: Access{PublicRead: values["public_read"].(bool), AllowRegistration: values["allow_registration"].(bool), RegistrationNoticeZH: values["registration_notice_zh"].(string), RegistrationNoticeEN: values["registration_notice_en"].(string), AllowUserRepositories: values["allow_user_repositories"].(bool)}} case CodeSourceName: diff --git a/internal/sitesettings/service.go b/internal/sitesettings/service.go index a1bcc8b..b7da55b 100644 --- a/internal/sitesettings/service.go +++ b/internal/sitesettings/service.go @@ -2,6 +2,7 @@ package sitesettings import ( "context" + "encoding/json" "errors" "log/slog" "strings" @@ -28,7 +29,7 @@ func New(ctx context.Context, store Store, publicReadSeed bool) (*Service, error s := &Service{store: store} now := time.Now().UTC() records := []Record{ - {Name: RepositoryDefaultsName, Revision: 1, Values: map[string]any{"retention_mode": "days", "retention_days": 30, "retention_count": 20}, UpdatedAt: now, UpdatedBy: "system:bootstrap"}, + {Name: RepositoryDefaultsName, Revision: 1, Values: map[string]any{"git_timeout_minutes": 15, "retention_mode": "days", "retention_days": 30, "retention_count": 20}, UpdatedAt: now, UpdatedBy: "system:bootstrap"}, {Name: AccessName, Revision: 1, Values: map[string]any{"public_read": publicReadSeed, "allow_registration": false, "registration_notice_zh": DefaultNoticeZH, "registration_notice_en": DefaultNoticeEN, "allow_user_repositories": false}, UpdatedAt: now, UpdatedBy: "system:bootstrap"}, {Name: CodeSourceName, Revision: 1, Values: map[string]any{"name": "", "host": "", "username": ""}, UpdatedAt: now, UpdatedBy: "system:bootstrap"}, } @@ -50,6 +51,10 @@ func (s *Service) Snapshot() Snapshot { } return Snapshot{} } +func (s *Service) GitTimeoutMinutes() int { + return s.Snapshot().RepositoryDefaults.Values.GitTimeoutMinutes +} + func (s *Service) PublicRead() bool { return s.Snapshot().Access.Values.PublicRead } // AllowUserRepositories reports whether ordinary signed-in users may register repositories. @@ -123,6 +128,13 @@ func (s *Service) Update(ctx context.Context, section string, revision int64, da s.mu.Lock() defer s.mu.Unlock() next := s.Snapshot() + if section == RepositoryDefaultsName { + var fields map[string]json.RawMessage + _ = json.Unmarshal(data, &fields) // decodeValues has already validated the object. + if _, present := fields["git_timeout_minutes"]; !present { + values["git_timeout_minutes"] = next.RepositoryDefaults.Values.GitTimeoutMinutes + } + } if next.Access.Revision == 0 || next.RepositoryDefaults.Revision == 0 || next.CodeSource.Revision == 0 { return Record{}, contract.Fail("SETTINGS_UNAVAILABLE", "Site settings are not initialized.", 503) } diff --git a/internal/sso/community.go b/internal/sso/community.go index 5dc5260..c5f05ab 100644 --- a/internal/sso/community.go +++ b/internal/sso/community.go @@ -10,7 +10,6 @@ import ( "html/template" "io" "net/http" - "os" "strconv" "time" ) @@ -37,8 +36,12 @@ func (h *Handler) admitUser(ctx context.Context, u User) (User, bool, error) { u, e = h.store.UpsertUser(ctx, u) return u, first, e } -func NewCommunity(ctx context.Context, root, origin string, store Store, settings *sitesettings.Service) (*Bootstrap, error) { +func NewCommunity(ctx context.Context, root, origin string, store Store, settings *sitesettings.Service, vault ...CommunitySecrets) (*Bootstrap, error) { b := &Bootstrap{store: store} + credentials, e := communityCredentials(ctx, vault) + if e != nil { + return nil, e + } cfg := Config{Enabled: true, Origin: origin, ClientID: "community", SigningAlgorithm: "RS256"} if e := cfg.Validate(); e != nil { return nil, e @@ -52,7 +55,7 @@ func NewCommunity(ctx context.Context, root, origin string, store Store, setting } } primary := &Handler{config: cfg, store: store, Settings: settings, Providers: map[string]*Handler{}} - if id, secret := os.Getenv("GOOGLE_CLIENT_ID"), os.Getenv("GOOGLE_CLIENT_SECRET"); id != "" && secret != "" { + if id, secret := credentials.GoogleID, credentials.GoogleSecret; id != "" && secret != "" { c := cfg c.ClientID = id h, e := newHandler(ctx, c, secret, store, "https://accounts.google.com") @@ -64,7 +67,7 @@ func NewCommunity(ctx context.Context, root, origin string, store Store, setting h.oauth.RedirectURL = origin + Prefix + "google/callback" primary.Providers["google"] = h } - if id, secret := os.Getenv("GITHUB_CLIENT_ID"), os.Getenv("GITHUB_CLIENT_SECRET"); id != "" && secret != "" { + if id, secret := credentials.GitHubID, credentials.GitHubSecret; id != "" && secret != "" { h := &Handler{config: cfg, store: store, Settings: settings, Provider: "github", issuer: "https://github.com", client: &http.Client{Timeout: 10 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}, oauth: oauth2.Config{ClientID: id, ClientSecret: secret, RedirectURL: origin + Prefix + "github/callback", Endpoint: oauth2.Endpoint{AuthURL: "https://github.com/login/oauth/authorize", TokenURL: "https://github.com/login/oauth/access_token", AuthStyle: oauth2.AuthStyleInParams}, Scopes: []string{"read:user"}}} primary.Providers["github"] = h } diff --git a/internal/sso/community_secrets.go b/internal/sso/community_secrets.go new file mode 100644 index 0000000..9d8a163 --- /dev/null +++ b/internal/sso/community_secrets.go @@ -0,0 +1,45 @@ +package sso + +import ( + "context" + "encoding/json" + "errors" + "github.com/context4ai/sourcegraph/internal/secretstore" + "os" +) + +type CommunitySecrets interface { + Get(context.Context, string) ([]byte, error) + Insert(context.Context, string, []byte) error +} +type oauthCredentials struct{ GoogleID, GoogleSecret, GitHubID, GitHubSecret string } + +func communityCredentials(ctx context.Context, vaults []CommunitySecrets) (oauthCredentials, error) { + seed := oauthCredentials{os.Getenv("GOOGLE_CLIENT_ID"), os.Getenv("GOOGLE_CLIENT_SECRET"), os.Getenv("GITHUB_CLIENT_ID"), os.Getenv("GITHUB_CLIENT_SECRET")} + if len(vaults) == 0 || vaults[0] == nil { + return seed, nil + } + var result oauthCredentials + v := vaults[0] + data, e := v.Get(ctx, "community-oauth") + if errors.Is(e, secretstore.ErrMissing) { + if seed == (oauthCredentials{}) { + return seed, nil + } + if (seed.GoogleID == "") != (seed.GoogleSecret == "") || (seed.GitHubID == "") != (seed.GitHubSecret == "") { + return result, errors.New("OAuth client ID and secret must be configured together") + } + data, _ = json.Marshal(seed) + if e = v.Insert(ctx, "community-oauth", data); e != nil && !errors.Is(e, secretstore.ErrExists) { + return result, e + } + data, e = v.Get(ctx, "community-oauth") + } + if e != nil { + return result, e + } + if json.Unmarshal(data, &result) != nil { + return result, errors.New("persisted OAuth configuration is invalid") + } + return result, nil +} diff --git a/internal/sso/community_secrets_test.go b/internal/sso/community_secrets_test.go new file mode 100644 index 0000000..2290cb4 --- /dev/null +++ b/internal/sso/community_secrets_test.go @@ -0,0 +1,52 @@ +package sso + +import ( + "context" + "github.com/context4ai/sourcegraph/internal/secretstore" + "github.com/context4ai/sourcegraph/internal/sqlstore" + "path/filepath" + "testing" +) + +func TestOAuthCredentialsPersistWithoutEnvironment(t *testing.T) { + ctx := context.Background() + db, e := sqlstore.Open(ctx, "sqlite", filepath.Join(t.TempDir(), "db")) + if e != nil { + t.Fatal(e) + } + defer db.Close() + vault, e := secretstore.NewSQL(ctx, db) + if e != nil { + t.Fatal(e) + } + t.Setenv("GOOGLE_CLIENT_ID", "") + t.Setenv("GOOGLE_CLIENT_SECRET", "") + t.Setenv("GITHUB_CLIENT_ID", "") + t.Setenv("GITHUB_CLIENT_SECRET", "") + empty, e := communityCredentials(ctx, []CommunitySecrets{vault}) + if e != nil || empty != (oauthCredentials{}) { + t.Fatal(e) + } + t.Setenv("GITHUB_CLIENT_ID", "example-client") + t.Setenv("GITHUB_CLIENT_SECRET", "example-secret") + first, e := communityCredentials(ctx, []CommunitySecrets{vault}) + if e != nil { + t.Fatal(e) + } + t.Setenv("GITHUB_CLIENT_ID", "") + t.Setenv("GITHUB_CLIENT_SECRET", "") + fresh, e := secretstore.NewSQL(ctx, db) + if e != nil { + t.Fatal(e) + } + after, e := communityCredentials(ctx, []CommunitySecrets{fresh}) + if e != nil || after != first { + t.Fatal("OAuth identity changed after restart", e) + } + t.Setenv("GITHUB_CLIENT_ID", "different-client") + t.Setenv("GITHUB_CLIENT_SECRET", "different-secret") + after, e = communityCredentials(ctx, []CommunitySecrets{fresh}) + if e != nil || after != first { + t.Fatal("database credentials overwritten", e) + } +} diff --git a/internal/storagecontract/contract_test.go b/internal/storagecontract/contract_test.go index 3d506f3..774e590 100644 --- a/internal/storagecontract/contract_test.go +++ b/internal/storagecontract/contract_test.go @@ -1,9 +1,11 @@ package storagecontract import ( + "bytes" "context" "errors" "github.com/context4ai/sourcegraph/internal/control" + "github.com/context4ai/sourcegraph/internal/secretstore" "github.com/context4ai/sourcegraph/internal/service" "github.com/context4ai/sourcegraph/internal/sitesettings" "github.com/context4ai/sourcegraph/internal/sqlstore" @@ -33,6 +35,7 @@ func TestBackendContract(t *testing.T) { var repos service.Store var auth sso.Store var settings sitesettings.Store + var secrets func(...secretstore.KeySeed) (*secretstore.Store, error) if driver == "mongodb" { db, e := control.Connect(ctx, os.Getenv("TEST_MONGODB_URL")) if e != nil { @@ -42,6 +45,9 @@ func TestBackendContract(t *testing.T) { repos = db auth = &sso.MongoStore{DB: db.AuthDatabase()} settings = &sitesettings.MongoStore{DB: db.AuthDatabase()} + secrets = func(seed ...secretstore.KeySeed) (*secretstore.Store, error) { + return secretstore.New(ctx, db.AuthDatabase(), seed...) + } } else { dsn := os.Getenv("TEST_POSTGRES_URL") if driver == "sqlite" { @@ -55,6 +61,9 @@ func TestBackendContract(t *testing.T) { repos = &control.SQL{DB: db} auth = &sso.SQLStore{DB: db} settings = &sitesettings.SQLStore{DB: db} + secrets = func(seed ...secretstore.KeySeed) (*secretstore.Store, error) { + return secretstore.NewSQL(ctx, db, seed...) + } } r := control.Repository{ID: control.ID("context4ai/context"), Name: "context4ai/context", Revision: 1, JobFailures: []control.JobFailure{{ID: "failed-task", Code: "INDEX_FAILED", Attempt: 2}}} if e := repos.Create(ctx, r); e != nil { @@ -98,6 +107,27 @@ func TestBackendContract(t *testing.T) { } var wg sync.WaitGroup var mu sync.Mutex + key := bytes.Repeat([]byte{42}, 32) + vault, e := secrets(func() ([]byte, error) { return key, nil }) + if e != nil { + t.Fatal(e) + } + if !bytes.Equal(vault.Key(), key) { + t.Fatal("legacy key not imported") + } + token, e := vault.Ensure(ctx, "service-api-token", []byte("existing-service-token-long-enough-for-contract")) + if e != nil { + t.Fatal(e) + } + fresh, e := secrets(func() ([]byte, error) { t.Fatal("existing key must not read host seed"); return nil, nil }) + if e != nil { + t.Fatal(e) + } + recovered, e := fresh.Ensure(ctx, "service-api-token", []byte("must-not-replace")) + if e != nil || !bytes.Equal(token, recovered) { + t.Fatal("secret changed across reopen", e) + } + winners := 0 for _, id := range []string{"google-owner", "github-owner"} { wg.Add(1) diff --git a/scripts/git-timeout.test.mjs b/scripts/git-timeout.test.mjs new file mode 100644 index 0000000..7242404 --- /dev/null +++ b/scripts/git-timeout.test.mjs @@ -0,0 +1,14 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {createRequire} from 'node:module'; +const require=createRequire(import.meta.url); +globalThis.App={t:zh=>zh}; +const M=require('../web/assets/management.js'); +test('Git timeout choices and inherit are explicit',()=>{ + for(const n of [0,15,30,45,60]) assert.equal(M.gitTimeout(String(n)),n); + for(const n of ['',null,undefined,true,1,16,61,90]) assert.throws(()=>M.gitTimeout(n)); + assert.throws(()=>M.gitTimeout(0,false)); + assert.match(M.gitTimeoutOptions(0),/value="0" selected/); + assert.match(M.gitTimeoutOptions(60),/value="60" selected/); + assert.doesNotMatch(M.gitTimeoutOptions(15,false),/value="0"/); +}); diff --git a/web/assets/code-credentials.js b/web/assets/code-credentials.js index 67750e7..870405b 100644 --- a/web/assets/code-credentials.js +++ b/web/assets/code-credentials.js @@ -14,11 +14,11 @@ const box=panel.querySelector('.credential-forms'); function tokenStatus(v,source){ if(!v.configured)return source?.anonymous?t('未配置 · 公开仓库可匿名拉取','Not configured · public repositories are fetched anonymously'):t('未配置','Not configured'); - const labels={ready:['已配置','Configured'],expired:['已过期,请更新','Expired; update required'],expiring:['将在 7 天内过期,请更新','Expires within 7 days; update required'],invalid:['认证失败,请更新','Authentication rejected; update required']}; + const labels={unreadable:['无法解密,请重新填写令牌','Cannot decrypt; enter a new token'],ready:['已配置','Configured'],expired:['已过期,请更新','Expired; update required'],expiring:['将在 7 天内过期,请更新','Expires within 7 days; update required'],invalid:['认证失败,请更新','Authentication rejected; update required']}; const text=labels[v.state]||labels.ready; return t(text[0],text[1])+(!v.expires_at?t(' · 未填写过期时间',' · No expiry provided'):'')+(v.source==='environment'?t(' · 来自部署环境变量',' · From the deployment environment'):''); } - function statusClass(v){return ['expired','invalid'].includes(v.state)?'text-danger':v.state==='expiring'?'text-amber-600':'text-muted';} + function statusClass(v){return ['expired','invalid','unreadable'].includes(v.state)?'text-danger':v.state==='expiring'?'text-amber-600':'text-muted';} function hostLine(source){ if(source.host)return t('从 ','Fetches from ')+source.host+t(' 拉取代码',''); return global&&source.editable?t('尚未填写主机;填写后才能登记这个代码源的仓库','No host yet; set one before registering repositories from this source'):t('管理员尚未配置主机,暂不能登记这个代码源的仓库','An administrator has not configured a host; repositories cannot use this source yet'); diff --git a/web/assets/management.js b/web/assets/management.js index 71ebe50..7054c75 100644 --- a/web/assets/management.js +++ b/web/assets/management.js @@ -37,6 +37,13 @@ function syncIntervalOptions(selected) { return SYNC_INTERVAL_MINUTES.map(minutes => '').join(''); } + function gitTimeout(value, inherit = true) { + const minutes = Number(value); + if (!['number','string'].includes(typeof value) || String(value).trim() === '' || ![15,30,45,60,...(inherit?[0]:[])].includes(minutes)) throw new Error('同步超时需选择 15、30、45 或 60 分钟 / Choose 15, 30, 45 or 60 minutes'); + return minutes; + } + function gitTimeoutLabel(minutes) { return Number(minutes) === 0 ? root.App.t('继承全局','Inherit global') : root.App.t(minutes+' 分钟',minutes+' minutes'); } + function gitTimeoutOptions(selected, inherit = true) { return [...(inherit?[0]:[]),15,30,45,60].map(n=>'').join(''); } function directoryPath(value) { const path = typeof value === 'string' ? value.trim().replace(/\/$/, '') : ''; if (!path || path.length > 1024 || path.startsWith('/') || /^[A-Za-z]:/.test(path) || /[\\*?\[\]\x00-\x1f\x7f]/.test(path) || path.split('/').some(x => !x || x === '.' || x === '..' || x.toLowerCase() === '.git')) throw new Error('请填写安全的仓库相对目录 / Use a safe repository-relative directory'); @@ -133,6 +140,6 @@ root.addEventListener('pagehide',()=>{stopped=true;clearTimeout(timer);}); return {refresh, restart:()=>{count=0;stopped=false;refresh();}}; } - root.Management = {active,orderedJobs,time,validTime,bytes,repositoryName,splitBranches,branches,policy,SYNC_INTERVAL_MINUTES,DEFAULT_SYNC_INTERVAL_MINUTES,DEFAULT_PATH_GROUP,syncInterval,syncIntervalLabel,syncIntervalOptions,directoryPath,pathGroups,groupPathRows,indexModeLabel,indexTiming,jobReceipts,state,status,error,manage,operationKeys,aggregate,poll}; + root.Management = {gitTimeout,gitTimeoutLabel,gitTimeoutOptions,active,orderedJobs,time,validTime,bytes,repositoryName,splitBranches,branches,policy,SYNC_INTERVAL_MINUTES,DEFAULT_SYNC_INTERVAL_MINUTES,DEFAULT_PATH_GROUP,syncInterval,syncIntervalLabel,syncIntervalOptions,directoryPath,pathGroups,groupPathRows,indexModeLabel,indexTiming,jobReceipts,state,status,error,manage,operationKeys,aggregate,poll}; if (typeof module !== 'undefined') module.exports = root.Management; })(typeof window === 'undefined' ? globalThis : window); diff --git a/web/assets/operations-page.js b/web/assets/operations-page.js index 58327bc..55845ba 100644 --- a/web/assets/operations-page.js +++ b/web/assets/operations-page.js @@ -14,9 +14,9 @@ function health(){ if(data?.credential_warnings?.length)return data.credential_warnings.map(w=>{ const name=w.kind==='git'?'Git':'GitHub'; - const expired=w.state==='expired'||w.state==='invalid'; + const expired=w.state==='expired'||w.state==='invalid'||w.state==='unreadable'; const days=w.expires_at?Math.max(0,Math.ceil((Date.parse(w.expires_at)-Date.now())/86400000)):0; - const text=expired?t('全局 '+name+' 凭证已经失效,请尽快更新','Global '+name+' credential is invalid or expired; update it soon'):w.state==='unavailable'?t('全局凭证状态暂时无法读取','Global credential status is unavailable'):t('全局 '+name+' 凭证即将于 '+days+' 天后失效,请更新','Global '+name+' credential expires in '+days+' days; update it'); + const text=w.state==='unreadable'?t('全局 '+name+' 凭证无法解密,请重新填写令牌','Global '+name+' credential cannot be decrypted; enter a new token'):expired?t('全局 '+name+' 凭证已经失效,请尽快更新','Global '+name+' credential is invalid or expired; update it soon'):w.state==='unavailable'?t('全局凭证状态暂时无法读取','Global credential status is unavailable'):t('全局 '+name+' 凭证即将于 '+days+' 天后失效,请更新','Global '+name+' credential expires in '+days+' days; update it'); return ''+esc(text)+''; }).join(''); if(!repos)return t('服务正常','Service is up'); diff --git a/web/assets/repo-page.js b/web/assets/repo-page.js index ae05922..e7f6fdd 100644 --- a/web/assets/repo-page.js +++ b/web/assets/repo-page.js @@ -185,11 +185,11 @@ (rest>0?'':''); } - function ensureDraft(){if(!draft)draft={branches:repo.policy.branches.slice(),defaultBranch:!!repo.policy.default_branch,publicRead:repo.public_read!==false,pending:'',kind:repo.policy.days?'days':'count',value:String(repo.policy.days||repo.policy.count),syncInterval:String(repo.sync_interval_minutes||M.DEFAULT_SYNC_INTERVAL_MINUTES),skipGenerated:!repo.index_generated,indexSymlinks:repo.index_symlinks!==false,pathState:{groups:repo.path_groups||[]}};} + function ensureDraft(){if(!draft)draft={branches:repo.policy.branches.slice(),defaultBranch:!!repo.policy.default_branch,publicRead:repo.public_read!==false,pending:'',kind:repo.policy.days?'days':'count',value:String(repo.policy.days||repo.policy.count),gitTimeout:String(repo.git_timeout_minutes||0),syncInterval:String(repo.sync_interval_minutes||M.DEFAULT_SYNC_INTERVAL_MINUTES),skipGenerated:!repo.index_generated,indexSymlinks:repo.index_symlinks!==false,pathState:{groups:repo.path_groups||[]}};} function settings(){ if(repo.deleted)return '

'+t('仓库已标记删除,无法修改设置。请在「管理」里重试永久删除','This repository is marked for deletion and cannot be edited. Retry permanent deletion from Manage')+'

'; const can=A.user()&&canRepo(), k=keep(repo.policy); - if(!can)return '
'+rows([[t('索引模式','Indexing mode'),M.indexModeLabel(repo.index_mode)],[t('监听分支','Branches'),mono(esc((repo.policy.branches||[]).join(', ')))],[t('保留','Retention'),t('最近 '+k[0]+' '+k[1],'Last '+k[0]+' '+k[1])],[t('同步间隔','Sync interval'),M.syncIntervalLabel(repo.sync_interval_minutes||M.DEFAULT_SYNC_INTERVAL_MINUTES)],[t('软链目标','Symlink targets'),repo.index_symlinks!==false?t('新索引展开仓库内软链','New indexes follow repository symlinks'):t('新索引不展开软链','New indexes do not follow symlinks')],[t('压缩文件','Minified files'),repo.index_generated?t('压缩 JS/CSS 与 source map 建立索引','Minified JS/CSS and source maps are indexed'):t('压缩 JS/CSS 与 source map 不建立索引','Minified JS/CSS and source maps are not indexed')]])+ + if(!can)return '
'+rows([[t('索引模式','Indexing mode'),M.indexModeLabel(repo.index_mode)],[t('监听分支','Branches'),mono(esc((repo.policy.branches||[]).join(', ')))],[t('保留','Retention'),t('最近 '+k[0]+' '+k[1],'Last '+k[0]+' '+k[1])],[t('同步间隔','Sync interval'),M.syncIntervalLabel(repo.sync_interval_minutes||M.DEFAULT_SYNC_INTERVAL_MINUTES)],[t('Git 同步超时','Git sync timeout'),M.gitTimeoutLabel(repo.git_timeout_minutes||0)],[t('软链目标','Symlink targets'),repo.index_symlinks!==false?t('新索引展开仓库内软链','New indexes follow repository symlinks'):t('新索引不展开软链','New indexes do not follow symlinks')],[t('压缩文件','Minified files'),repo.index_generated?t('压缩 JS/CSS 与 source map 建立索引','Minified JS/CSS and source maps are indexed'):t('压缩 JS/CSS 与 source map 不建立索引','Minified JS/CSS and source maps are not indexed')]])+ '
'; ensureDraft(); const inp='mt-2 rounded-control bg-ink/[0.04] px-3 py-2 font-mono text-[13.5px] outline-none ring-1 ring-transparent focus:ring-ink/20'; @@ -199,7 +199,8 @@ (repo.index_mode==='monorepo'?'
':'')+ '
'+ '
'+ - '
'+ + '
'+ + '

'+t('默认继承全局;用于 Git 拉取及文件对象补拉,新任务启动时生效。','Inherit global by default; used for Git fetching and object hydration when a new task starts.')+'

'+ ''+ ''+ ''+ @@ -209,13 +210,14 @@ '
'+t('保存前先预览影响','Preview the impact before saving')+'
'+ ''; } - function capture(){if(!$('#pbranches')||!branchInput)return;draft={...branchInput.snapshot(),publicRead:$('#ppublic').checked,kind:$('#pkind').value,value:$('#pvalue').value,syncInterval:$('#psync-interval').value,skipGenerated:$('#pskip-generated').checked,indexSymlinks:$('#pindex-symlinks').checked,pathState:pathInput?pathInput.snapshot():draft?.pathState};} - function target(){const branches=branchInput.value(),groups=pathInput?.value();capture();return {policy:M.policy(branches,draft.kind,draft.value,branchInput.usesDefault()),public_read:draft.publicRead,sync_interval_minutes:M.syncInterval(draft.syncInterval),index_generated:!draft.skipGenerated,index_symlinks:draft.indexSymlinks,enabled:repo.enabled,deleted:repo.deleted,...(groups?{path_groups:groups}:{})};} + function capture(){if(!$('#pbranches')||!branchInput)return;draft={...branchInput.snapshot(),publicRead:$('#ppublic').checked,kind:$('#pkind').value,value:$('#pvalue').value,gitTimeout:$('#pgit-timeout').value,syncInterval:$('#psync-interval').value,skipGenerated:$('#pskip-generated').checked,indexSymlinks:$('#pindex-symlinks').checked,pathState:pathInput?pathInput.snapshot():draft?.pathState};} + function target(){const branches=branchInput.value(),groups=pathInput?.value();capture();return {policy:M.policy(branches,draft.kind,draft.value,branchInput.usesDefault()),public_read:draft.publicRead,git_timeout_minutes:M.gitTimeout(draft.gitTimeout),sync_interval_minutes:M.syncInterval(draft.syncInterval),index_generated:!draft.skipGenerated,index_symlinks:draft.indexSymlinks,enabled:repo.enabled,deleted:repo.deleted,...(groups?{path_groups:groups}:{})};} function invalidate(){capture();preview=null;if($('#save'))$('#save').disabled=true;if($('#impact'))$('#impact').textContent=t('保存前先预览影响','Preview the impact before saving');if($('#policy-error'))$('#policy-error').textContent='';} function impact(v,body){ const cur=repo.policy,next=body.policy,n=v.retained_versions; const samePolicy=!!cur.default_branch===!!next.default_branch&&JSON.stringify(cur.branches)===JSON.stringify(next.branches)&&(cur.days||0)===(next.days||0)&&(cur.count||0)===(next.count||0); const pathsChanged=repo.index_mode==='monorepo'&&JSON.stringify(repo.path_groups||[])!==JSON.stringify(body.path_groups||[]); + const timeoutChanged=(repo.git_timeout_minutes||0)!==body.git_timeout_minutes; const intervalChanged=(repo.sync_interval_minutes||M.DEFAULT_SYNC_INTERVAL_MINUTES)!==body.sync_interval_minutes; const shrink=cur.days&&next.days?next.days'+t('监听分支变化,需要重新同步','Branches change; a new sync is needed')+''); lines.push(shrink?''+t('超出新保留范围的版本不再可查','Versions outside the new window stop being searchable')+'':t('更早的提交在需要时准备','Older commits are prepared when needed')); @@ -290,7 +293,7 @@ if($('#more-jobs'))$('#more-jobs').onclick=()=>{jobLimit+=JOB_PAGE;render();}; if($('#policy-login'))$('#policy-login').onclick=()=>M.manage(canRepo()); if($('#preview'))$('#preview').onclick=previewPolicy; - if($('#policy-form')){branchInput=BranchInput.mount($('#pbranches'),{...draft,onChange:invalidate});if($('#ppaths'))pathInput=PathInput.mount($('#ppaths'),{...draft.pathState,onChange:invalidate});$('#policy-form').onsubmit=savePolicy;['#pvalue','#pkind','#psync-interval','#ppublic','#pskip-generated'].forEach(s=>$(s).oninput=()=>{invalidate();$('#pvalue').max=$('#pkind').value==='days'?365:4096;});} + if($('#policy-form')){branchInput=BranchInput.mount($('#pbranches'),{...draft,onChange:invalidate});if($('#ppaths'))pathInput=PathInput.mount($('#ppaths'),{...draft.pathState,onChange:invalidate});$('#policy-form').onsubmit=savePolicy;['#pvalue','#pkind','#pgit-timeout','#psync-interval','#ppublic','#pskip-generated'].forEach(s=>$(s).oninput=()=>{invalidate();$('#pvalue').max=$('#pkind').value==='days'?365:4096;});} } async function load(force=false){if(loading||!name)return;if(!force&&(busy||tab==='policy'||$('#confirm').open||(document.activeElement&&document.activeElement.id==='manager-q')))return;loading=true;try{repo=await API.repo(name);$('#load-error').textContent='';render();}catch(e){$('#load-error').textContent=M.error(e);throw e;}finally{loading=false;}} const poller=M.poll(()=>load(),15000,80);$('#sync').disabled=true;$('#search').classList.add('hidden');$('#sync').onclick=()=>prepare('sync'); diff --git a/web/assets/repos-page.js b/web/assets/repos-page.js index 7d5c618..037826d 100644 --- a/web/assets/repos-page.js +++ b/web/assets/repos-page.js @@ -8,6 +8,7 @@ $('#f-index-mode').onchange=()=>{$('#f-paths-wrap').classList.toggle('hidden',$('#f-index-mode').value!=='monorepo');$('#form-error').textContent='';}; function renderSyncInterval(){const input=$('#f-sync-interval');input.innerHTML=M.syncIntervalOptions(input.value||M.DEFAULT_SYNC_INTERVAL_MINUTES);} renderSyncInterval(); + $('#f-git-timeout').innerHTML=M.gitTimeoutOptions(0); const initial=new URLSearchParams(location.search); $('#filter').value=initial.get('q')||''; $('#show-deleted').checked=initial.get('deleted')==='1'; @@ -96,7 +97,7 @@ $('#f-name').addEventListener('input',()=>{$('#form-error').textContent='';}); $('#f-sync-interval').addEventListener('change',()=>{$('#form-error').textContent='';}); async function submit(e){if(e)e.preventDefault();if(submitting||!M.manage(canRegister())||!$('#form').reportValidity())return;let body; - try {const name=M.repositoryName($('#f-name').value);if(rows.some(r=>r.name===name))throw new Error(t('该仓库已登记,请到详情页修改','This repository is already registered. Edit it from its details page'));body={name,code_provider:$('#f-code-provider').value,index_mode:$('#f-index-mode').value,public_read:$('#f-ack').checked,policy:{branches:branchInput.value(),...(branchInput.usesDefault()?{default_branch:true}:{})},sync_interval_minutes:M.syncInterval($('#f-sync-interval').value)};if(body.index_mode==='monorepo')body.path_groups=pathInput.value();} + try {const name=M.repositoryName($('#f-name').value);if(rows.some(r=>r.name===name))throw new Error(t('该仓库已登记,请到详情页修改','This repository is already registered. Edit it from its details page'));body={name,git_timeout_minutes:M.gitTimeout($('#f-git-timeout').value),code_provider:$('#f-code-provider').value,index_mode:$('#f-index-mode').value,public_read:$('#f-ack').checked,policy:{branches:branchInput.value(),...(branchInput.usesDefault()?{default_branch:true}:{})},sync_interval_minutes:M.syncInterval($('#f-sync-interval').value)};if(body.index_mode==='monorepo')body.path_groups=pathInput.value();} catch(e){$('#form-error').textContent=e.message;return;} submitting=true;$('#submit').disabled=true;$('#submit').textContent=t('提交中','Submitting');$('#form-error').textContent=''; try {const r=await API.request('/api/admin/v1/repos',{method:'POST',body,headers:{'Idempotency-Key':keys.get(body)}});keys.done(body);A.toast(t('已登记,首轮同步已排队','Registered, first sync queued'));location.assign('/sourcegraph/repo?repo='+encodeURIComponent(r.name)+'&tab=jobs');} diff --git a/web/assets/settings-page.js b/web/assets/settings-page.js index b6fb76d..d7e8940 100644 --- a/web/assets/settings-page.js +++ b/web/assets/settings-page.js @@ -12,13 +12,13 @@ function status(key,text,isError=false){const el=$('#'+sections[key].id+'-status');el.textContent=text;el.classList.toggle('text-danger',isError);el.classList.toggle('text-muted',!isError);} function enabled(key){const s=sections[key],off=loading||s.busy||!s.data||!permitted();$('#'+s.id+'-fields').disabled=off;$('#save-'+s.id).disabled=off;$('#reload-'+s.id).disabled=Object.values(sections).some(x=>x.busy)||loading||!permitted();} function apply(key,data){if(!data||!Number.isInteger(data.revision)||!data.values)throw new Error(t('系统设置响应无效','Invalid settings response'));sections[key].data=data; - if(key==='repository_defaults'){const v=data.values;$('#default-retention-mode').value=v.retention_mode;$('#default-retention-days').value=v.retention_days;$('#default-retention-count').value=v.retention_count;syncMode();}else {$('#public-read').checked=data.values.public_read===true;$('#allow-registration').checked=data.values.allow_registration===true;$('#registration-notice-zh').value=data.values.registration_notice_zh||'';$('#registration-notice-en').value=data.values.registration_notice_en||'';$('#allow-user-repositories').checked=data.values.allow_user_repositories===true;} + if(key==='repository_defaults'){const v=data.values;$('#default-git-timeout').value=v.git_timeout_minutes||15;$('#default-retention-mode').value=v.retention_mode;$('#default-retention-days').value=v.retention_days;$('#default-retention-count').value=v.retention_count;syncMode();}else {$('#public-read').checked=data.values.public_read===true;$('#allow-registration').checked=data.values.allow_registration===true;$('#registration-notice-zh').value=data.values.registration_notice_zh||'';$('#registration-notice-en').value=data.values.registration_notice_en||'';$('#allow-user-repositories').checked=data.values.allow_user_repositories===true;} status(key,'');enabled(key);} async function load(only){if(loading||Object.values(sections).some(x=>x.busy)||!gate())return;loading=true;$('#settings-retry').disabled=true;$('#settings-load-error').textContent='';Object.keys(sections).forEach(enabled); try{const data=await API.request('/api/admin/settings');if(!permitted())return;for(const key of only?[only]:Object.keys(sections))apply(key,data.sections?.[key]);$('#settings-retry').classList.add('hidden');} catch(e){if(only)status(only,message(e)+t(',输入已保留','; input retained'),true);else{$('#settings-load-error').textContent=message(e);$('#settings-retry').classList.remove('hidden');}if(e.status===401||e.status===403){await API.loadSession();gate();}} finally{loading=false;$('#settings-retry').disabled=false;Object.keys(sections).forEach(enabled);}} - function values(key){if(key==='access')return {public_read:$('#public-read').checked,allow_registration:$('#allow-registration').checked,registration_notice_zh:$('#registration-notice-zh').value,registration_notice_en:$('#registration-notice-en').value,allow_user_repositories:$('#allow-user-repositories').checked};const mode=$('#default-retention-mode').value,days=Number($('#default-retention-days').value),count=Number($('#default-retention-count').value);if(!['days','count'].includes(mode)||!Number.isInteger(days)||days<1||days>365||!Number.isInteger(count)||count<1||count>4096)throw new Error(t('天数为 1 到 365,提交数为 1 到 4096','Use 1 to 365 days or 1 to 4096 commits'));return {retention_mode:mode,retention_days:days,retention_count:count};} + function values(key){if(key==='access')return {public_read:$('#public-read').checked,allow_registration:$('#allow-registration').checked,registration_notice_zh:$('#registration-notice-zh').value,registration_notice_en:$('#registration-notice-en').value,allow_user_repositories:$('#allow-user-repositories').checked};const mode=$('#default-retention-mode').value,days=Number($('#default-retention-days').value),count=Number($('#default-retention-count').value);if(!['days','count'].includes(mode)||!Number.isInteger(days)||days<1||days>365||!Number.isInteger(count)||count<1||count>4096)throw new Error(t('天数为 1 到 365,提交数为 1 到 4096','Use 1 to 365 days or 1 to 4096 commits'));const timeout=Number($('#default-git-timeout').value);if(![15,30,45,60].includes(timeout))throw new Error(t('同步超时必须为 15、30、45 或 60 分钟','Choose 15, 30, 45 or 60 minutes'));return {git_timeout_minutes:timeout,retention_mode:mode,retention_days:days,retention_count:count};} async function save(key,e){e.preventDefault();const s=sections[key];if(loading||s.busy||!s.data||!gate())return;let draft;try{draft=values(key);}catch(err){status(key,err.message,true);return;}s.busy=true;Object.keys(sections).forEach(enabled);status(key,t('正在保存…','Saving…')); try{const data=await API.request('/api/admin/settings/'+key,{method:'PATCH',body:{revision:s.data.revision,values:draft}});if(!Number.isInteger(data.revision)||!data.values)throw new Error(t('保存回执无效,请重新加载确认','Invalid save receipt. Reload to confirm.'));s.data=data;status(key,t('已保存','Saved')); if(key==='access'){try{API.policy=await API.request('/access-policy');document.dispatchEvent(new Event('app:policy'));}catch{status(key,t('已保存,刷新页面后生效','Saved; refresh to apply'));}}} diff --git a/web/repos.html b/web/repos.html index 5d3e237..cda0c04 100644 --- a/web/repos.html +++ b/web/repos.html @@ -69,6 +69,7 @@

同步间隔 +

默认继承全局,可单独设置,最多 60 分钟。

diff --git a/web/settings.html b/web/settings.html index 8671960..4fb978d 100644 --- a/web/settings.html +++ b/web/settings.html @@ -31,9 +31,13 @@
-

新仓库默认值

登记新仓库时的初始值,不影响已有仓库

+

仓库默认设置

保留策略用于新仓库;同步超时适用于所有继承全局设置的仓库

+
+ +

单位为分钟,默认 15,最多 60。用于 Git 拉取及文件对象补拉;新任务生效,进行中的任务保持原超时。

+
每个仓库的最大索引数量的限制方式