diff --git a/AGENTS.md b/AGENTS.md index 9f5d70be..e1e13b60 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1085,6 +1085,18 @@ described through `describeRejectedOrigin` so a password or token in the URL never reaches stderr. Never build a credential-bearing client with a nil `CheckRedirect`. +Transfers to server-chosen URLs (presigned object-store URLs, the ti fs download +redirect, Starter export files, and `ti update` release metadata and artifacts) +carry no credential header, so they may cross origins, but they must never +travel in cleartext. Build the request with `transport.NewSecureRequest` and +send it with `transport.DoSecure`: the URL and every redirect must be `https`, +or plain `http` only to a loopback host, never with user info, and a redirect +from `https` to `http` is refused even to loopback. Never put the raw transfer +URL in an error, because its query carries the presigned signature; a refused +URL is named only by `transport.Origin`, and net/http's own error for a redirect +whose `Location` does not parse is dropped (`transport.IsLocationParseError`) +because it repeats the raw `Location` value. + Do not store secrets in logs, telemetry, generated docs examples, or test fixtures. diff --git a/cmd/ti-telemetry-migrate/main.go b/cmd/ti-telemetry-migrate/main.go index 06ee943d..43abc237 100644 --- a/cmd/ti-telemetry-migrate/main.go +++ b/cmd/ti-telemetry-migrate/main.go @@ -3,31 +3,55 @@ package main import ( "context" "database/sql" + "errors" "flag" + "io" "log/slog" "os" "strings" "time" _ "github.com/go-sql-driver/mysql" + "github.com/tidbcloud/ti-cli/internal/telemetrybackend" "github.com/tidbcloud/ti-cli/internal/telemetrybackend/migrations" ) func main() { logger := slog.New(slog.NewJSONHandler(os.Stderr, nil)) + os.Exit(run(os.Args[1:], os.Getenv, logger)) +} + +func run(args []string, getenv func(string) string, logger *slog.Logger) int { + flags := flag.NewFlagSet("ti-telemetry-migrate", flag.ContinueOnError) + flags.SetOutput(io.Discard) var targetVersion int64 - flag.Int64Var(&targetVersion, "to-version", 0, "apply migrations through this version") - flag.Parse() + flags.Int64Var(&targetVersion, "to-version", 0, "apply migrations through this version") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + flags.SetOutput(os.Stderr) + flags.PrintDefaults() + return 0 + } + logger.Error("invalid arguments", "error", err.Error()) + return 2 + } - dsn := strings.TrimSpace(os.Getenv("TIDB_DSN")) + dsn := strings.TrimSpace(getenv("TIDB_DSN")) if dsn == "" { logger.Error("TIDB_DSN is required") - os.Exit(1) + return 1 + } + // Same rule as the API server: in production (also when + // TELEMETRY_ENVIRONMENT is unset or not a known local/test name) the DSN + // must use verified TLS. + if err := telemetrybackend.ValidateTiDBDSN(getenv("TELEMETRY_ENVIRONMENT"), dsn); err != nil { + logger.Error("refusing TIDB_DSN", "error", err.Error()) + return 1 } db, err := sql.Open("mysql", dsn) if err != nil { logger.Error("open TiDB connection failed") - os.Exit(1) + return 1 } defer db.Close() ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) @@ -39,7 +63,8 @@ func main() { } if err != nil { logger.Error("apply telemetry schema migrations failed") - os.Exit(1) + return 1 } logger.Info("telemetry schema migrations applied") + return 0 } diff --git a/cmd/ti-telemetry-migrate/main_test.go b/cmd/ti-telemetry-migrate/main_test.go new file mode 100644 index 00000000..19fda415 --- /dev/null +++ b/cmd/ti-telemetry-migrate/main_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "bytes" + "log/slog" + "strings" + "testing" +) + +func TestRunRefusesProductionDSNWithoutVerifiedTLS(t *testing.T) { + for _, tc := range []struct { + name string + environment string + }{ + {"unset environment defaults to production", ""}, + {"explicit production", "production"}, + {"misspelled production", "prod"}, + } { + t.Run(tc.name, func(t *testing.T) { + env := map[string]string{ + "TIDB_DSN": "user:secret-password@tcp(203.0.113.10:4000)/telemetry?tls=skip-verify", + "TELEMETRY_ENVIRONMENT": tc.environment, + } + var logs bytes.Buffer + code := run(nil, func(key string) string { return env[key] }, slog.New(slog.NewJSONHandler(&logs, nil))) + if code != 1 { + t.Fatalf("run exit code = %d, want 1", code) + } + if !strings.Contains(logs.String(), "verified TLS") { + t.Fatalf("logs = %s, want a verified TLS refusal", logs.String()) + } + if strings.Contains(logs.String(), "secret-password") { + t.Fatalf("logs leak the DSN password: %s", logs.String()) + } + if strings.Contains(logs.String(), "migrations failed") { + t.Fatalf("run tried to migrate before refusing the DSN: %s", logs.String()) + } + }) + } +} + +func TestRunRequiresDSN(t *testing.T) { + var logs bytes.Buffer + code := run(nil, func(string) string { return "" }, slog.New(slog.NewJSONHandler(&logs, nil))) + if code != 1 || !strings.Contains(logs.String(), "TIDB_DSN is required") { + t.Fatalf("run exit code = %d, logs = %s", code, logs.String()) + } +} + +func TestRunRejectsUnknownFlag(t *testing.T) { + var logs bytes.Buffer + code := run([]string{"--unknown"}, func(string) string { return "" }, slog.New(slog.NewJSONHandler(&logs, nil))) + if code != 2 { + t.Fatalf("run exit code = %d, want 2", code) + } +} diff --git a/docs/telemetry-backend-design.md b/docs/telemetry-backend-design.md index d8d708d3..0aa08e3c 100644 --- a/docs/telemetry-backend-design.md +++ b/docs/telemetry-backend-design.md @@ -43,7 +43,7 @@ TELEMETRY_TRUSTED_PROXY_CIDRS=172.16.0.0/12 TIDB_DSN=tdc_telemetry:password@tcp(gateway01.us-east-1.prod.aws.tidbcloud.com:4000)/tdc_telemetry?tls=true&parseTime=true ``` -`TIDB_DSN` is an application credential. It must remain only in the server-side `.env`, must never be stored in GitHub repository or Environment secrets, must never be committed to git, and must never be logged. GitHub may store only deployment transport credentials such as the SSH host, username, and key. For TiDB Cloud, the DSN must enable TLS with certificate and identity verification. `TELEMETRY_ENVIRONMENT=production` enables this strict TLS validation. +`TIDB_DSN` is an application credential. It must remain only in the server-side `.env`, must never be stored in GitHub repository or Environment secrets, must never be committed to git, and must never be logged. GitHub may store only deployment transport credentials such as the SSH host, username, and key. For TiDB Cloud, the DSN must enable TLS with certificate and identity verification. Both the API server and `ti-telemetry-migrate` enforce this strict TLS validation. Every `TELEMETRY_ENVIRONMENT` value counts as production, an unset one included, except the local and test names `development`, `local`, `test` and `telemetry-e2e`, so a misspelling such as `prod` still gets the strict rule. Deployments upgraded from the former dual-sink implementation may still have `POSTHOG_API_HOST` and `POSTHOG_PROJECT_TOKEN` in their server-local `.env`. The backend ignores those variables; operators should remove them after deployment. This change stops future forwarding but does not delete historical data already held by an external service. diff --git a/e2e/installer_https_test.go b/e2e/installer_https_test.go new file mode 100644 index 00000000..cf82aa34 --- /dev/null +++ b/e2e/installer_https_test.go @@ -0,0 +1,167 @@ +package e2e + +import ( + "encoding/pem" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "testing" +) + +// installShellDownload returns install.sh's error() and download() functions +// followed by a call `download "$1" "$2"`, so the real download code runs alone. +func installShellDownload(t *testing.T) string { + t.Helper() + data, err := os.ReadFile(filepath.Join("..", "scripts", "install.sh")) + if err != nil { + t.Fatal(err) + } + script := string(data) + var errorLine string + for _, line := range strings.Split(script, "\n") { + if strings.HasPrefix(line, "error() {") { + errorLine = line + break + } + } + start := strings.Index(script, "\ndownload() {\n") + if errorLine == "" || start < 0 { + t.Fatal("install.sh no longer defines error() and download()") + } + end := strings.Index(script[start+1:], "\n}\n") + if end < 0 { + t.Fatal("install.sh download() is not terminated") + } + body := script[start+1 : start+1+end+3] + return "set -eu\nRED=''\nRESET=''\n" + errorLine + "\n" + body + "download \"$1\" \"$2\"\n" +} + +func TestUnixInstallerDownloadsStayOnHTTPS(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("shell installer is not supported on Windows") + } + curl, err := exec.LookPath("curl") + if err != nil { + t.Skip("curl not found") + } + var plainHits atomic.Int32 + plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + plainHits.Add(1) + _, _ = w.Write([]byte("tampered")) + })) + defer plain.Close() + secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/redirect" { + http.Redirect(w, r, plain.URL+"/ti.tar.gz", http.StatusFound) + return + } + _, _ = w.Write([]byte("genuine")) + })) + defer secure.Close() + + dir := t.TempDir() + caFile := filepath.Join(dir, "ca.pem") + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: secure.Certificate().Raw}) + if err := os.WriteFile(caFile, caPEM, 0o600); err != nil { + t.Fatal(err) + } + script := installShellDownload(t) + download := func(rawURL, out string) ([]byte, error) { + cmd := exec.Command("sh", "-c", script, "sh", rawURL, out) + cmd.Env = []string{ + "PATH=" + filepath.Dir(curl) + ":/usr/bin:/bin", + "CURL_CA_BUNDLE=" + caFile, + "SSL_CERT_FILE=" + caFile, + } + return cmd.CombinedOutput() + } + + refused := func(name, rawURL string) { + t.Helper() + out := filepath.Join(dir, strings.ReplaceAll(name, " ", "-")) + if output, err := download(rawURL, out); err == nil { + t.Fatalf("%s: download succeeded:\n%s", name, output) + } + if data, _ := os.ReadFile(out); string(data) == "tampered" { + t.Fatalf("%s: download wrote the cleartext bytes", name) + } + if hits := plainHits.Load(); hits != 0 { + t.Fatalf("%s: the plain http server was reached %d times", name, hits) + } + } + // Needs no TLS, so it runs even where curl can't be pointed at the test CA. + refused("plain http", plain.URL+"/ti.tar.gz") + + okFile := filepath.Join(dir, "ok") + if output, err := download(secure.URL+"/ti.tar.gz", okFile); err != nil { + if os.Getenv("CI") != "" { + t.Fatalf("curl does not trust the test certificate: %v\n%s", err, output) + } + t.Skipf("curl does not trust the test certificate here: %v\n%s", err, output) + } + if data, _ := os.ReadFile(okFile); string(data) != "genuine" { + t.Fatalf("https download wrote %q", data) + } + refused("https redirect to http", secure.URL+"/redirect") +} + +func TestUnixInstallerRequiresCurl(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("shell installer is not supported on Windows") + } + sh, err := exec.LookPath("sh") + if err != nil { + t.Skip("sh not found") + } + // A PATH with only sh: no curl, and no wget fallback either. + bin := t.TempDir() + if err := os.Symlink(sh, filepath.Join(bin, "sh")); err != nil { + t.Fatal(err) + } + cmd := exec.Command(filepath.Join(bin, "sh"), "-c", installShellDownload(t), "sh", "https://example.invalid/ti.tar.gz", filepath.Join(t.TempDir(), "out")) + cmd.Env = []string{"PATH=" + bin} + output, err := cmd.CombinedOutput() + if err == nil || !strings.Contains(string(output), "curl is required") { + t.Fatalf("download without curl = %v\n%s", err, output) + } +} + +func TestInstallersRefuseDownloadsThatLeaveHTTPS(t *testing.T) { + shell, err := os.ReadFile(filepath.Join("..", "scripts", "install.sh")) + if err != nil { + t.Fatal(err) + } + if strings.Count(string(shell), "curl -f") != strings.Count(string(shell), "--proto '=https' --proto-redir '=https'") { + t.Fatal("every curl download in install.sh must use --proto '=https' --proto-redir '=https'") + } + if strings.Contains(string(shell), "wget -") { + t.Fatal("install.sh must not download with wget: it can't refuse an https-to-http redirect") + } + + powerShell, err := os.ReadFile(filepath.Join("..", "scripts", "install.ps1")) + if err != nil { + t.Fatal(err) + } + ps := string(powerShell) + if got := strings.Count(ps, "Invoke-WebRequest"); got != 1 { + t.Fatalf("install.ps1 calls Invoke-WebRequest %d times; every download must go through Save-HttpsDownload", got) + } + if !strings.Contains(ps, `$FinalUri.Scheme -ne "https"`) { + t.Fatal("install.ps1 must check that the final download URL is https") + } + for _, call := range []string{ + "Save-HttpsDownload $ArchiveUrl $ArchivePath", + "Save-HttpsDownload $ChecksumsUrl $ChecksumsPath", + "Save-HttpsDownload $CompanionUrl $CompanionPath", + "Save-HttpsDownload $CompanionChecksumsUrl $CompanionChecksumsPath", + } { + if !strings.Contains(ps, call) { + t.Fatalf("install.ps1 is missing %q", call) + } + } +} diff --git a/e2e/installer_test.go b/e2e/installer_test.go index 963a41db..c2dbf818 100644 --- a/e2e/installer_test.go +++ b/e2e/installer_test.go @@ -165,6 +165,7 @@ url='' while [ "$#" -gt 0 ]; do case "$1" in -o) out="$2"; shift 2 ;; + --proto|--proto-redir) shift 2 ;; -*) shift ;; *) url="$1"; shift ;; esac diff --git a/e2e/telemetry_test.go b/e2e/telemetry_test.go index 0d010a90..84e4580a 100644 --- a/e2e/telemetry_test.go +++ b/e2e/telemetry_test.go @@ -108,7 +108,9 @@ func telemetryMigratorBinary(t *testing.T) string { func runTelemetryMigrator(t *testing.T, ctx context.Context, binary, dsn string, args ...string) { t.Helper() command := exec.CommandContext(ctx, binary, args...) - command.Env = append(os.Environ(), "TIDB_DSN="+dsn) + // Match the API server below: the e2e TiDB is not on verified TLS, and the + // migrator, like the API, treats an unset environment as production. + command.Env = append(os.Environ(), "TIDB_DSN="+dsn, "TELEMETRY_ENVIRONMENT=telemetry-e2e") output, err := command.CombinedOutput() if err != nil { t.Fatalf("run telemetry migrator: %v\n%s", err, output) diff --git a/internal/api/client.go b/internal/api/client.go index dc407a04..306d43af 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -279,6 +279,17 @@ func (c *Client) do(req *http.Request) (*http.Response, error) { // requestError maps a failed round trip to an API error. A same-origin // redirect loop is reported as such rather than as a network problem. func requestError(req *http.Request, err error) error { + // net/http reports a redirect whose Location does not parse with the raw + // Location value in the message. A ti fs download redirect carries a presigned + // signature there, so that cause is dropped rather than shown. + if apitransport.IsLocationParseError(err) { + return &Error{ + Code: "api.invalid_redirect", + Category: "api", + ExitCode: 1, + Message: fmt.Sprintf("API request failed: %s sent a redirect with an invalid Location header", req.URL.Host), + } + } if errors.Is(err, apitransport.ErrTooManyRedirects) { return &Error{ Code: "api.redirect_loop", diff --git a/internal/api/fs/client.go b/internal/api/fs/client.go index cbca9124..b949f093 100644 --- a/internal/api/fs/client.go +++ b/internal/api/fs/client.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "net/http" "net/url" @@ -14,6 +15,7 @@ import ( "time" "github.com/tidbcloud/ti-cli/internal/api" + apitransport "github.com/tidbcloud/ti-cli/internal/api/transport" "github.com/tidbcloud/ti-cli/internal/apperr" ) @@ -230,20 +232,40 @@ func (c *Client) readFile(ctx context.Context, remotePath string, offset, length func (c *Client) readRedirectedFile(ctx context.Context, res *http.Response, rangeHeader string, offset, length int64, ranged bool) ([]byte, error) { defer res.Body.Close() + // res.Location's parse error repeats the raw Location value, which can carry + // a presigned signature, so it is not kept as the cause. location, err := res.Location() if err != nil { - return nil, apperr.Wrap("api.invalid_redirect", "api", 1, "ti fs returned an invalid download redirect", err) + return nil, apperr.New("api.invalid_redirect", "api", 1, "ti fs returned an invalid download redirect") } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, location.String(), nil) + req, err := apitransport.NewSecureRequest(ctx, http.MethodGet, location.String(), nil) if err != nil { - return nil, apperr.Wrap("api.build_request", "runtime", 1, "build redirected ti fs download request", err) + if errors.Is(err, apitransport.ErrInsecureURL) { + return nil, apperr.Wrap("api.insecure_redirect", "api", 1, "ti fs redirected the download to a URL that is not https", err) + } + return nil, apperr.Wrap("api.invalid_redirect", "api", 1, "ti fs returned an invalid download redirect", err) } req.Header.Set("User-Agent", c.api.UserAgent) if rangeHeader != "" { req.Header.Set("Range", rangeHeader) } - redirected, err := http.DefaultClient.Do(req) + redirected, err := apitransport.DoSecure(transferClient, req) if err != nil { + if errors.Is(err, apitransport.ErrInsecureURL) { + return nil, apperr.Wrap("api.insecure_redirect", "api", 1, "ti fs redirected the download to a URL that is not https", err) + } + if errors.Is(err, apitransport.ErrMalformedURL) { + return nil, apperr.Wrap("api.invalid_redirect", "api", 1, "ti fs returned an invalid download redirect", err) + } + if errors.Is(err, apitransport.ErrTooManyRedirects) { + return nil, &api.Error{ + Code: "api.redirect_loop", + Category: "api", + ExitCode: 1, + Message: fmt.Sprintf("API request failed: %s redirected more than %d times", req.URL.Host, apitransport.MaxRedirects), + Cause: err, + } + } return nil, &api.Error{ Code: "api.network_error", Category: "api", diff --git a/internal/api/fs/secure_transfer_test.go b/internal/api/fs/secure_transfer_test.go new file mode 100644 index 00000000..3abfa331 --- /dev/null +++ b/internal/api/fs/secure_transfer_test.go @@ -0,0 +1,307 @@ +package fs + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/tidbcloud/ti-cli/internal/api" + apitransport "github.com/tidbcloud/ti-cli/internal/api/transport" + "github.com/tidbcloud/ti-cli/internal/apperr" +) + +const insecureSignedURL = "http://bucket.example.com/signed/large.bin?X-Amz-Signature=secret-signature" + +func TestReadFileRefusesRedirectToPlainHTTP(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, insecureSignedURL, http.StatusFound) + })) + defer server.Close() + + client := testBearerClient(t, server.URL) + _, err := client.ReadFile(context.Background(), "/workspace/large.bin") + assertInsecureRedirect(t, err) +} + +func TestReadFileRefusesStorageRedirectToPlainHTTP(t *testing.T) { + storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, insecureSignedURL, http.StatusTemporaryRedirect) + })) + defer storage.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, storage.URL+"/signed/large.bin?X-Amz-Signature=first-signature", http.StatusFound) + })) + defer server.Close() + + client := testBearerClient(t, server.URL) + _, err := client.ReadFile(context.Background(), "/workspace/large.bin") + assertInsecureRedirect(t, err) + if strings.Contains(err.Error(), "first-signature") { + t.Fatalf("error leaks the first signature: %v", err) + } +} + +func TestReadFileFollowsRelativeRedirect(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/fs/workspace/large.bin" { + w.Header().Set("Location", "/signed/large.bin") + w.WriteHeader(http.StatusFound) + return + } + _, _ = w.Write([]byte("large bytes")) + })) + defer server.Close() + + client := testBearerClient(t, server.URL) + data, err := client.ReadFile(context.Background(), "/workspace/large.bin") + if err != nil { + t.Fatalf("ReadFile failed: %v", err) + } + if string(data) != "large bytes" { + t.Fatalf("unexpected redirected data %q", data) + } +} + +func TestPresignedTransfersRefusePlainHTTP(t *testing.T) { + ctx := context.Background() + calls := map[string]func(string) error{ + "uploadPresignedPartV2": func(rawURL string) error { + _, err := uploadPresignedPartV2(ctx, PresignedPart{URL: rawURL}, []byte("data")) + return err + }, + "uploadPresignedPatchPart": func(rawURL string) error { + return uploadPresignedPatchPart(ctx, rawURL, nil, []byte("data")) + }, + "uploadPresignedPartWithChecksum": func(rawURL string) error { + return uploadPresignedPartWithChecksum(ctx, rawURL, nil, []byte("data"), "", true) + }, + "readPresignedPart": func(rawURL string) error { + _, err := readPresignedPart(ctx, rawURL, nil) + return err + }, + } + redirecting := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, insecureSignedURL, http.StatusTemporaryRedirect) + })) + defer redirecting.Close() + + for name, call := range calls { + for _, rawURL := range []string{insecureSignedURL, redirecting.URL + "/part?X-Amz-Signature=first-signature"} { + err := call(rawURL) + if !errors.Is(err, apitransport.ErrInsecureURL) { + t.Errorf("%s(%s) error = %v, want ErrInsecureURL", name, rawURL, err) + continue + } + if strings.Contains(err.Error(), "signature") { + t.Errorf("%s error leaks the presigned signature: %v", name, err) + } + } + } +} + +func assertInsecureRedirect(t *testing.T, err error) { + t.Helper() + var appErr *apperr.Error + if !errors.As(err, &appErr) || appErr.Code != "api.insecure_redirect" { + t.Fatalf("error = %v, want api.insecure_redirect", err) + } + if !errors.Is(err, apitransport.ErrInsecureURL) { + t.Fatalf("error = %v, want ErrInsecureURL", err) + } + if strings.Contains(err.Error(), "secret-signature") { + t.Fatalf("error leaks the presigned signature: %v", err) + } +} + +func TestReadFileReportsRedirectLoop(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "/signed/large.bin", http.StatusFound) + })) + defer server.Close() + + client := testBearerClient(t, server.URL) + _, err := client.ReadFile(context.Background(), "/workspace/large.bin") + var apiErr *api.Error + if !errors.As(err, &apiErr) || apiErr.Code != "api.redirect_loop" { + t.Fatalf("error = %v, want api.redirect_loop", err) + } + if !errors.Is(err, apitransport.ErrTooManyRedirects) { + t.Fatalf("error = %v, want ErrTooManyRedirects", err) + } +} + +func TestReadFileMalformedRedirectDoesNotLeak(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", "https://bucket.example.com/large%zz.bin?X-Amz-Signature=secret-signature") + w.WriteHeader(http.StatusFound) + })) + defer server.Close() + + client := testBearerClient(t, server.URL) + _, err := client.ReadFile(context.Background(), "/workspace/large.bin") + var apiErr *api.Error + if !errors.As(err, &apiErr) || apiErr.Code != "api.invalid_redirect" { + t.Fatalf("error = %v, want api.invalid_redirect", err) + } + if strings.Contains(err.Error(), "secret-signature") || strings.Contains(err.Error(), "large%zz") { + t.Fatalf("error leaks the redirect URL: %v", err) + } +} + +func TestPresignedTransfersRefuseMalformedURLWithoutLeaking(t *testing.T) { + ctx := context.Background() + const malformed = "https://bucket.example.com/part%zz?X-Amz-Signature=secret-signature" + for name, call := range map[string]func() error{ + "uploadPresignedPartV2": func() error { + _, err := uploadPresignedPartV2(ctx, PresignedPart{URL: malformed}, []byte("data")) + return err + }, + "uploadPresignedPatchPart": func() error { + return uploadPresignedPatchPart(ctx, malformed, nil, []byte("data")) + }, + "uploadPresignedPartWithChecksum": func() error { + return uploadPresignedPartWithChecksum(ctx, malformed, nil, []byte("data"), "", true) + }, + "readPresignedPart": func() error { + _, err := readPresignedPart(ctx, malformed, nil) + return err + }, + } { + err := call() + if !errors.Is(err, apitransport.ErrMalformedURL) { + t.Errorf("%s error = %v, want ErrMalformedURL", name, err) + continue + } + if strings.Contains(err.Error(), "secret-signature") { + t.Errorf("%s error leaks the presigned signature: %v", name, err) + } + } +} + +// TestPresignedTransfersOverHTTPS pins the happy path: an https presigned URL +// still gets the body, the server-chosen headers (never Host), the checksum +// headers, and returns the ETag or the bytes. It swaps the package transfer +// client for one that trusts the test server, so it must not run in parallel. +func TestPresignedTransfersOverHTTPS(t *testing.T) { + type seen struct { + method, body, sha256, crc32c, custom, host string + } + var mu sync.Mutex + var got []seen + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + mu.Lock() + got = append(got, seen{ + method: r.Method, + body: string(body), + sha256: r.Header.Get("x-amz-checksum-sha256"), + crc32c: r.Header.Get("x-amz-checksum-crc32c"), + custom: r.Header.Get("x-amz-meta-test"), + host: r.Host, + }) + mu.Unlock() + if r.URL.Query().Get("X-Amz-Signature") != "sig" { + w.WriteHeader(http.StatusForbidden) + return + } + if r.Method == http.MethodGet { + _, _ = w.Write([]byte("stored bytes")) + return + } + w.Header().Set("ETag", `"etag-1"`) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + previous := transferClient + transferClient = server.Client() + t.Cleanup(func() { transferClient = previous }) + + ctx := context.Background() + rawURL := server.URL + "/bucket/part?X-Amz-Signature=sig" + headers := map[string]string{"x-amz-meta-test": "yes", "Host": "evil.example.com"} + + etag, err := uploadPresignedPartV2(ctx, PresignedPart{URL: rawURL, Headers: headers}, []byte("part one")) + if err != nil || etag != `"etag-1"` { + t.Fatalf("uploadPresignedPartV2 = %q, %v", etag, err) + } + if err := uploadPresignedPatchPart(ctx, rawURL, map[string]string{"x-amz-checksum-sha256": "placeholder", "x-amz-meta-test": "yes"}, []byte("part two")); err != nil { + t.Fatalf("uploadPresignedPatchPart: %v", err) + } + if err := uploadPresignedPartWithChecksum(ctx, rawURL, headers, []byte("part three"), "", true); err != nil { + t.Fatalf("uploadPresignedPartWithChecksum: %v", err) + } + data, err := readPresignedPart(ctx, rawURL, headers) + if err != nil || string(data) != "stored bytes" { + t.Fatalf("readPresignedPart = %q, %v", data, err) + } + + sum := sha256.Sum256([]byte("part two")) + want := []seen{ + {method: http.MethodPut, body: "part one", custom: "yes"}, + {method: http.MethodPut, body: "part two", custom: "yes", sha256: base64.StdEncoding.EncodeToString(sum[:])}, + {method: http.MethodPut, body: "part three", custom: "yes", crc32c: computeCRC32C([]byte("part three"))}, + {method: http.MethodGet, custom: "yes"}, + } + if len(got) != len(want) { + t.Fatalf("server saw %d requests, want %d: %#v", len(got), len(want), got) + } + for i := range want { + if got[i].host != strings.TrimPrefix(server.URL, "https://") { + t.Errorf("request %d used Host %q, want the URL host", i, got[i].host) + } + got[i].host = "" + if got[i] != want[i] { + t.Errorf("request %d = %#v, want %#v", i, got[i], want[i]) + } + } +} + +// A 300 response is not followed by net/http, so ReadFile parses its Location +// itself; that parse error must not carry the raw value either. +func TestReadFileMalformedMultipleChoicesLocationDoesNotLeak(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", "https://bucket.example.com/large%zz.bin?X-Amz-Signature=secret-signature") + w.WriteHeader(http.StatusMultipleChoices) + })) + defer server.Close() + + client := testBearerClient(t, server.URL) + _, err := client.ReadFile(context.Background(), "/workspace/large.bin") + var appErr *apperr.Error + if !errors.As(err, &appErr) || appErr.Code != "api.invalid_redirect" { + t.Fatalf("error = %v, want api.invalid_redirect", err) + } + if strings.Contains(err.Error(), "secret-signature") || strings.Contains(err.Error(), "large%zz") { + t.Fatalf("error leaks the redirect URL: %v", err) + } +} + +func TestReadFileMalformedStorageLocationDoesNotLeak(t *testing.T) { + storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", "https://bucket.example.com/large%zz.bin?X-Amz-Signature=secret-signature") + w.WriteHeader(http.StatusTemporaryRedirect) + })) + defer storage.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", storage.URL+"/signed/large.bin") + w.WriteHeader(http.StatusMultipleChoices) + })) + defer server.Close() + + client := testBearerClient(t, server.URL) + _, err := client.ReadFile(context.Background(), "/workspace/large.bin") + var appErr *apperr.Error + if !errors.As(err, &appErr) || appErr.Code != "api.invalid_redirect" { + t.Fatalf("error = %v, want api.invalid_redirect", err) + } + if strings.Contains(err.Error(), "secret-signature") { + t.Fatalf("error leaks the presigned signature: %v", err) + } +} diff --git a/internal/api/fs/upload.go b/internal/api/fs/upload.go index 5e5617da..5b10df3e 100644 --- a/internal/api/fs/upload.go +++ b/internal/api/fs/upload.go @@ -21,6 +21,7 @@ import ( "unicode/utf8" tiapi "github.com/tidbcloud/ti-cli/internal/api" + apitransport "github.com/tidbcloud/ti-cli/internal/api/transport" "github.com/tidbcloud/ti-cli/internal/apperr" ) @@ -951,12 +952,19 @@ func uploadPresignedPart(ctx context.Context, rawURL string, headers map[string] var errPresignExpired = errors.New("presigned URL expired") +// transferClient carries file bytes to and from the presigned and redirected +// object-store URLs the ti fs server returns. Those URLs, and every redirect they +// lead to, must be https (http only to loopback for local tests). DoSecure +// applies the same policy again on every call; wrapping here is a second line of +// defense for any code that sends with this client directly. +var transferClient = apitransport.SecureTransferClient(nil) + func IsPresignExpired(err error) bool { return errors.Is(err, errPresignExpired) } func uploadPresignedPartV2(ctx context.Context, part PresignedPart, data []byte) (string, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodPut, part.URL, bytes.NewReader(data)) + req, err := apitransport.NewSecureRequest(ctx, http.MethodPut, part.URL, bytes.NewReader(data)) if err != nil { return "", err } @@ -967,7 +975,7 @@ func uploadPresignedPartV2(ctx context.Context, part PresignedPart, data []byte) req.Header.Set(k, v) } req.ContentLength = int64(len(data)) - res, err := http.DefaultClient.Do(req) + res, err := apitransport.DoSecure(transferClient, req) if err != nil { return "", err } @@ -985,7 +993,7 @@ func uploadPresignedPartV2(ctx context.Context, part PresignedPart, data []byte) } func uploadPresignedPatchPart(ctx context.Context, rawURL string, headers map[string]string, data []byte) error { - req, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, bytes.NewReader(data)) + req, err := apitransport.NewSecureRequest(ctx, http.MethodPut, rawURL, bytes.NewReader(data)) if err != nil { return err } @@ -1004,7 +1012,7 @@ func uploadPresignedPatchPart(ctx context.Context, rawURL string, headers map[st sum := sha256.Sum256(data) req.Header.Set("x-amz-checksum-sha256", base64.StdEncoding.EncodeToString(sum[:])) } - res, err := http.DefaultClient.Do(req) + res, err := apitransport.DoSecure(transferClient, req) if err != nil { return err } @@ -1018,7 +1026,7 @@ func uploadPresignedPatchPart(ctx context.Context, rawURL string, headers map[st } func uploadPresignedPartWithChecksum(ctx context.Context, rawURL string, headers map[string]string, data []byte, checksumCRC32C string, sendChecksum bool) error { - req, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, bytes.NewReader(data)) + req, err := apitransport.NewSecureRequest(ctx, http.MethodPut, rawURL, bytes.NewReader(data)) if err != nil { return err } @@ -1035,7 +1043,7 @@ func uploadPresignedPartWithChecksum(ctx context.Context, rawURL string, headers } req.Header.Set("x-amz-checksum-crc32c", checksumCRC32C) } - res, err := http.DefaultClient.Do(req) + res, err := apitransport.DoSecure(transferClient, req) if err != nil { return err } @@ -1049,7 +1057,7 @@ func uploadPresignedPartWithChecksum(ctx context.Context, rawURL string, headers } func readPresignedPart(ctx context.Context, rawURL string, headers map[string]string) ([]byte, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + req, err := apitransport.NewSecureRequest(ctx, http.MethodGet, rawURL, nil) if err != nil { return nil, err } @@ -1059,7 +1067,7 @@ func readPresignedPart(ctx context.Context, rawURL string, headers map[string]st } req.Header.Set(k, v) } - res, err := http.DefaultClient.Do(req) + res, err := apitransport.DoSecure(transferClient, req) if err != nil { return nil, err } diff --git a/internal/api/transport/secure.go b/internal/api/transport/secure.go new file mode 100644 index 00000000..85e2aeb6 --- /dev/null +++ b/internal/api/transport/secure.go @@ -0,0 +1,169 @@ +package transport + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" +) + +// ErrInsecureURL reports that a transfer URL, or a redirect it led to, is not +// https. File bytes, export dumps, and release artifacts must not travel in +// cleartext, whatever URL a server hands back. +var ErrInsecureURL = errors.New("refused a transfer URL that is not https") + +// ErrMalformedURL reports a transfer URL that does not parse. It never carries +// the URL itself: url.Parse and http.NewRequest repeat the whole raw URL in their +// errors, presigned signature included. +var ErrMalformedURL = errors.New("refused a malformed transfer URL") + +// CheckSecureURL accepts https, and plain http only for localhost or a loopback +// IP (local test servers). It refuses user info in the URL. The error names only +// scheme://host, so a presigned signature in the query never reaches a message. +func CheckSecureURL(u *url.URL) error { + if u == nil || u.Opaque != "" || u.Hostname() == "" { + return fmt.Errorf("%w: missing host", ErrInsecureURL) + } + if u.User != nil { + return fmt.Errorf("%w: %s carries user info", ErrInsecureURL, Origin(u)) + } + switch strings.ToLower(u.Scheme) { + case "https": + return nil + case "http": + if isLoopbackHost(u.Hostname()) { + return nil + } + } + return fmt.Errorf("%w: %s", ErrInsecureURL, Origin(u)) +} + +// SecureRedirectThen returns a CheckRedirect policy that refuses any hop to a URL +// CheckSecureURL rejects, refuses any hop from https to plain http (loopback +// included, so a remote https server can't point a transfer at a local http +// service), keeps net/http's limit of MaxRedirects (a custom policy replaces it), +// and then defers to caller. A nil caller gets only these rules. +func SecureRedirectThen(caller func(req *http.Request, via []*http.Request) error) func(req *http.Request, via []*http.Request) error { + return func(req *http.Request, via []*http.Request) error { + if len(via) >= MaxRedirects { + return ErrTooManyRedirects + } + if req == nil { + return fmt.Errorf("%w: missing redirect target", ErrInsecureURL) + } + if err := CheckSecureURL(req.URL); err != nil { + return err + } + if len(via) > 0 && via[len(via)-1].URL != nil && + strings.EqualFold(via[len(via)-1].URL.Scheme, "https") && !strings.EqualFold(req.URL.Scheme, "https") { + return fmt.Errorf("%w: https redirect to %s", ErrInsecureURL, Origin(req.URL)) + } + if caller != nil { + return caller(req, via) + } + return nil + } +} + +// SecureTransferClient returns a copy of base (an empty client when nil) whose +// redirects must stay on https. Use it for transfers to server-chosen URLs such +// as presigned object-store URLs, which are on another origin by design. +func SecureTransferClient(base *http.Client) *http.Client { + copied := http.Client{} + if base != nil { + copied = *base + } + copied.CheckRedirect = SecureRedirectThen(copied.CheckRedirect) + return &copied +} + +// NewSecureRequest builds a request for a server-chosen transfer URL. The URL +// must parse and pass CheckSecureURL before a request exists. A URL that does +// not parse is reported as ErrMalformedURL with the parser's reason only: the +// *url.Error's URL field repeats the raw URL, but its Err (for example +// `invalid URL escape "%zz"`) does not. +func NewSecureRequest(ctx context.Context, method, rawURL string, body io.Reader) (*http.Request, error) { + u, err := url.Parse(rawURL) + if err != nil { + var urlErr *url.Error + if errors.As(err, &urlErr) && urlErr.Err != nil { + return nil, fmt.Errorf("%w: %v", ErrMalformedURL, urlErr.Err) + } + return nil, ErrMalformedURL + } + if err := CheckSecureURL(u); err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, method, rawURL, body) + if err != nil { + return nil, ErrMalformedURL + } + return req, nil +} + +// DoSecure checks req.URL with CheckSecureURL, then sends it through +// SecureTransferClient(client), so every redirect is held to the same rules +// whatever client the caller passes (a plain client, or nil, is fine). +// net/http wraps every failure in a *url.Error that repeats the full request URL, +// presigned signature included. A refused redirect comes back as the policy error +// alone; any other failure keeps its *url.Error with the URL's user info, query, +// and fragment removed. Build req with NewSecureRequest so building it can't leak +// the URL either. +func DoSecure(client *http.Client, req *http.Request) (*http.Response, error) { + if err := CheckSecureURL(req.URL); err != nil { + return nil, err + } + res, err := SecureTransferClient(client).Do(req) + if err != nil { + var urlErr *url.Error + if !errors.As(err, &urlErr) { + return nil, err + } + if errors.Is(err, ErrInsecureURL) || errors.Is(err, ErrTooManyRedirects) { + return nil, urlErr.Err + } + if IsLocationParseError(urlErr.Err) { + return nil, fmt.Errorf("%w: a redirect had an invalid Location header", ErrMalformedURL) + } + redacted := *urlErr + redacted.URL = redactURL(urlErr.URL) + return nil, &redacted + } + return res, nil +} + +// IsLocationParseError reports whether err is net/http's failure to parse a +// redirect's Location header. net/http puts the raw Location value in that +// message (and does so before CheckRedirect runs), and a redirect to object +// storage carries a presigned signature there, so the error must not be shown. +func IsLocationParseError(err error) bool { + return err != nil && strings.Contains(err.Error(), "failed to parse Location header") +} + +// redactURL drops the parts of a URL that can carry credentials: user info, the +// query (where presigned signatures live), and the fragment. +func redactURL(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return "[redacted URL]" + } + u.User = nil + u.RawQuery = "" + u.ForceQuery = false + u.Fragment = "" + u.RawFragment = "" + return u.String() +} + +// isLoopbackHost reports whether host is localhost or a loopback IP address. +func isLoopbackHost(host string) bool { + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} diff --git a/internal/api/transport/secure_test.go b/internal/api/transport/secure_test.go new file mode 100644 index 00000000..92f406f8 --- /dev/null +++ b/internal/api/transport/secure_test.go @@ -0,0 +1,312 @@ +package transport + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestCheckSecureURL(t *testing.T) { + for _, raw := range []string{ + "https://bucket.s3.amazonaws.com/key?X-Amz-Signature=abc", + "HTTPS://bucket.oss-ap-southeast-1.aliyuncs.com/key", + "http://127.0.0.1:8080/upload", + "http://localhost/upload", + "http://[::1]:9000/upload", + } { + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("parse %q: %v", raw, err) + } + if err := CheckSecureURL(u); err != nil { + t.Errorf("CheckSecureURL(%q) = %v, want nil", raw, err) + } + } + for _, raw := range []string{ + "http://bucket.s3.amazonaws.com/key?X-Amz-Signature=secret-signature", + "ftp://example.com/file", + "https://user:pass@example.com/file", + "/relative/path", + "mailto:someone@example.com", + } { + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("parse %q: %v", raw, err) + } + err = CheckSecureURL(u) + if !errors.Is(err, ErrInsecureURL) { + t.Errorf("CheckSecureURL(%q) = %v, want ErrInsecureURL", raw, err) + continue + } + for _, leaked := range []string{"secret-signature", "pass", "/key", "/file"} { + if strings.Contains(err.Error(), leaked) { + t.Errorf("CheckSecureURL(%q) error %q leaks %q", raw, err, leaked) + } + } + } + if err := CheckSecureURL(nil); !errors.Is(err, ErrInsecureURL) { + t.Fatalf("CheckSecureURL(nil) = %v, want ErrInsecureURL", err) + } +} + +func TestDoSecureRefusesInsecureURLBeforeSending(t *testing.T) { + var called atomic.Bool + client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + called.Store(true) + return nil, errors.New("must not send") + })} + req, err := http.NewRequest(http.MethodPut, "http://bucket.example.com/key?X-Amz-Signature=secret-signature", nil) + if err != nil { + t.Fatal(err) + } + _, err = DoSecure(SecureTransferClient(client), req) + if !errors.Is(err, ErrInsecureURL) { + t.Fatalf("DoSecure error = %v, want ErrInsecureURL", err) + } + if strings.Contains(err.Error(), "secret-signature") { + t.Fatalf("DoSecure error leaks the signature: %v", err) + } + if called.Load() { + t.Fatal("DoSecure sent a request to an insecure URL") + } +} + +func TestDoSecureRefusesRedirectToHTTP(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "http://bucket.example.com/key?X-Amz-Signature=secret-signature", http.StatusFound) + })) + defer server.Close() + + req, err := http.NewRequest(http.MethodGet, server.URL+"/start?X-Amz-Signature=first-signature", nil) + if err != nil { + t.Fatal(err) + } + _, err = DoSecure(SecureTransferClient(nil), req) + if !errors.Is(err, ErrInsecureURL) { + t.Fatalf("DoSecure error = %v, want ErrInsecureURL", err) + } + for _, leaked := range []string{"secret-signature", "first-signature"} { + if strings.Contains(err.Error(), leaked) { + t.Fatalf("DoSecure error %q leaks %q", err, leaked) + } + } +} + +func TestDoSecureRedactsNetworkErrors(t *testing.T) { + failure := errors.New("connection refused") + client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, failure + })} + req, err := http.NewRequest(http.MethodPut, "https://bucket.example.com/key?X-Amz-Signature=secret-signature#frag", nil) + if err != nil { + t.Fatal(err) + } + _, err = DoSecure(SecureTransferClient(client), req) + if !errors.Is(err, failure) { + t.Fatalf("DoSecure error = %v, want the transport failure", err) + } + var urlErr *url.Error + if !errors.As(err, &urlErr) || urlErr.Op != "Put" { + t.Fatalf("DoSecure error = %#v, want a *url.Error for Put", err) + } + for _, leaked := range []string{"secret-signature", "X-Amz", "frag"} { + if strings.Contains(err.Error(), leaked) { + t.Fatalf("DoSecure error %q leaks %q", err, leaked) + } + } + if !strings.Contains(err.Error(), "https://bucket.example.com/key") { + t.Fatalf("DoSecure error %q lost the scheme, host, and path", err) + } +} + +func TestDoSecureFollowsRelativeRedirect(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/start" { + w.Header().Set("Location", "/final") + w.WriteHeader(http.StatusFound) + return + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + req, err := http.NewRequest(http.MethodGet, server.URL+"/start", nil) + if err != nil { + t.Fatal(err) + } + res, err := DoSecure(SecureTransferClient(nil), req) + if err != nil { + t.Fatalf("DoSecure: %v", err) + } + _ = res.Body.Close() + if res.StatusCode != http.StatusNoContent || res.Request.URL.Path != "/final" { + t.Fatalf("got %d at %s, want 204 at /final", res.StatusCode, res.Request.URL.Path) + } +} + +func TestDoSecureStopsAfterMaxRedirects(t *testing.T) { + var hops atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hops.Add(1) + http.Redirect(w, r, "/again", http.StatusFound) + })) + defer server.Close() + + // The timeout makes a missing limit fail fast instead of redirecting forever. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL+"/start", nil) + if err != nil { + t.Fatal(err) + } + _, err = DoSecure(SecureTransferClient(nil), req) + if !errors.Is(err, ErrTooManyRedirects) { + t.Fatalf("DoSecure error = %v, want ErrTooManyRedirects", err) + } + // Like net/http's default policy: the original request plus nine redirects + // are sent, and the tenth redirect is refused. + if got := hops.Load(); got != MaxRedirects { + t.Fatalf("server saw %d requests, want %d", got, MaxRedirects) + } +} + +func TestSecureTransferClientKeepsCallerPolicy(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/start" { + http.Redirect(w, r, "/final", http.StatusFound) + return + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + base := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }} + res, err := SecureTransferClient(base).Get(server.URL + "/start") + if err != nil { + t.Fatalf("GET: %v", err) + } + _ = res.Body.Close() + if res.StatusCode != http.StatusFound { + t.Fatalf("status = %d, want the caller policy to stop at 302", res.StatusCode) + } + if base.CheckRedirect == nil { + t.Fatal("SecureTransferClient changed the base client") + } +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +func TestDoSecureHoldsAnyClientToTheRedirectRule(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "http://bucket.example.com/key?X-Amz-Signature=secret-signature", http.StatusFound) + })) + defer server.Close() + + for name, client := range map[string]*http.Client{"plain client": {}, "nil client": nil, "default client": http.DefaultClient} { + req, err := http.NewRequest(http.MethodGet, server.URL+"/start", nil) + if err != nil { + t.Fatal(err) + } + _, err = DoSecure(client, req) + if !errors.Is(err, ErrInsecureURL) { + t.Fatalf("%s: DoSecure error = %v, want ErrInsecureURL", name, err) + } + } + if http.DefaultClient.CheckRedirect != nil { + t.Fatal("DoSecure changed http.DefaultClient") + } +} + +func TestDoSecureRefusesHTTPSRedirectToLoopbackHTTP(t *testing.T) { + var localHit atomic.Bool + local := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + localHit.Store(true) + w.WriteHeader(http.StatusNoContent) + })) + defer local.Close() + remote := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, local.URL+"/admin", http.StatusTemporaryRedirect) + })) + defer remote.Close() + + req, err := http.NewRequest(http.MethodPut, remote.URL+"/part?X-Amz-Signature=secret-signature", strings.NewReader("file bytes")) + if err != nil { + t.Fatal(err) + } + _, err = DoSecure(remote.Client(), req) + if !errors.Is(err, ErrInsecureURL) { + t.Fatalf("DoSecure error = %v, want ErrInsecureURL", err) + } + if strings.Contains(err.Error(), "secret-signature") { + t.Fatalf("DoSecure error leaks the signature: %v", err) + } + if localHit.Load() { + t.Fatal("an https server redirected the transfer to a loopback http service") + } +} + +func TestNewSecureRequest(t *testing.T) { + ctx := context.Background() + if _, err := NewSecureRequest(ctx, http.MethodGet, "https://bucket.example.com/key?X-Amz-Signature=abc", nil); err != nil { + t.Fatalf("https URL: %v", err) + } + if _, err := NewSecureRequest(ctx, http.MethodGet, "http://127.0.0.1:9000/key", nil); err != nil { + t.Fatalf("loopback http URL: %v", err) + } + for raw, want := range map[string]error{ + "https://bucket.example.com/a%zz?X-Amz-Signature=secret-signature": ErrMalformedURL, + "https://bucket.example.com/a\x7f?X-Amz-Signature=secret-signature": ErrMalformedURL, + "http://bucket.example.com/key?X-Amz-Signature=secret-signature": ErrInsecureURL, + "https://user:secret-signature@bucket.example.com/key": ErrInsecureURL, + } { + req, err := NewSecureRequest(ctx, http.MethodGet, raw, nil) + if req != nil || !errors.Is(err, want) { + t.Errorf("NewSecureRequest(%q) = %v, %v; want %v", raw, req, err, want) + continue + } + if strings.Contains(err.Error(), "secret-signature") || strings.Contains(err.Error(), "/a") || strings.Contains(err.Error(), "bucket.example.com/") { + t.Errorf("NewSecureRequest(%q) error %q leaks the URL", raw, err) + } + if want == ErrMalformedURL && !strings.Contains(err.Error(), "invalid") { + t.Errorf("NewSecureRequest(%q) error %q lost the parser's reason", raw, err) + } + } +} + +func TestDoSecureDropsMalformedLocationFromLaterHop(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/start" { + http.Redirect(w, r, "/object", http.StatusFound) + return + } + w.Header().Set("Location", "https://bucket.example.com/x%zz?X-Amz-Signature=secret-signature") + w.WriteHeader(http.StatusFound) + })) + defer server.Close() + + req, err := http.NewRequest(http.MethodGet, server.URL+"/start", nil) + if err != nil { + t.Fatal(err) + } + _, err = DoSecure(nil, req) + if !errors.Is(err, ErrMalformedURL) { + t.Fatalf("DoSecure error = %v, want ErrMalformedURL", err) + } + for _, leaked := range []string{"secret-signature", "x%zz"} { + if strings.Contains(err.Error(), leaked) { + t.Fatalf("DoSecure error %q leaks %q", err, leaked) + } + } +} diff --git a/internal/db/product/starter/export.go b/internal/db/product/starter/export.go index f034f483..46947386 100644 --- a/internal/db/product/starter/export.go +++ b/internal/db/product/starter/export.go @@ -14,6 +14,7 @@ import ( "unicode" apistarter "github.com/tidbcloud/ti-cli/internal/api/starter" + apitransport "github.com/tidbcloud/ti-cli/internal/api/transport" "github.com/tidbcloud/ti-cli/internal/apperr" "github.com/tidbcloud/ti-cli/internal/authz" rootdb "github.com/tidbcloud/ti-cli/internal/db" @@ -299,11 +300,11 @@ func (s Service) downloadExportFile(ctx context.Context, outputPath string, job } func (s Service) copyExportURL(ctx context.Context, outputPath, rel, rawURL string) (int64, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + req, err := apitransport.NewSecureRequest(ctx, http.MethodGet, rawURL, nil) if err != nil { return 0, err } - resp, err := s.fileHTTPClient().Do(req) + resp, err := apitransport.DoSecure(s.fileHTTPClient(), req) if err != nil { return 0, err } @@ -338,11 +339,12 @@ func (s Service) copyExportURL(ctx context.Context, outputPath, rel, rawURL stri return written, nil } +// fileHTTPClient downloads export files from the URLs the API returns. The +// dump holds database contents, so the URL and every redirect must be https. +// DoSecure applies the same policy again; wrapping here is a second line of +// defense for any code that sends with this client directly. func (s Service) fileHTTPClient() *http.Client { - if s.HTTPClient != nil { - return s.HTTPClient - } - return &http.Client{} + return apitransport.SecureTransferClient(s.HTTPClient) } func mkdirAllInRoot(root *os.Root, rel string) error { diff --git a/internal/db/product/starter/export_secure_test.go b/internal/db/product/starter/export_secure_test.go new file mode 100644 index 00000000..5469f90b --- /dev/null +++ b/internal/db/product/starter/export_secure_test.go @@ -0,0 +1,98 @@ +package starter + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestDownloadExportedDataRefusesPlainHTTPURLs(t *testing.T) { + const insecureURL = "http://bucket.example.com/a.csv?X-Amz-Signature=secret-signature" + for name, payloadURL := range map[string]func(*http.Request) string{ + "direct": func(*http.Request) string { return insecureURL }, + "redirect": func(r *http.Request) string { return fileURL(r, "/payload/a.csv") }, + } { + t.Run(name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/v1beta1/clusters/cluster-1": + _, _ = w.Write([]byte(`{"clusterId":"cluster-1","displayName":"demo","servicePlan":"Starter"}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files"): + _, _ = w.Write([]byte(`{"files":[{"name":"a.csv","size":5}]}`)) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files:download"): + _, _ = w.Write([]byte(`{"files":[{"name":"a.csv","url":"` + payloadURL(r) + `"}]}`)) + case r.URL.Path == "/payload/a.csv": + http.Redirect(w, r, insecureURL, http.StatusFound) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + })) + defer server.Close() + + out := t.TempDir() + result, err := testService(server.URL).DownloadExportedData(context.Background(), DownloadExportedDataOptions{ + Profile: testProfile(), + ClusterID: "cluster-1", + ExportID: "export-1", + OutputPath: out, + }) + if err == nil { + t.Fatal("DownloadExportedData succeeded with a plain http URL") + } + if result.Failed != 1 || len(result.Files) != 1 { + t.Fatalf("unexpected result: %#v", result) + } + fileErr := result.Files[0].Error + if !strings.Contains(fileErr, "not https") { + t.Fatalf("file error = %q, want an https refusal", fileErr) + } + if strings.Contains(fileErr, "secret-signature") { + t.Fatalf("file error leaks the presigned signature: %q", fileErr) + } + if _, statErr := os.Stat(filepath.Join(out, "a.csv")); !os.IsNotExist(statErr) { + t.Fatalf("refused download still wrote a file: %v", statErr) + } + }) + } +} + +func TestDownloadExportedDataMalformedURLDoesNotLeak(t *testing.T) { + const malformedURL = "https://bucket.example.com/a%zz.csv?X-Amz-Signature=secret-signature" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/v1beta1/clusters/cluster-1": + _, _ = w.Write([]byte(`{"clusterId":"cluster-1","displayName":"demo","servicePlan":"Starter"}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files"): + _, _ = w.Write([]byte(`{"files":[{"name":"a.csv","size":5}]}`)) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files:download"): + _, _ = w.Write([]byte(`{"files":[{"name":"a.csv","url":"` + malformedURL + `"}]}`)) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + })) + defer server.Close() + + result, err := testService(server.URL).DownloadExportedData(context.Background(), DownloadExportedDataOptions{ + Profile: testProfile(), + ClusterID: "cluster-1", + ExportID: "export-1", + OutputPath: t.TempDir(), + }) + if err == nil || result.Failed != 1 || len(result.Files) != 1 { + t.Fatalf("DownloadExportedData = %#v, %v; want one failed file", result, err) + } + fileErr := result.Files[0].Error + if !strings.Contains(fileErr, "malformed") { + t.Fatalf("file error = %q, want a malformed URL refusal", fileErr) + } + if strings.Contains(fileErr, "secret-signature") || strings.Contains(fileErr, "a%zz") { + t.Fatalf("file error leaks the URL: %q", fileErr) + } + if strings.Contains(err.Error(), "secret-signature") { + t.Fatalf("error leaks the URL: %v", err) + } +} diff --git a/internal/telemetrybackend/config.go b/internal/telemetrybackend/config.go index bb42a659..7549c15e 100644 --- a/internal/telemetrybackend/config.go +++ b/internal/telemetrybackend/config.go @@ -135,14 +135,39 @@ func (c Config) Validate() error { if c.RateLimitPerMinute <= 0 || c.RateLimitBurst <= 0 { return fmt.Errorf("telemetry rate limits must be positive") } - if c.TiDBDSN == "" { + return ValidateTiDBDSN(c.Environment, c.TiDBDSN) +} + +// nonProductionEnvironments are the only TELEMETRY_ENVIRONMENT values that may +// skip the verified TLS rule. Every other value, including a typo such as +// "prod" and remote ones such as "staging", is held to the production rule, so a +// mistake fails closed. +var nonProductionEnvironments = map[string]bool{ + "development": true, + "local": true, + "telemetry-e2e": true, + "test": true, +} + +// IsProductionEnvironment reports whether the verified TLS rule applies to a +// TELEMETRY_ENVIRONMENT value: an empty value and any value outside +// nonProductionEnvironments count as production. +func IsProductionEnvironment(environment string) bool { + return !nonProductionEnvironments[strings.ToLower(strings.TrimSpace(environment))] +} + +// ValidateTiDBDSN checks TIDB_DSN the same way for the API server and the +// migration runner: it must parse, and in production (see +// IsProductionEnvironment) it must use verified TLS. +func ValidateTiDBDSN(environment, rawDSN string) error { + if rawDSN == "" { return fmt.Errorf("TIDB_DSN is required") } - dsn, err := mysql.ParseDSN(c.TiDBDSN) + dsn, err := mysql.ParseDSN(rawDSN) if err != nil { return fmt.Errorf("TIDB_DSN is invalid") } - if strings.EqualFold(c.Environment, "production") { + if IsProductionEnvironment(environment) { switch strings.ToLower(dsn.TLSConfig) { case "", "false", "skip-verify", "preferred": return fmt.Errorf("TIDB_DSN must enable verified TLS in production") diff --git a/internal/telemetrybackend/config_test.go b/internal/telemetrybackend/config_test.go index 0c41040d..b694685f 100644 --- a/internal/telemetrybackend/config_test.go +++ b/internal/telemetrybackend/config_test.go @@ -57,3 +57,45 @@ func TestLoadConfigRejectsInvalidAndInconsistentLimits(t *testing.T) { t.Fatalf("LoadConfig error = %v, want inconsistent limit error", err) } } + +func TestValidateTiDBDSN(t *testing.T) { + const ( + verified = "user:password@tcp(localhost:4000)/telemetry?tls=true" + unverified = "user:password@tcp(localhost:4000)/telemetry?tls=skip-verify" + plain = "user:password@tcp(localhost:4000)/telemetry" + ) + for _, tc := range []struct { + name string + environment string + dsn string + wantErr string + }{ + {"production with verified TLS", "production", verified, ""}, + {"empty environment means production", "", unverified, "verified TLS"}, + {"blank environment means production", " ", plain, "verified TLS"}, + {"production without TLS", "Production", plain, "verified TLS"}, + {"non-production without TLS", "telemetry-e2e", plain, ""}, + {"test without TLS", " Test ", unverified, ""}, + {"local without TLS", "local", plain, ""}, + {"development without TLS", "development", plain, ""}, + {"typo prod is production", "prod", unverified, "verified TLS"}, + {"typo prduction is production", "prduction", plain, "verified TLS"}, + {"staging is production", "staging", unverified, "verified TLS"}, + {"unknown name is production", "qa-east", plain, "verified TLS"}, + {"missing DSN", "telemetry-e2e", "", "required"}, + {"invalid DSN", "telemetry-e2e", "not a dsn", "invalid"}, + } { + t.Run(tc.name, func(t *testing.T) { + err := ValidateTiDBDSN(tc.environment, tc.dsn) + if tc.wantErr == "" { + if err != nil { + t.Fatalf("ValidateTiDBDSN = %v, want nil", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("ValidateTiDBDSN = %v, want error containing %q", err, tc.wantErr) + } + }) + } +} diff --git a/internal/update/secure_download_test.go b/internal/update/secure_download_test.go new file mode 100644 index 00000000..b5420d80 --- /dev/null +++ b/internal/update/secure_download_test.go @@ -0,0 +1,75 @@ +package update + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + apitransport "github.com/tidbcloud/ti-cli/internal/api/transport" + "github.com/tidbcloud/ti-cli/internal/apperr" + "github.com/tidbcloud/ti-cli/internal/version" +) + +func TestDownloadRefusesPlainHTTP(t *testing.T) { + const insecureURL = "http://releases.example.com/ti.tar.gz?token=secret-token" + redirecting := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, insecureURL, http.StatusFound) + })) + defer redirecting.Close() + + c := newClient(version.Info{}, "https://releases.invalid", nil) + for _, rawURL := range []string{insecureURL, redirecting.URL + "/ti.tar.gz"} { + _, err := c.download(context.Background(), rawURL) + var appErr *apperr.Error + if !errors.As(err, &appErr) || appErr.Code != "update.insecure_url" { + t.Fatalf("download(%s) error = %v, want update.insecure_url", rawURL, err) + } + if !errors.Is(err, apitransport.ErrInsecureURL) { + t.Fatalf("download(%s) error = %v, want ErrInsecureURL", rawURL, err) + } + if strings.Contains(err.Error(), "secret-token") { + t.Fatalf("download error leaks the query: %v", err) + } + } +} + +func TestDownloadDrive9CompanionRefusesPlainHTTPBaseURL(t *testing.T) { + c := newClient(version.Info{}, "https://releases.invalid", nil) + _, cleanup, err := c.downloadDrive9Companion(context.Background(), "http://drive9.example.com/releases/latest") + cleanup() + if !errors.Is(err, apitransport.ErrInsecureURL) { + t.Fatalf("downloadDrive9Companion error = %v, want ErrInsecureURL", err) + } +} + +func TestDownloadMalformedURLDoesNotLeak(t *testing.T) { + c := newClient(version.Info{}, "https://releases.invalid", nil) + _, err := c.download(context.Background(), "https://releases.example.com/ti%zz.tar.gz?token=secret-token") + var appErr *apperr.Error + if !errors.As(err, &appErr) || appErr.Code != "update.invalid_url" { + t.Fatalf("download error = %v, want update.invalid_url", err) + } + if !errors.Is(err, apitransport.ErrMalformedURL) { + t.Fatalf("download error = %v, want ErrMalformedURL", err) + } + if strings.Contains(err.Error(), "secret-token") { + t.Fatalf("download error leaks the URL: %v", err) + } +} + +func TestDownloadReportsRedirectLoop(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "/again", http.StatusFound) + })) + defer server.Close() + + c := newClient(version.Info{}, "https://releases.invalid", nil) + _, err := c.download(context.Background(), server.URL+"/ti.tar.gz") + var appErr *apperr.Error + if !errors.As(err, &appErr) || appErr.Code != "update.redirect_loop" { + t.Fatalf("download error = %v, want update.redirect_loop", err) + } +} diff --git a/internal/update/update.go b/internal/update/update.go index 7a952cad..8f9eeb29 100644 --- a/internal/update/update.go +++ b/internal/update/update.go @@ -12,7 +12,6 @@ import ( "errors" "fmt" "io" - "net" "net/http" "net/url" "os" @@ -23,6 +22,7 @@ import ( "strings" "time" + apitransport "github.com/tidbcloud/ti-cli/internal/api/transport" "github.com/tidbcloud/ti-cli/internal/apperr" "github.com/tidbcloud/ti-cli/internal/credentialenv" "github.com/tidbcloud/ti-cli/internal/version" @@ -326,12 +326,14 @@ func newClient(info version.Info, baseURL string, httpClient *http.Client) clien if strings.TrimSpace(baseURL) == "" { baseURL = DefaultReleaseAPIBaseURL } - if httpClient == nil { - httpClient = http.DefaultClient - } + // Release metadata, checksums, the ti archive, and the ti-drive9 companion all + // come through this client, and the archive and companion get executed, so + // every URL and redirect must be https (http only to loopback for local tests). + // DoSecure applies the same policy again; wrapping here is a second line of + // defense for any code that sends with this client directly. return client{ baseURL: strings.TrimRight(baseURL, "/"), - httpClient: httpClient, + httpClient: apitransport.SecureTransferClient(httpClient), } } @@ -349,21 +351,10 @@ func testReleaseAPIBaseURL(info version.Info) string { return "" } parsed, err := url.Parse(value) - if err != nil || parsed.Host == "" { + if err != nil || apitransport.CheckSecureURL(parsed) != nil { return "" } - if parsed.Scheme == "https" || (parsed.Scheme == "http" && isLoopbackHost(parsed.Hostname())) { - return value - } - return "" -} - -func isLoopbackHost(host string) bool { - if strings.EqualFold(host, "localhost") { - return true - } - ip := net.ParseIP(host) - return ip != nil && ip.IsLoopback() + return value } func (c client) fetchRelease(ctx context.Context, target string) (release, error) { @@ -393,14 +384,26 @@ func (c client) getJSON(ctx context.Context, url string, dst any) error { } func (c client) download(ctx context.Context, url string) ([]byte, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + req, err := apitransport.NewSecureRequest(ctx, http.MethodGet, url, nil) if err != nil { + if errors.Is(err, apitransport.ErrInsecureURL) { + return nil, apperr.Wrap("update.insecure_url", "runtime", 1, "download release metadata or artifact", err) + } return nil, apperr.Wrap("update.invalid_url", "runtime", 1, "build release request", err) } req.Header.Set("User-Agent", "ti") req.Header.Set("Accept", "application/json, application/octet-stream") - resp, err := c.httpClient.Do(req) + resp, err := apitransport.DoSecure(c.httpClient, req) if err != nil { + if errors.Is(err, apitransport.ErrInsecureURL) { + return nil, apperr.Wrap("update.insecure_url", "runtime", 1, "download release metadata or artifact", err) + } + if errors.Is(err, apitransport.ErrMalformedURL) { + return nil, apperr.Wrap("update.invalid_url", "runtime", 1, "release download sent an invalid redirect", err) + } + if errors.Is(err, apitransport.ErrTooManyRedirects) { + return nil, apperr.Wrap("update.redirect_loop", "runtime", 1, "release download redirected too many times", err) + } return nil, apperr.Wrap("update.network_error", "runtime", 1, "download release metadata or artifact", err) } defer resp.Body.Close() diff --git a/internal/update/update_test.go b/internal/update/update_test.go index 82bb18bc..f09dada7 100644 --- a/internal/update/update_test.go +++ b/internal/update/update_test.go @@ -562,6 +562,7 @@ func TestReleaseAPIBaseURLOverrideNeedsLocalBuildAndOptIn(t *testing.T) { {"local rejects plain http to a remote host", "local", "1", "http://releases.example.com", DefaultReleaseAPIBaseURL}, {"local rejects other schemes", "local", "1", "ftp://releases.example.com", DefaultReleaseAPIBaseURL}, {"local rejects a value with no host", "local", "1", "https://", DefaultReleaseAPIBaseURL}, + {"local rejects user info", "local", "1", "https://user:pass@releases.example.com", DefaultReleaseAPIBaseURL}, } { t.Run(tc.name, func(t *testing.T) { t.Setenv("TI_ALLOW_TEST_ENDPOINTS", tc.allow) diff --git a/scripts/install.ps1 b/scripts/install.ps1 index 193db767..0fb20430 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -205,6 +205,29 @@ if ((Test-Path $Target) -and -not $Yes) { } } +# Every download must stay on https, redirects included: the archive, its +# checksums and the companion all come through here, and the checksums travel +# over the same channel, so one cleartext hop could replace both. PowerShell 7.4+ +# already refuses an https-to-http redirect; this also covers Windows PowerShell +# 5.1 by checking the final URL, and deletes a file that arrived any other way. +function Save-HttpsDownload([string]$Uri, [string]$OutFile) { + $Response = Invoke-WebRequest -Uri $Uri -OutFile $OutFile -PassThru -UseBasicParsing + $FinalUri = $null + $Base = $Response.BaseResponse + # Check the property exists first, so this also works under Set-StrictMode. + if ($Base.PSObject.Properties['ResponseUri']) { + # Windows PowerShell 5.1: HttpWebResponse + $FinalUri = $Base.ResponseUri + } elseif ($Base.PSObject.Properties['RequestMessage']) { + # PowerShell 7: HttpResponseMessage + $FinalUri = $Base.RequestMessage.RequestUri + } + if (-not $FinalUri -or $FinalUri.Scheme -ne "https") { + Remove-Item -Force -ErrorAction SilentlyContinue -LiteralPath $OutFile + Fail "refused a download that did not stay on https: $Uri" + } +} + $TempDir = New-Item -ItemType Directory -Path (Join-Path ([System.IO.Path]::GetTempPath()) ("ti-install-" + [System.Guid]::NewGuid().ToString())) try { @@ -212,10 +235,10 @@ try { $ChecksumsPath = Join-Path $TempDir.FullName "ti_checksums.txt" $CompanionPath = Join-Path $TempDir.FullName $CompanionArtifact $CompanionChecksumsPath = Join-Path $TempDir.FullName "drive9_checksums.txt" - Invoke-WebRequest -Uri $ArchiveUrl -OutFile $ArchivePath - Invoke-WebRequest -Uri $ChecksumsUrl -OutFile $ChecksumsPath - Invoke-WebRequest -Uri $CompanionUrl -OutFile $CompanionPath - Invoke-WebRequest -Uri $CompanionChecksumsUrl -OutFile $CompanionChecksumsPath + Save-HttpsDownload $ArchiveUrl $ArchivePath + Save-HttpsDownload $ChecksumsUrl $ChecksumsPath + Save-HttpsDownload $CompanionUrl $CompanionPath + Save-HttpsDownload $CompanionChecksumsUrl $CompanionChecksumsPath $checksumLine = Get-Content $ChecksumsPath | Where-Object { $_ -match "\s+$([regex]::Escape($Artifact))$" } | Select-Object -First 1 if (-not $checksumLine) { diff --git a/scripts/install.sh b/scripts/install.sh index 314901a8..ca8f2310 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -83,21 +83,20 @@ need() { need tar +# Every download must stay on https, redirects included: the archive, its +# checksums and the companion all come through here, and the checksums travel +# over the same channel, so one cleartext hop could replace both. curl enforces +# that with --proto/--proto-redir. wget has no equivalent (--https-only applies +# only to recursive downloads), so it is not used. download() { if command -v curl >/dev/null 2>&1; then if [ -t 2 ]; then - curl -fSL --progress-bar -o "$2" "$1" + curl -fSL --proto '=https' --proto-redir '=https' --progress-bar -o "$2" "$1" else - curl -fsSL -o "$2" "$1" - fi - elif command -v wget >/dev/null 2>&1; then - if [ -t 2 ]; then - wget --show-progress -q -O "$2" "$1" 2>&1 - else - wget -q -O "$2" "$1" + curl -fsSL --proto '=https' --proto-redir '=https' -o "$2" "$1" fi else - error "neither curl nor wget found" + error "curl is required to download ti over https only; install curl and retry" fi }