From 69e2b3c86455adb52f4b8ae27615fb7853c5769a Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 7 Oct 2026 14:54:03 -0700 Subject: [PATCH 01/22] Optimize verified DRS downloads and cache checkout --- cmd/clean/main.go | 2 +- internal/lfs/repo_paths_test.go | 69 +++++++++++++++++ .../docker_syfon_e2e_helpers_test.go | 6 +- .../docker_syfon/docker_syfon_e2e_test.go | 74 ++++++++++++++----- 4 files changed, 131 insertions(+), 20 deletions(-) diff --git a/cmd/clean/main.go b/cmd/clean/main.go index fb895750..6ff55efa 100644 --- a/cmd/clean/main.go +++ b/cmd/clean/main.go @@ -27,7 +27,7 @@ var Cmd = &cobra.Command{ Use: "clean -- ", Short: "Clean a file by converting its content to an LFS pointer (invoked by git)", Long: `git drs clean reads raw file content from stdin, hashes it, stores it in -the local object cache (.git/lfs/objects), and writes an LFS pointer +the configured local LFS object cache, and writes an LFS pointer to stdout. It also records a DRS map entry so that 'git drs push' can upload the object to the configured DRS server. diff --git a/internal/lfs/repo_paths_test.go b/internal/lfs/repo_paths_test.go index abb003cc..da78453e 100644 --- a/internal/lfs/repo_paths_test.go +++ b/internal/lfs/repo_paths_test.go @@ -146,6 +146,75 @@ func TestResolveLFSRoot_ConfigRelative(t *testing.T) { } } +func TestGetGitRootDirectoriesFromRepositorySubdirectory(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not found in PATH") + } + + repo := t.TempDir() + mustRun(t, repo, "git", "init") + canonicalGitDir, err := filepath.EvalSymlinks(filepath.Join(repo, ".git")) + if err != nil { + t.Fatalf("resolve temporary git directory path: %v", err) + } + subdir := filepath.Join(repo, "nested", "package") + if err := os.MkdirAll(subdir, 0o755); err != nil { + t.Fatalf("mkdir repository subdirectory: %v", err) + } + + oldwd := mustChdir(t, subdir) + t.Cleanup(func() { _ = os.Chdir(oldwd) }) + + gitCommonDir, lfsRoot, err := GetGitRootDirectories(context.Background()) + if err != nil { + t.Fatalf("GetGitRootDirectories from subdirectory: %v", err) + } + if want := canonicalGitDir; gitCommonDir != want { + t.Errorf("git common dir: want %q, got %q", want, gitCommonDir) + } + if want := filepath.Join(canonicalGitDir, "lfs"); lfsRoot != want { + t.Errorf("LFS root: want %q, got %q", want, lfsRoot) + } +} + +func TestGetGitRootDirectoriesFromWorktreeSubdirectoryWithConfiguredStorage(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not found in PATH") + } + + repo := t.TempDir() + mustRun(t, repo, "git", "init") + canonicalGitDir, err := filepath.EvalSymlinks(filepath.Join(repo, ".git")) + if err != nil { + t.Fatalf("resolve temporary git directory path: %v", err) + } + mustRun(t, repo, "git", "config", "user.name", "git-drs test") + mustRun(t, repo, "git", "config", "user.email", "git-drs-test@example.invalid") + mustRun(t, repo, "git", "config", "lfs.storage", "shared-lfs") + mustRun(t, repo, "git", "-c", "user.name=git-drs test", "-c", "user.email=git-drs-test@example.invalid", "commit", "--allow-empty", "-m", "initial") + + worktree := filepath.Join(t.TempDir(), "linked-worktree") + mustRun(t, repo, "git", "worktree", "add", worktree, "HEAD") + subdir := filepath.Join(worktree, "nested", "package") + if err := os.MkdirAll(subdir, 0o755); err != nil { + t.Fatalf("mkdir worktree subdirectory: %v", err) + } + + oldwd := mustChdir(t, subdir) + t.Cleanup(func() { _ = os.Chdir(oldwd) }) + + gitCommonDir, lfsRoot, err := GetGitRootDirectories(context.Background()) + if err != nil { + t.Fatalf("GetGitRootDirectories from linked worktree subdirectory: %v", err) + } + if want := canonicalGitDir; gitCommonDir != want { + t.Errorf("git common dir: want shared directory %q, got %q", want, gitCommonDir) + } + if want := filepath.Join(canonicalGitDir, "shared-lfs"); lfsRoot != want { + t.Errorf("LFS root: want configured path under shared git directory %q, got %q", want, lfsRoot) + } +} + // --- test helpers --- func mustRun(t *testing.T, dir string, name string, args ...string) { diff --git a/tests/integration/docker_syfon/docker_syfon_e2e_helpers_test.go b/tests/integration/docker_syfon/docker_syfon_e2e_helpers_test.go index 456ffca8..37c804dc 100644 --- a/tests/integration/docker_syfon/docker_syfon_e2e_helpers_test.go +++ b/tests/integration/docker_syfon/docker_syfon_e2e_helpers_test.go @@ -53,7 +53,11 @@ func runCommandMust(t *testing.T, dir string, extraEnv []string, name string, ar func runCommandOutput(t *testing.T, dir string, extraEnv []string, name string, args ...string) (string, error) { t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + timeout := 2 * time.Minute + if os.Getenv("SYFON_E2E_LARGE_FILE_MIB") != "" { + timeout = time.Hour + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() cmd := exec.CommandContext(ctx, name, args...) diff --git a/tests/integration/docker_syfon/docker_syfon_e2e_test.go b/tests/integration/docker_syfon/docker_syfon_e2e_test.go index befc8c96..1436387d 100644 --- a/tests/integration/docker_syfon/docker_syfon_e2e_test.go +++ b/tests/integration/docker_syfon/docker_syfon_e2e_test.go @@ -8,8 +8,10 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "io" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -78,11 +80,29 @@ func TestGitDrsDockerMinIOE2E(t *testing.T) { t.Fatalf("write source file: %v", err) } largePath := filepath.Join(repoDir, "data", "multipart.bin") - largeData := bytes.Repeat([]byte("syfon-git-drs-multipart-e2e-"), 7*1024*1024/len("syfon-git-drs-multipart-e2e-")+1) - largeData = largeData[:7*1024*1024] - if err := os.WriteFile(largePath, largeData, 0o644); err != nil { - t.Fatalf("write multipart file: %v", err) + largeMiB := int64(7) + if configured := os.Getenv("SYFON_E2E_LARGE_FILE_MIB"); configured != "" { + largeMiB, err = strconv.ParseInt(configured, 10, 64) + if err != nil || largeMiB < 1 { + t.Fatalf("invalid SYFON_E2E_LARGE_FILE_MIB: %q", configured) + } + } + largeSize := largeMiB * 1024 * 1024 + block := bytes.Repeat([]byte("syfon-git-drs-multipart-e2e-"), 1024*1024/len("syfon-git-drs-multipart-e2e-")+1)[:1024*1024] + largeFile, err := os.Create(largePath) + if err != nil { + t.Fatalf("create multipart file: %v", err) } + for i := int64(0); i < largeMiB; i++ { + if _, err := largeFile.Write(block); err != nil { + largeFile.Close() + t.Fatalf("write multipart block %d: %v", i, err) + } + } + if err := largeFile.Close(); err != nil { + t.Fatalf("close multipart file: %v", err) + } + largeSum := hashFile(t, largePath) runCommand(t, repoDir, nil, "git", "drs", "track", "*.txt", "*.bin") runCommand(t, repoDir, nil, "git", "config", "--local", "drs.multipart-threshold", fmt.Sprintf("%d", dockerE2EMultipartMB)) runCommand(t, repoDir, nil, "git", "add", ".gitattributes", "data/source.txt", "data/multipart.bin") @@ -128,13 +148,15 @@ func TestGitDrsDockerMinIOE2E(t *testing.T) { runCommand(t, cloneDir, nil, "git", "config", "--local", "drs.multipart-threshold", fmt.Sprintf("%d", dockerE2EMultipartMB)) logRepoSnapshot(t, cloneDir, "pre-pull") - restoreLargeObject := temporarilyRemoveMinIOObject(t, minioEnv.s3Client, minioEnv.bucket, largeDid, int64(len(largeData))) - if out, err := runCommandOutput(t, cloneDir, nil, "git", "drs", "pull", "origin"); err == nil { - t.Fatalf("expected first multipart pull to fail, but it succeeded:\n%s", out) - } else { - t.Logf("expected first multipart pull failure captured for retry path") + if largeMiB == 7 { + restoreLargeObject := temporarilyRemoveMinIOObject(t, minioEnv.s3Client, minioEnv.bucket, largeDid, largeSize) + if out, err := runCommandOutput(t, cloneDir, nil, "git", "drs", "pull", "origin"); err == nil { + t.Fatalf("expected first multipart pull to fail, but it succeeded:\n%s", out) + } else { + t.Logf("expected first multipart pull failure captured for retry path") + } + restoreLargeObject() } - restoreLargeObject() runCommand(t, cloneDir, nil, "git", "drs", "pull", "origin") logRepoSnapshot(t, cloneDir, "post-pull") @@ -145,17 +167,18 @@ func TestGitDrsDockerMinIOE2E(t *testing.T) { if !bytes.Equal(got, smallData) { t.Fatalf("pulled bytes mismatch: got %q want %q", string(got), string(smallData)) } - gotLarge, err := os.ReadFile(filepath.Join(cloneDir, "data", "multipart.bin")) - if err != nil { - t.Fatalf("read pulled multipart file: %v", err) + gotLargePath := filepath.Join(cloneDir, "data", "multipart.bin") + gotLargeInfo, err := os.Stat(gotLargePath) + if err != nil || gotLargeInfo.Size() != largeSize { + t.Fatalf("pulled multipart size mismatch: info=%v err=%v want=%d", gotLargeInfo, err, largeSize) } - if !bytes.Equal(gotLarge, largeData) { - t.Fatalf("pulled multipart bytes mismatch: got %d bytes want %d bytes", len(gotLarge), len(largeData)) + if gotLargeSum := hashFile(t, gotLargePath); gotLargeSum != largeSum { + t.Fatalf("pulled multipart hash mismatch: got %x want %x", gotLargeSum, largeSum) } verifyProviderTransferMetrics(t, server.url, minioEnv, []providerTransferLogEvent{ newProviderDownloadEvent("docker-minio-download-small-"+smallDid, minioEnv, smallDid, int64(len(smallData))), - newProviderDownloadEvent("docker-minio-download-large-"+largeDid, minioEnv, largeDid, int64(len(largeData))), - }, int64(len(smallData)+len(largeData))) + newProviderDownloadEvent("docker-minio-download-large-"+largeDid, minioEnv, largeDid, largeSize), + }, int64(len(smallData))+largeSize) smallSum := sha256.Sum256(smallData) smallSumHex := hex.EncodeToString(smallSum[:]) @@ -164,7 +187,6 @@ func TestGitDrsDockerMinIOE2E(t *testing.T) { t.Fatalf("small file checksum lookup mismatch: expected DID %s and hash %s in output %q", smallDid, smallSumHex, smallHashOut) } - largeSum := sha256.Sum256(largeData) largeSumHex := hex.EncodeToString(largeSum[:]) largeHashOut := runCommand(t, cloneDir, nil, "git", "drs", "query", "--remote", "origin", "--checksum", "--pretty", largeSumHex) if !strings.Contains(largeHashOut, largeDid) || !strings.Contains(largeHashOut, largeSumHex) { @@ -181,6 +203,22 @@ func TestGitDrsDockerMinIOE2E(t *testing.T) { assertDRSRecordExists(t, server.url, smallDid) } +func hashFile(t *testing.T, path string) [32]byte { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + t.Fatal(err) + } + var sum [32]byte + copy(sum[:], h.Sum(nil)) + return sum +} + func TestGitDrsDockerAddURLE2E(t *testing.T) { if strings.TrimSpace(os.Getenv(dockerE2EEnvVar)) != "1" { t.Skipf("set %s=1 to run the Docker-backed MinIO integration test", dockerE2EEnvVar) From d77f5986097dd885dc216294b84d771811dd085e Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 7 Oct 2026 15:34:57 -0700 Subject: [PATCH 02/22] Fix Terra DRS URI pull and resumable downloads --- cmd/pull/main.go | 30 ++- cmd/pull/pull_test.go | 383 +++++++++++++++++++++++++++++ internal/resolver/resolver.go | 87 +++++-- internal/resolver/resolver_test.go | 27 +- 4 files changed, 480 insertions(+), 47 deletions(-) diff --git a/cmd/pull/main.go b/cmd/pull/main.go index 1a61f3ce..9c13add3 100644 --- a/cmd/pull/main.go +++ b/cmd/pull/main.go @@ -43,6 +43,9 @@ var ( newRemoteClient = func(cfg *config.Config, remote config.Remote, logger *slog.Logger) (*remoteruntime.GitContext, error) { return remoteruntime.New(cfg, remote, logger) } + newAnVILResolver = func(ctx context.Context, endpoint string) (resolver.Resolver, error) { + return resolver.NewAnVIL(ctx, endpoint) + } loadWorktreeInventory = lfs.GetTrackedLfsFilesAt loadWorktreeFile = lfs.GetTrackedLfsFileAt ) @@ -150,7 +153,7 @@ var Cmd = &cobra.Command{ return fmt.Errorf("remote %q does not support resolving and downloading DRS objects", remote) } if drsCtx.IsReadOnly() { - anvil, err = resolver.NewAnVIL(ctx, drsCtx.Endpoint) + anvil, err = newAnVILResolver(ctx, drsCtx.Endpoint) if err != nil { return err } @@ -220,7 +223,6 @@ var Cmd = &cobra.Command{ prefetched[oid] = *obj prefetchedAccess[obj.Id] = access } else { - progress.OnStage("Looking up DRS records") checksumOIDs := make([]string, 0, len(missingOIDs)) for _, oid := range missingOIDs { if lfs.IsDRSURI(oid) { @@ -235,16 +237,22 @@ var Cmd = &cobra.Command{ } checksumOIDs = append(checksumOIDs, oid) } - recsByOID, err := lookup.ObjectsByHashesForScope(ctx, drsCtx, checksumOIDs) - if err != nil { - return fmt.Errorf("look up DRS records by checksum: %w", err) - } - for _, oid := range checksumOIDs { - recs := recsByOID[oid] - if len(recs) == 0 { - return fmt.Errorf("no matching DRS record found for oid %s in the configured scope", oid) + if len(checksumOIDs) > 0 { + if anvil != nil { + return fmt.Errorf("remote %q can only hydrate DRS URI pointers; selected file has a SHA-256 pointer", remote) + } + progress.OnStage("Looking up DRS records") + recsByOID, err := lookup.ObjectsByHashesForScope(ctx, drsCtx, checksumOIDs) + if err != nil { + return fmt.Errorf("look up DRS records by checksum: %w", err) + } + for _, oid := range checksumOIDs { + recs := recsByOID[oid] + if len(recs) == 0 { + return fmt.Errorf("no matching DRS record found for oid %s in the configured scope", oid) + } + prefetched[oid] = recs[0] } - prefetched[oid] = recs[0] } } if len(prefetched) > 0 && len(missingOIDs) > 1 { diff --git a/cmd/pull/pull_test.go b/cmd/pull/pull_test.go index 420d10c1..d6d46a60 100644 --- a/cmd/pull/pull_test.go +++ b/cmd/pull/pull_test.go @@ -3,18 +3,28 @@ package pull import ( "bytes" "context" + "crypto/rand" + "crypto/rsa" "crypto/sha256" + "crypto/tls" + "crypto/x509" "encoding/hex" + "encoding/json" + "encoding/pem" "errors" + "fmt" "io" "log/slog" "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" + "reflect" "runtime" "strconv" "strings" + "sync" "testing" "time" @@ -25,6 +35,7 @@ import ( "github.com/calypr/git-drs/internal/gitrepo" "github.com/calypr/git-drs/internal/lfs" "github.com/calypr/git-drs/internal/remoteruntime" + "github.com/calypr/git-drs/internal/resolver" internaltransfer "github.com/calypr/git-drs/internal/transfer" drsapi "github.com/calypr/syfon/apigen/drs" syclient "github.com/calypr/syfon/client" @@ -557,6 +568,378 @@ func TestPullSingleFileDoesNotRepeatFailedChecksumLookup(t *testing.T) { } } +type terraPullFixture struct { + repo string + filename string + oid string + objectID string + payload []byte + resolver *httptest.Server + data *httptest.Server + mu sync.Mutex + apiPaths []string + dataRequests []http.Header + tokenRequests int +} + +type terraPullFixtureOptions struct { + resolverStatus int + useADC bool +} + +func newTerraPullFixture(t *testing.T, checksum string, dataHandler http.HandlerFunc, options ...terraPullFixtureOptions) *terraPullFixture { + t.Helper() + resetPullFlagsForTest() + var opts terraPullFixtureOptions + if len(options) > 0 { + opts = options[0] + } + f := &terraPullFixture{ + repo: t.TempDir(), + filename: "1614321.merge_output.gvcf.gz", + oid: "drs://drs.anv0:v2_4f770147-e372-339b-b9fa-0a7a83cf30cf", + objectID: "v2_4f770147-e372-339b-b9fa-0a7a83cf30cf", + payload: []byte("AnVIL object content"), + } + actual := sha256.Sum256(f.payload) + if checksum == "" { + checksum = hex.EncodeToString(actual[:]) + } + if dataHandler == nil { + dataHandler = func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("X-Access-Key"); got != "test-data-key" { + http.Error(w, "missing provider access header", http.StatusUnauthorized) + return + } + _, _ = w.Write(f.payload) + } + } + f.data = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + f.dataRequests = append(f.dataRequests, r.Header.Clone()) + f.mu.Unlock() + dataHandler(w, r) + })) + f.resolver = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/token" { + f.mu.Lock() + f.tokenRequests++ + f.mu.Unlock() + if err := r.ParseForm(); err != nil || r.Form.Get("grant_type") != "urn:ietf:params:oauth:grant-type:jwt-bearer" || r.Form.Get("assertion") == "" { + http.Error(w, "invalid service-account token request", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"access_token":"test-resolver-token","token_type":"Bearer","expires_in":3600}`)) + return + } + f.mu.Lock() + f.apiPaths = append(f.apiPaths, r.URL.Path) + f.mu.Unlock() + if r.Header.Get("Authorization") != "Bearer test-resolver-token" { + http.Error(w, "missing resolver authorization", http.StatusUnauthorized) + return + } + if opts.resolverStatus != 0 { + http.Error(w, "resolver denied access", opts.resolverStatus) + return + } + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/ga4gh/drs/v1/objects/" + f.objectID: + _, _ = fmt.Fprintf(w, `{"id":%q,"size":%d,"checksums":[{"type":"sha256","checksum":%q}],"access_methods":[{"type":"https","access_id":"access-1"}]}`, f.objectID, len(f.payload), checksum) + case "/ga4gh/drs/v1/objects/" + f.objectID + "/access/access-1": + _, _ = fmt.Fprintf(w, `{"url":%q,"headers":["X-Access-Key: test-data-key"]}`, f.data.URL) + default: + http.NotFound(w, r) + } + })) + if opts.useADC { + f.configureADC(t) + } + + runGitCmdTest(t, f.repo, "init", "-q") + t.Chdir(f.repo) + if err := os.WriteFile(".gitattributes", []byte(f.filename+" filter=drs diff=drs merge=drs -text\n"), 0o644); err != nil { + t.Fatal(err) + } + pointer := fmt.Sprintf("version https://calypr.github.io/spec/v1\noid %s\nsize %d\n", f.oid, len(f.payload)) + if err := os.WriteFile(f.filename, []byte(pointer), 0o644); err != nil { + t.Fatal(err) + } + runGitCmdTest(t, f.repo, "add", f.filename) + + oldNewAnVILResolver := newAnVILResolver + t.Cleanup(func() { + newAnVILResolver = oldNewAnVILResolver + resetPullFlagsForTest() + }) + if _, err := config.UpdateRemote("anvil", config.RemoteSelect{Terra: &config.TerraRemote{Endpoint: f.resolver.URL, Mode: "read-only"}}); err != nil { + t.Fatalf("configure Terra remote: %v", err) + } + newAnVILResolver = func(ctx context.Context, endpoint string) (resolver.Resolver, error) { + if opts.useADC { + return resolver.NewAnVIL(ctx, endpoint) + } + client := f.resolver.Client() + base := client.Transport + client.Transport = pullRoundTripFunc(func(req *http.Request) (*http.Response, error) { + authorized := req.Clone(req.Context()) + authorized.Header.Set("Authorization", "Bearer test-resolver-token") + return base.RoundTrip(authorized) + }) + return resolver.NewAnVILWithClient(endpoint, client) + } + includePatterns = []string{f.filename} + t.Cleanup(func() { + f.resolver.Close() + f.data.Close() + }) + return f +} + +func (f *terraPullFixture) configureADC(t *testing.T) { + t.Helper() + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("generate service-account key: %v", err) + } + privateDER, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + t.Fatalf("marshal service-account key: %v", err) + } + credentials, err := json.Marshal(map[string]string{ + "type": "service_account", + "project_id": "git-drs-test", + "private_key_id": "test-key-id", + "private_key": string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateDER})), + "client_email": "git-drs-test@example.iam.gserviceaccount.com", + "client_id": "1234567890", + "token_uri": f.resolver.URL + "/token", + "universe_domain": "googleapis.com", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + }) + if err != nil { + t.Fatalf("marshal service-account credentials: %v", err) + } + credentialsPath := filepath.Join(t.TempDir(), "service-account.json") + if err := os.WriteFile(credentialsPath, credentials, 0o600); err != nil { + t.Fatalf("write service-account credentials: %v", err) + } + t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", credentialsPath) + + roots := x509.NewCertPool() + roots.AddCert(f.resolver.Certificate()) + baseTransport, ok := http.DefaultTransport.(*http.Transport) + if !ok { + t.Fatal("http.DefaultTransport is not an *http.Transport") + } + transport := baseTransport.Clone() + transport.TLSClientConfig = &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12} + oldTransport, oldClient := http.DefaultTransport, http.DefaultClient + http.DefaultTransport = transport + client := *oldClient + client.Transport = transport + http.DefaultClient = &client + t.Cleanup(func() { + http.DefaultTransport = oldTransport + http.DefaultClient = oldClient + }) +} + +func (f *terraPullFixture) run(t *testing.T) error { + t.Helper() + return Cmd.RunE(Cmd, nil) +} + +func (f *terraPullFixture) requests() ([]string, []http.Header) { + f.mu.Lock() + defer f.mu.Unlock() + paths := append([]string(nil), f.apiPaths...) + data := make([]http.Header, len(f.dataRequests)) + for i, headers := range f.dataRequests { + data[i] = headers.Clone() + } + return paths, data +} + +func (f *terraPullFixture) tokenRequestCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.tokenRequests +} + +func (f *terraPullFixture) assertHydrated(t *testing.T) { + t.Helper() + got, err := os.ReadFile(filepath.Join(f.repo, f.filename)) + if err != nil { + t.Fatalf("read hydrated file: %v", err) + } + if !bytes.Equal(got, f.payload) { + t.Fatalf("hydrated content = %q, want %q", got, f.payload) + } +} + +func (f *terraPullFixture) assertStillPointer(t *testing.T) { + t.Helper() + got, err := os.ReadFile(filepath.Join(f.repo, f.filename)) + if err != nil { + t.Fatalf("read worktree pointer: %v", err) + } + if !bytes.Contains(got, []byte(f.oid)) { + t.Fatalf("worktree file after failed pull = %q, want original DRS pointer", got) + } +} + +func TestPullTerraDRSURIUsesAuthenticatedAnVILResolverAndReusesCache(t *testing.T) { + f := newTerraPullFixture(t, "", nil) + if err := f.run(t); err != nil { + t.Fatalf("Terra DRS URI pull failed: %v", err) + } + f.assertHydrated(t) + wantPaths := []string{ + "/ga4gh/drs/v1/objects/" + f.objectID, + "/ga4gh/drs/v1/objects/" + f.objectID + "/access/access-1", + } + paths, dataRequests := f.requests() + if !reflect.DeepEqual(paths, wantPaths) { + t.Fatalf("AnVIL API paths = %v, want %v", paths, wantPaths) + } + if len(dataRequests) != 1 || dataRequests[0].Get("X-Access-Key") != "test-data-key" { + t.Fatalf("data request headers = %v, want one request with provider access header", dataRequests) + } + if err := f.run(t); err != nil { + t.Fatalf("second pull from local cache failed: %v", err) + } + pathsAfterReuse, dataAfterReuse := f.requests() + if !reflect.DeepEqual(pathsAfterReuse, paths) || len(dataAfterReuse) != len(dataRequests) { + t.Fatalf("cache reuse repeated network requests: API=%v data=%d; prior API=%v data=%d", pathsAfterReuse, len(dataAfterReuse), paths, len(dataRequests)) + } + f.assertHydrated(t) +} + +func TestPullTerraDRSURIUsesADCServiceAccountAgainstLocalTokenEndpoint(t *testing.T) { + f := newTerraPullFixture(t, "", nil, terraPullFixtureOptions{useADC: true}) + if err := f.run(t); err != nil { + t.Fatalf("Terra DRS URI pull with service-account ADC failed: %v", err) + } + f.assertHydrated(t) + if got := f.tokenRequestCount(); got != 1 { + t.Fatalf("OAuth token endpoint requests = %d, want one service-account token exchange", got) + } + paths, dataRequests := f.requests() + wantPaths := []string{ + "/ga4gh/drs/v1/objects/" + f.objectID, + "/ga4gh/drs/v1/objects/" + f.objectID + "/access/access-1", + } + if !reflect.DeepEqual(paths, wantPaths) { + t.Fatalf("AnVIL API paths = %v, want %v", paths, wantPaths) + } + if len(dataRequests) != 1 || dataRequests[0].Get("X-Access-Key") != "test-data-key" { + t.Fatalf("data request headers = %v, want one request with provider access header", dataRequests) + } +} + +func TestPullTerraDRSURIRejectsChecksumMismatch(t *testing.T) { + f := newTerraPullFixture(t, strings.Repeat("0", sha256.Size*2), nil) + err := f.run(t) + if err == nil || !strings.Contains(err.Error(), "checksum mismatch") { + t.Fatalf("Terra pull error = %v, want checksum mismatch", err) + } + f.assertStillPointer(t) +} + +func TestPullTerraDRSURIReportsUnauthorizedDataResponse(t *testing.T) { + f := newTerraPullFixture(t, "", func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + }) + err := f.run(t) + if err == nil || !strings.Contains(err.Error(), "HTTP 401") { + t.Fatalf("Terra pull error = %v, want HTTP 401", err) + } + f.assertStillPointer(t) +} + +func TestPullTerraDRSURIReportsUnauthorizedResolverResponse(t *testing.T) { + f := newTerraPullFixture(t, "", nil, terraPullFixtureOptions{resolverStatus: http.StatusUnauthorized}) + err := f.run(t) + if err == nil || !strings.Contains(err.Error(), "not authorized to access AnVIL DRS object") { + t.Fatalf("Terra pull error = %v, want resolver authorization error", err) + } + f.assertStillPointer(t) +} + +func TestPullTerraDRSURIWithLargePointerReachesResolverBeforeAuthorizationFailure(t *testing.T) { + f := newTerraPullFixture(t, "", nil, terraPullFixtureOptions{resolverStatus: http.StatusUnauthorized}) + pointer := "version https://calypr.github.io/spec/v1\n" + + "oid drs://drs.anv0:v2_4f770147-e372-339b-b9fa-0a7a83cf30cf\n" + + "size 6151668906\n" + if err := os.WriteFile(filepath.Join(f.repo, f.filename), []byte(pointer), 0o644); err != nil { + t.Fatalf("write large Terra pointer: %v", err) + } + runGitCmdTest(t, f.repo, "add", f.filename) + + err := f.run(t) + if err == nil || !strings.Contains(err.Error(), "not authorized to access AnVIL DRS object") { + t.Fatalf("large Terra pointer pull error = %v, want resolver authorization error", err) + } + if strings.Contains(err.Error(), "DRS client unavailable") || strings.Contains(err.Error(), "by checksum") { + t.Fatalf("large Terra pointer was routed to checksum lookup: %v", err) + } + paths, _ := f.requests() + want := []string{"/ga4gh/drs/v1/objects/" + f.objectID} + if !reflect.DeepEqual(paths, want) { + t.Fatalf("resolver API paths = %v, want object lookup %v", paths, want) + } + got, readErr := os.ReadFile(filepath.Join(f.repo, f.filename)) + if readErr != nil || string(got) != pointer { + t.Fatalf("worktree pointer after resolver authorization failure = %q, %v", got, readErr) + } +} + +func TestPullTerraDRSURIResumesInterruptedDownload(t *testing.T) { + const partialSize = 8 + var dataCalls int + var f *terraPullFixture + f = newTerraPullFixture(t, "", func(w http.ResponseWriter, r *http.Request) { + dataCalls++ + if dataCalls == 1 { + w.Header().Set("Content-Length", strconv.Itoa(len(f.payload))) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(f.payload[:partialSize]) + w.(http.Flusher).Flush() + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Errorf("hijack interrupted response: %v", err) + return + } + _ = conn.Close() + return + } + if got := r.Header.Get("Range"); got != "bytes=8-" { + http.Error(w, "expected resumed byte range", http.StatusRequestedRangeNotSatisfiable) + return + } + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", partialSize, len(f.payload)-1, len(f.payload))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(f.payload[partialSize:]) + }) + if err := f.run(t); err == nil { + t.Fatal("interrupted transfer unexpectedly succeeded") + } + if err := f.run(t); err != nil { + t.Fatalf("resumed Terra pull failed: %v", err) + } + if dataCalls != 2 { + t.Fatalf("data requests = %d, want initial transfer plus resumed request", dataCalls) + } + _, dataRequests := f.requests() + if len(dataRequests) != 2 || dataRequests[1].Get("Range") != "bytes=8-" { + t.Fatalf("resumption request headers = %v, want Range: bytes=8-", dataRequests) + } + f.assertHydrated(t) +} + func TestPullUsesTrackedInventoryForHydratedFiles(t *testing.T) { repo := t.TempDir() runGitCmdTest(t, repo, "init") diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index 1e17e9a9..6809990c 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -15,6 +15,7 @@ import ( "os" "path" "path/filepath" + "strconv" "strings" "golang.org/x/oauth2" @@ -73,47 +74,85 @@ func DownloadToCache(ctx context.Context, r Resolver, drsURI, destination string if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { return err } - tmp, err := os.CreateTemp(filepath.Dir(destination), ".git-drs-download-*") + partialPath := destination + ".git-drs-partial" + tmp, err := os.OpenFile(partialPath, os.O_CREATE|os.O_RDWR, 0o600) if err != nil { return err } - tmpName := tmp.Name() - defer os.Remove(tmpName) + defer tmp.Close() + info, err := tmp.Stat() + if err != nil { + return err + } + offset := info.Size() + if offset >= obj.Size { + offset = 0 + } + if offset == 0 { + if err := tmp.Truncate(0); err != nil { + return err + } + } + hasher := sha256.New() + if offset > 0 { + if _, err := io.Copy(hasher, io.NewSectionReader(tmp, 0, offset)); err != nil { + return err + } + } + if _, err := tmp.Seek(offset, io.SeekStart); err != nil { + return err + } req, err := http.NewRequestWithContext(ctx, http.MethodGet, access.URL, nil) if err != nil { - tmp.Close() return err } + if offset > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) + } for _, header := range access.Headers { key, value, ok := strings.Cut(header, ":") key = strings.TrimSpace(key) if !ok || key == "" { - tmp.Close() return fmt.Errorf("AnVIL resolver returned an invalid access header") } req.Header.Add(key, strings.TrimSpace(value)) } resp, err := http.DefaultClient.Do(req) if err != nil { - tmp.Close() return fmt.Errorf("AnVIL data download failed") } defer resp.Body.Close() + if offset > 0 && resp.StatusCode == http.StatusOK { + if err := tmp.Truncate(0); err != nil { + return err + } + if _, err := tmp.Seek(0, io.SeekStart); err != nil { + return err + } + hasher.Reset() + offset = 0 + } + if offset > 0 && resp.StatusCode == http.StatusPartialContent { + prefix := "bytes " + strconv.FormatInt(offset, 10) + "-" + if !strings.HasPrefix(resp.Header.Get("Content-Range"), prefix) { + return fmt.Errorf("AnVIL data download returned an unexpected content range") + } + } if resp.StatusCode < 200 || resp.StatusCode >= 300 { - tmp.Close() return fmt.Errorf("AnVIL data download returned HTTP %d", resp.StatusCode) } - hasher := sha256.New() + if offset == 0 && resp.StatusCode == http.StatusPartialContent { + return fmt.Errorf("AnVIL data download returned a partial response without a range request") + } _, copyErr := io.Copy(io.MultiWriter(tmp, hasher), resp.Body) - closeErr := tmp.Close() if copyErr != nil { return fmt.Errorf("AnVIL data download interrupted: %w", copyErr) } - if closeErr != nil { - return closeErr + if err := tmp.Sync(); err != nil { + return err } if obj.Size >= 0 { - info, statErr := os.Stat(tmpName) + info, statErr := tmp.Stat() if statErr != nil { return statErr } @@ -137,10 +176,14 @@ func DownloadToCache(ctx context.Context, r Resolver, drsURI, destination string } actual := hasher.Sum(nil) if subtle.ConstantTimeCompare(actual, expectedBytes) != 1 { + _ = os.Remove(partialPath) return fmt.Errorf("AnVIL data checksum mismatch for %s", checksumType) } } - return os.Rename(tmpName, destination) + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(partialPath, destination) } func isHTTPAccessURL(raw string) bool { @@ -182,15 +225,27 @@ type AnVILResolver struct { } func NewAnVIL(ctx context.Context, endpoint string) (*AnVILResolver, error) { - u, err := trustedEndpoint(endpoint) - if err != nil { + if _, err := trustedEndpoint(endpoint); err != nil { return nil, err } creds, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/cloud-platform") if err != nil { return nil, fmt.Errorf("%w: run `gcloud auth application-default login`: %v", ErrCredentials, err) } - return &AnVILResolver{endpoint: u, client: &http.Client{Transport: &oauth2.Transport{Base: http.DefaultTransport, Source: creds.TokenSource}}}, nil + return NewAnVILWithClient(endpoint, &http.Client{Transport: &oauth2.Transport{Base: http.DefaultTransport, Source: creds.TokenSource}}) +} + +// NewAnVILWithClient uses a caller-supplied HTTP client for a trusted HTTPS +// resolver endpoint. The caller is responsible for attaching authentication. +func NewAnVILWithClient(endpoint string, client *http.Client) (*AnVILResolver, error) { + u, err := trustedEndpoint(endpoint) + if err != nil { + return nil, err + } + if client == nil { + return nil, fmt.Errorf("AnVIL resolver HTTP client is required") + } + return &AnVILResolver{endpoint: u, client: client}, nil } func trustedEndpoint(endpoint string) (*url.URL, error) { diff --git a/internal/resolver/resolver_test.go b/internal/resolver/resolver_test.go index 1d6cbfec..5266a179 100644 --- a/internal/resolver/resolver_test.go +++ b/internal/resolver/resolver_test.go @@ -8,25 +8,12 @@ import ( "fmt" "net/http" "net/http/httptest" - "net/url" "os" "path/filepath" "strings" "testing" ) -func NewAnVILWithClient(endpoint string, client *http.Client) (*AnVILResolver, error) { - // Local HTTP test servers use a caller supplied client without ADC tokens. - u, err := url.Parse(endpoint) - if err != nil { - return nil, err - } - if client == nil { - return nil, fmt.Errorf("HTTP client is required") - } - return &AnVILResolver{endpoint: u, client: client}, nil -} - func TestAnVILTrustedEndpointRequiresHTTPS(t *testing.T) { if _, err := trustedEndpoint("http://resolver.example"); err == nil { t.Fatal("HTTP resolver accepted for ADC bearer token requests") @@ -37,7 +24,7 @@ func TestAnVILTrustedEndpointRequiresHTTPS(t *testing.T) { } func TestAnVILResolverContract(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/ga4gh/drs/v1/objects/object-1": _, _ = w.Write([]byte(`{"id":"object-1","size":4,"checksums":[{"type":"sha256","checksum":"abcd"}],"access_methods":[{"type":"https","access_id":"a1"}]}`)) @@ -77,7 +64,7 @@ func TestDownloadToCacheUsesAccessURLHeaders(t *testing.T) { })) defer download.Close() - resolverServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resolverServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/ga4gh/drs/v1/objects/object-1": _, _ = w.Write([]byte(`{"id":"object-1","size":4,"access_methods":[{"type":"https","access_id":"a1"}]}`)) @@ -109,7 +96,7 @@ func TestDownloadToCacheUsesInlineAccessURL(t *testing.T) { })) defer download.Close() - resolverServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resolverServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/ga4gh/drs/v1/objects/object-1" { t.Errorf("inline access URL should not require an access endpoint request: %s", r.URL.Path) http.NotFound(w, r) @@ -136,7 +123,7 @@ func TestDownloadToCacheSelectsUsableAccessMethodAfterFirst(t *testing.T) { })) defer download.Close() - resolverServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resolverServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/ga4gh/drs/v1/objects/object-1": _, _ = w.Write([]byte(`{"id":"object-1","size":4,"access_methods":[{"type":"gs","access_url":{"url":"gs://anvil-bucket/object-1"}},{"type":"https","access_id":"a2"}]}`)) @@ -228,7 +215,7 @@ func TestDownloadToCacheValidatesSHA256BeforePromotion(t *testing.T) { })) defer download.Close() - resolverServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resolverServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/ga4gh/drs/v1/objects/object-1" { http.NotFound(w, r) return @@ -273,7 +260,7 @@ func TestDownloadToCacheValidatesSHA256BeforePromotion(t *testing.T) { func TestAnVILResolverAcceptsCompactDRSURI(t *testing.T) { const compactURI = "drs://drs.anv0:v2_e68887be-c583-375a-a773-48771192c8fa" - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/ga4gh/drs/v1/objects/v2_e68887be-c583-375a-a773-48771192c8fa" { t.Fatalf("unexpected resolver path: %s", r.URL.Path) } @@ -295,7 +282,7 @@ func TestAnVILResolverAcceptsCompactDRSURI(t *testing.T) { } func TestAnVILResolverRedactsErrorBodiesAndClassifiesAuthorization(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "Bearer secret signed=https://secret", http.StatusForbidden) })) defer server.Close() From e79c6c882d399c9d3c13374b2380c71f082f0dba Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 7 Oct 2026 15:38:48 -0700 Subject: [PATCH 03/22] Report missing explicit pull paths --- cmd/pull/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/pull/main.go b/cmd/pull/main.go index 9c13add3..149f123d 100644 --- a/cmd/pull/main.go +++ b/cmd/pull/main.go @@ -113,6 +113,11 @@ var Cmd = &cobra.Command{ } pointers := collectPointerFiles(inventory, patterns, gitPaths.DRSObjectsDir()) if len(pointers) == 0 { + if path, exact := exactIncludePath(patterns); exact { + if _, statErr := os.Stat(filepath.Join(inventoryRoot, filepath.FromSlash(path))); os.IsNotExist(statErr) { + return fmt.Errorf("include path %q does not exist in this repository", path) + } + } logg.Debug("no matching pointer files to hydrate") return nil } From a6ea3728fbefe90328579608c5db048b8e3ea617 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 7 Oct 2026 16:05:49 -0700 Subject: [PATCH 04/22] Verify TDR CRC32C and MD5 download checksums --- cmd/pull/pull_test.go | 22 ++++++++++++++++++--- internal/resolver/resolver.go | 37 ++++++++++++++++++++++++++--------- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/cmd/pull/pull_test.go b/cmd/pull/pull_test.go index d6d46a60..58f4a640 100644 --- a/cmd/pull/pull_test.go +++ b/cmd/pull/pull_test.go @@ -3,6 +3,7 @@ package pull import ( "bytes" "context" + "crypto/md5" "crypto/rand" "crypto/rsa" "crypto/sha256" @@ -13,6 +14,7 @@ import ( "encoding/pem" "errors" "fmt" + "hash/crc32" "io" "log/slog" "net/http" @@ -585,6 +587,7 @@ type terraPullFixture struct { type terraPullFixtureOptions struct { resolverStatus int useADC bool + crcChecksum string } func newTerraPullFixture(t *testing.T, checksum string, dataHandler http.HandlerFunc, options ...terraPullFixtureOptions) *terraPullFixture { @@ -601,10 +604,14 @@ func newTerraPullFixture(t *testing.T, checksum string, dataHandler http.Handler objectID: "v2_4f770147-e372-339b-b9fa-0a7a83cf30cf", payload: []byte("AnVIL object content"), } - actual := sha256.Sum256(f.payload) + actual := md5.Sum(f.payload) if checksum == "" { checksum = hex.EncodeToString(actual[:]) } + crcChecksum := fmt.Sprintf("%08x", crc32.Checksum(f.payload, crc32.MakeTable(crc32.Castagnoli))) + if opts.crcChecksum != "" { + crcChecksum = opts.crcChecksum + } if dataHandler == nil { dataHandler = func(w http.ResponseWriter, r *http.Request) { if got := r.Header.Get("X-Access-Key"); got != "test-data-key" { @@ -647,7 +654,7 @@ func newTerraPullFixture(t *testing.T, checksum string, dataHandler http.Handler w.Header().Set("Content-Type", "application/json") switch r.URL.Path { case "/ga4gh/drs/v1/objects/" + f.objectID: - _, _ = fmt.Fprintf(w, `{"id":%q,"size":%d,"checksums":[{"type":"sha256","checksum":%q}],"access_methods":[{"type":"https","access_id":"access-1"}]}`, f.objectID, len(f.payload), checksum) + _, _ = fmt.Fprintf(w, `{"id":%q,"size":%d,"checksums":[{"type":"crc32c","checksum":%q},{"type":"md5","checksum":%q}],"access_methods":[{"type":"https","access_id":"access-1"}]}`, f.objectID, len(f.payload), crcChecksum, checksum) case "/ga4gh/drs/v1/objects/" + f.objectID + "/access/access-1": _, _ = fmt.Fprintf(w, `{"url":%q,"headers":["X-Access-Key: test-data-key"]}`, f.data.URL) default: @@ -841,7 +848,7 @@ func TestPullTerraDRSURIUsesADCServiceAccountAgainstLocalTokenEndpoint(t *testin } func TestPullTerraDRSURIRejectsChecksumMismatch(t *testing.T) { - f := newTerraPullFixture(t, strings.Repeat("0", sha256.Size*2), nil) + f := newTerraPullFixture(t, strings.Repeat("0", md5.Size*2), nil) err := f.run(t) if err == nil || !strings.Contains(err.Error(), "checksum mismatch") { t.Fatalf("Terra pull error = %v, want checksum mismatch", err) @@ -849,6 +856,15 @@ func TestPullTerraDRSURIRejectsChecksumMismatch(t *testing.T) { f.assertStillPointer(t) } +func TestPullTerraDRSURIRejectsCRC32CMismatch(t *testing.T) { + f := newTerraPullFixture(t, "", nil, terraPullFixtureOptions{crcChecksum: "00000000"}) + err := f.run(t) + if err == nil || !strings.Contains(err.Error(), "crc32c") || !strings.Contains(err.Error(), "checksum mismatch") { + t.Fatalf("Terra pull error = %v, want CRC32C checksum mismatch", err) + } + f.assertStillPointer(t) +} + func TestPullTerraDRSURIReportsUnauthorizedDataResponse(t *testing.T) { f := newTerraPullFixture(t, "", func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "unauthorized", http.StatusUnauthorized) diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index 6809990c..61db9a89 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -3,12 +3,15 @@ package resolver import ( "context" + "crypto/md5" "crypto/sha256" "crypto/subtle" "encoding/hex" "encoding/json" "errors" "fmt" + "hash" + "hash/crc32" "io" "net/http" "net/url" @@ -93,9 +96,12 @@ func DownloadToCache(ctx context.Context, r Resolver, drsURI, destination string return err } } - hasher := sha256.New() + shaHasher := sha256.New() + md5Hasher := md5.New() + crcHasher := crc32.New(crc32.MakeTable(crc32.Castagnoli)) + hashers := io.MultiWriter(shaHasher, md5Hasher, crcHasher) if offset > 0 { - if _, err := io.Copy(hasher, io.NewSectionReader(tmp, 0, offset)); err != nil { + if _, err := io.Copy(hashers, io.NewSectionReader(tmp, 0, offset)); err != nil { return err } } @@ -129,7 +135,9 @@ func DownloadToCache(ctx context.Context, r Resolver, drsURI, destination string if _, err := tmp.Seek(0, io.SeekStart); err != nil { return err } - hasher.Reset() + shaHasher.Reset() + md5Hasher.Reset() + crcHasher.Reset() offset = 0 } if offset > 0 && resp.StatusCode == http.StatusPartialContent { @@ -144,7 +152,7 @@ func DownloadToCache(ctx context.Context, r Resolver, drsURI, destination string if offset == 0 && resp.StatusCode == http.StatusPartialContent { return fmt.Errorf("AnVIL data download returned a partial response without a range request") } - _, copyErr := io.Copy(io.MultiWriter(tmp, hasher), resp.Body) + _, copyErr := io.Copy(io.MultiWriter(tmp, hashers), resp.Body) if copyErr != nil { return fmt.Errorf("AnVIL data download interrupted: %w", copyErr) } @@ -162,16 +170,27 @@ func DownloadToCache(ctx context.Context, r Resolver, drsURI, destination string } for _, checksum := range obj.Checksums { checksumType := strings.ToLower(strings.TrimSpace(checksum.Type)) - if checksumType != "sha256" && checksumType != "sha-256" { + var hasher hash.Hash + switch checksumType { + case "sha256", "sha-256": + hasher = shaHasher + case "md5": + hasher = md5Hasher + case "crc32c": + hasher = crcHasher + default: continue } expected := strings.TrimSpace(checksum.Checksum) - if prefix, value, ok := strings.Cut(expected, ":"); ok && - (strings.EqualFold(strings.TrimSpace(prefix), "sha256") || strings.EqualFold(strings.TrimSpace(prefix), "sha-256")) { - expected = strings.TrimSpace(value) + if prefix, value, ok := strings.Cut(expected, ":"); ok { + prefix = strings.ToLower(strings.TrimSpace(prefix)) + if prefix == checksumType || + ((checksumType == "sha256" || checksumType == "sha-256") && (prefix == "sha256" || prefix == "sha-256")) { + expected = strings.TrimSpace(value) + } } expectedBytes, decodeErr := hex.DecodeString(expected) - if decodeErr != nil || len(expectedBytes) != sha256.Size { + if decodeErr != nil || len(expectedBytes) != hasher.Size() { return fmt.Errorf("AnVIL DRS object has an invalid %s checksum", checksumType) } actual := hasher.Sum(nil) From 943cad871a5a3a133ede8c30fea8a4c03f708d22 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 7 Oct 2026 19:05:55 -0700 Subject: [PATCH 05/22] Exercise Terra pull against TDR controller in CI --- .github/workflows/pr-checks.yaml | 33 +++++ cmd/pull/terra_tdr_integration_test.go | 102 +++++++++++++++ tests/terra-tdr/README.md | 11 ++ tests/terra-tdr/TerraDrsHarnessTest.java | 159 +++++++++++++++++++++++ tests/terra-tdr/run.sh | 64 +++++++++ 5 files changed, 369 insertions(+) create mode 100644 cmd/pull/terra_tdr_integration_test.go create mode 100644 tests/terra-tdr/README.md create mode 100644 tests/terra-tdr/TerraDrsHarnessTest.java create mode 100755 tests/terra-tdr/run.sh diff --git a/.github/workflows/pr-checks.yaml b/.github/workflows/pr-checks.yaml index 7bcdd100..3bad64d9 100644 --- a/.github/workflows/pr-checks.yaml +++ b/.github/workflows/pr-checks.yaml @@ -75,3 +75,36 @@ jobs: - name: Run tests run: go test -v -race ./... + + tdr-controller: + name: Pull through TDR controller + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Check out git-drs + uses: actions/checkout@v4 + + - name: Check out TDR + uses: actions/checkout@v4 + with: + repository: DataBiosphere/jade-data-repo + ref: 6433cb4ea296edf873daa30f1da29e7caf848b8b + path: tdr + persist-credentials: false + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + cache: gradle + cache-dependency-path: tdr/build.gradle + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.26.6' + + - name: Pull an AnVIL DRS pointer through TDR's controller + run: tests/terra-tdr/run.sh "$GITHUB_WORKSPACE/tdr" diff --git a/cmd/pull/terra_tdr_integration_test.go b/cmd/pull/terra_tdr_integration_test.go new file mode 100644 index 00000000..6dc3dd6d --- /dev/null +++ b/cmd/pull/terra_tdr_integration_test.go @@ -0,0 +1,102 @@ +//go:build integration + +package pull + +import ( + "bytes" + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "path/filepath" + "reflect" + "sync" + "testing" + + "github.com/calypr/git-drs/internal/config" + "github.com/calypr/git-drs/internal/resolver" +) + +func TestIntegrationPullAgainstTDRController(t *testing.T) { + endpoint := os.Getenv("GIT_DRS_TDR_HTTP_ENDPOINT") + if endpoint == "" { + t.Skip("GIT_DRS_TDR_HTTP_ENDPOINT is required") + } + tdrURL, err := url.Parse(endpoint) + if err != nil || tdrURL.Scheme != "http" || tdrURL.Host == "" { + t.Fatalf("invalid local TDR endpoint %q", endpoint) + } + + reverseProxy := httputil.NewSingleHostReverseProxy(tdrURL) + var requestMu sync.Mutex + var requestPaths []string + proxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestMu.Lock() + requestPaths = append(requestPaths, r.URL.Path) + requestMu.Unlock() + reverseProxy.ServeHTTP(w, r) + })) + t.Cleanup(proxy.Close) + oldResolver := newAnVILResolver + t.Cleanup(func() { + newAnVILResolver = oldResolver + resetPullFlagsForTest() + }) + newAnVILResolver = func(_ context.Context, endpoint string) (resolver.Resolver, error) { + client := proxy.Client() + base := client.Transport + client.Transport = pullRoundTripFunc(func(req *http.Request) (*http.Response, error) { + authorized := req.Clone(req.Context()) + authorized.Header.Set("Authorization", "Bearer tdr-ci-fixture") + return base.RoundTrip(authorized) + }) + return resolver.NewAnVILWithClient(endpoint, client) + } + + repo := t.TempDir() + runGitCmdTest(t, repo, "init", "-q") + t.Chdir(repo) + const filename = "1614321.merge_output.gvcf.gz" + const oid = "drs://drs.anv0:v2_4f770147-e372-339b-b9fa-0a7a83cf30cf" + payload := []byte("git-drs Terra DRS HTTP integration fixture\n") + if err := os.WriteFile(".gitattributes", []byte(filename+" filter=drs diff=drs merge=drs -text\n"), 0o644); err != nil { + t.Fatal(err) + } + pointer := fmt.Sprintf("version https://calypr.github.io/spec/v1\noid %s\nsize %d\n", oid, len(payload)) + if err := os.WriteFile(filename, []byte(pointer), 0o644); err != nil { + t.Fatal(err) + } + runGitCmdTest(t, repo, "add", filename) + if _, err := config.UpdateRemote("anvil", config.RemoteSelect{Terra: &config.TerraRemote{Endpoint: proxy.URL, Mode: "read-only"}}); err != nil { + t.Fatal(err) + } + resetPullFlagsForTest() + includePatterns = []string{filename} + requestMu.Lock() + requestPaths = nil + requestMu.Unlock() + + if err := Cmd.RunE(Cmd, nil); err != nil { + t.Fatalf("pull through TDR controller: %v", err) + } + got, err := os.ReadFile(filepath.Join(repo, filename)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, payload) { + t.Fatalf("hydrated content = %q, want %q", got, payload) + } + requestMu.Lock() + gotPaths := append([]string(nil), requestPaths...) + requestMu.Unlock() + wantPaths := []string{ + "/ga4gh/drs/v1/objects/v2_4f770147-e372-339b-b9fa-0a7a83cf30cf", + "/ga4gh/drs/v1/objects/v2_4f770147-e372-339b-b9fa-0a7a83cf30cf/access/https-access", + } + if !reflect.DeepEqual(gotPaths, wantPaths) { + t.Fatalf("TDR controller requests = %v, want %v", gotPaths, wantPaths) + } +} diff --git a/tests/terra-tdr/README.md b/tests/terra-tdr/README.md new file mode 100644 index 00000000..19279ce4 --- /dev/null +++ b/tests/terra-tdr/README.md @@ -0,0 +1,11 @@ +# TDR controller integration + +`run.sh` copies `TerraDrsHarnessTest.java` into a checkout of the pinned TDR source, starts its real DRS controller and generated API routes, then runs the git-drs pull integration test against that server. The harness mocks TDR authentication and `DrsService`; it serves a small file with TDR-style CRC32C and MD5 checksums. It needs no Terra credentials or cloud storage. + +Run locally with Java 17 and Go installed: + +```bash +tests/terra-tdr/run.sh /path/to/jade-data-repo +``` + +The CI job pins the TDR source commit in `.github/workflows/pr-checks.yaml`. This test covers the TDR HTTP controller contract and the git-drs pull path. It does not exercise TDR's database, Sam authorization, or cloud storage integrations. diff --git a/tests/terra-tdr/TerraDrsHarnessTest.java b/tests/terra-tdr/TerraDrsHarnessTest.java new file mode 100644 index 00000000..43a46fa1 --- /dev/null +++ b/tests/terra-tdr/TerraDrsHarnessTest.java @@ -0,0 +1,159 @@ +package bio.terra.app.controller; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.when; + +import bio.terra.app.configuration.ApplicationConfiguration; +import bio.terra.common.category.Unit; +import bio.terra.common.exception.UnauthorizedException; +import bio.terra.common.fixtures.AuthenticationFixtures; +import bio.terra.common.iam.AuthenticatedUserRequestFactory; +import bio.terra.common.iam.BearerTokenFactory; +import bio.terra.model.DRSAccessMethod; +import bio.terra.model.DRSAccessMethod.TypeEnum; +import bio.terra.model.DRSAccessURL; +import bio.terra.model.DRSChecksum; +import bio.terra.model.DRSObject; +import bio.terra.service.filedata.DrsService; +import jakarta.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.List; +import java.util.zip.CRC32C; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.context.annotation.Import; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * A real HTTP harness for git-drs Terra DRS integration tests. The TDR API controller and its + * generated API mapping are real; storage and authentication are stubbed. The test stays alive + * until the CI driver creates GIT_DRS_TDR_STOP_FILE. + */ +@SpringBootTest( + classes = TerraDrsHarnessTest.HarnessApplication.class, + webEnvironment = WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"google", "unittest"}) +@Tag(Unit.TAG) +class TerraDrsHarnessTest { + private static final String OBJECT_ID = "v2_4f770147-e372-339b-b9fa-0a7a83cf30cf"; + private static final String ACCESS_ID = "https-access"; + private static final byte[] FIXTURE_BYTES = + "git-drs Terra DRS HTTP integration fixture\n".getBytes(java.nio.charset.StandardCharsets.UTF_8); + private static final Duration STOP_TIMEOUT = Duration.ofMinutes(2); + + @LocalServerPort private int port; + + @MockitoBean private ApplicationConfiguration applicationConfiguration; + @MockitoBean private DrsService drsService; + @MockitoBean private AuthenticatedUserRequestFactory authenticatedUserRequestFactory; + @MockitoBean private BearerTokenFactory bearerTokenFactory; + + @BeforeEach + void configureFixtureAndSignalReady() throws Exception { + Path portFile = requiredPath("GIT_DRS_TDR_PORT_FILE"); + Path stopFile = requiredPath("GIT_DRS_TDR_STOP_FILE"); + Files.deleteIfExists(portFile); + Files.deleteIfExists(stopFile); + createParentDirectories(portFile); + createParentDirectories(stopFile); + + var testUser = AuthenticationFixtures.randomUserRequest(); + when(authenticatedUserRequestFactory.from(any())) + .thenAnswer( + invocation -> { + HttpServletRequest request = invocation.getArgument(0); + if (!"Bearer tdr-ci-fixture".equals(request.getHeader("Authorization"))) { + throw new UnauthorizedException("Invalid fixture bearer token"); + } + return testUser; + }); + when(drsService.lookupObjectByDrsId(any(), eq(OBJECT_ID), anyBoolean())) + .thenReturn(fixtureObject()); + when(drsService.getAccessUrlForObjectId(any(), eq(OBJECT_ID), eq(ACCESS_ID), isNull())) + .thenReturn(new DRSAccessURL().url("http://127.0.0.1:" + port + "/fixture-data")); + + Files.writeString(portFile, Integer.toString(port)); + } + + @Test + void serveTdrApiUntilCiDriverStops() throws Exception { + Path stopFile = requiredPath("GIT_DRS_TDR_STOP_FILE"); + long deadline = System.nanoTime() + STOP_TIMEOUT.toNanos(); + while (!Files.exists(stopFile) && System.nanoTime() < deadline) { + Thread.sleep(200); + } + if (!Files.exists(stopFile)) { + throw new IllegalStateException( + "Timed out after " + STOP_TIMEOUT + " waiting for " + stopFile); + } + } + + private static DRSObject fixtureObject() throws NoSuchAlgorithmException { + CRC32C crc32c = new CRC32C(); + crc32c.update(FIXTURE_BYTES, 0, FIXTURE_BYTES.length); + String crcHex = String.format("%08x", crc32c.getValue()); + String md5Hex = + java.util.HexFormat.of().formatHex(MessageDigest.getInstance("MD5").digest(FIXTURE_BYTES)); + + return new DRSObject() + .id(OBJECT_ID) + .selfUri("drs://localhost/" + OBJECT_ID) + .size((long) FIXTURE_BYTES.length) + .checksums( + List.of( + new DRSChecksum().type("crc32c").checksum(crcHex), + new DRSChecksum().type("md5").checksum(md5Hex))) + .accessMethods(List.of(new DRSAccessMethod().type(TypeEnum.HTTPS).accessId(ACCESS_ID))); + } + + private static Path requiredPath(String envName) { + String value = System.getenv(envName); + if (value == null || value.isBlank()) { + throw new IllegalStateException(envName + " must name a coordination file"); + } + return Path.of(value); + } + + private static void createParentDirectories(Path path) throws IOException { + Path parent = path.toAbsolutePath().getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + } + + @SpringBootConfiguration + @EnableAutoConfiguration(exclude = DataSourceAutoConfiguration.class) + @Import({DataRepositoryServiceApiController.class, FixtureDataController.class}) + static class HarnessApplication {} + + @RestController + static class FixtureDataController { + @GetMapping("/fixture-data") + ResponseEntity fixtureData() { + return ResponseEntity.ok() + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .body(FIXTURE_BYTES); + } + } +} diff --git a/tests/terra-tdr/run.sh b/tests/terra-tdr/run.sh new file mode 100755 index 00000000..af9ad790 --- /dev/null +++ b/tests/terra-tdr/run.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 /path/to/jade-data-repo" >&2 + exit 2 +fi + +git_drs_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +tdr_root=$(cd "$1" && pwd) +coord_dir=$(mktemp -d) +port_file="$coord_dir/port" +stop_file="$coord_dir/stop" +tdr_log="$coord_dir/tdr.log" + +cp "$git_drs_root/tests/terra-tdr/TerraDrsHarnessTest.java" \ + "$tdr_root/src/test/java/bio/terra/app/controller/TerraDrsHarnessTest.java" + +( + cd "$tdr_root" + GIT_DRS_TDR_PORT_FILE="$port_file" GIT_DRS_TDR_STOP_FILE="$stop_file" \ + ./gradlew testUnit --tests bio.terra.app.controller.TerraDrsHarnessTest \ + --no-daemon --console=plain +) >"$tdr_log" 2>&1 & +tdr_pid=$! + +stop_tdr() { + touch "$stop_file" + wait "$tdr_pid" || true +} +trap stop_tdr EXIT + +for ((seconds=0; seconds<900; seconds++)); do + if [[ -s "$port_file" ]]; then + break + fi + if ! kill -0 "$tdr_pid" 2>/dev/null; then + cat "$tdr_log" >&2 + exit 1 + fi + sleep 1 +done + +if [[ ! -s "$port_file" ]]; then + echo "TDR controller did not start within 15 minutes" >&2 + cat "$tdr_log" >&2 + exit 1 +fi + +port=$(cat "$port_file") +cd "$git_drs_root" +if ! GIT_DRS_TDR_HTTP_ENDPOINT="http://127.0.0.1:$port" \ + go test -race -tags=integration -count=1 ./cmd/pull \ + -run '^TestIntegrationPullAgainstTDRController$' -v; then + cat "$tdr_log" >&2 + exit 1 +fi + +touch "$stop_file" +trap - EXIT +if ! wait "$tdr_pid"; then + cat "$tdr_log" >&2 + exit 1 +fi From dfdad0369245da286b9e123adf295b76179baaef Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 7 Oct 2026 19:09:37 -0700 Subject: [PATCH 06/22] Format TDR harness for upstream checks --- tests/terra-tdr/TerraDrsHarnessTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/terra-tdr/TerraDrsHarnessTest.java b/tests/terra-tdr/TerraDrsHarnessTest.java index 43a46fa1..bf547a0d 100644 --- a/tests/terra-tdr/TerraDrsHarnessTest.java +++ b/tests/terra-tdr/TerraDrsHarnessTest.java @@ -30,7 +30,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; @@ -59,7 +58,8 @@ class TerraDrsHarnessTest { private static final String OBJECT_ID = "v2_4f770147-e372-339b-b9fa-0a7a83cf30cf"; private static final String ACCESS_ID = "https-access"; private static final byte[] FIXTURE_BYTES = - "git-drs Terra DRS HTTP integration fixture\n".getBytes(java.nio.charset.StandardCharsets.UTF_8); + "git-drs Terra DRS HTTP integration fixture\n" + .getBytes(java.nio.charset.StandardCharsets.UTF_8); private static final Duration STOP_TIMEOUT = Duration.ofMinutes(2); @LocalServerPort private int port; From 249fe220c38db5956940c16fe5870cf3db151cb5 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 7 Oct 2026 19:11:50 -0700 Subject: [PATCH 07/22] Show TDR harness result in CI logs --- tests/terra-tdr/run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/terra-tdr/run.sh b/tests/terra-tdr/run.sh index af9ad790..ecd860ca 100755 --- a/tests/terra-tdr/run.sh +++ b/tests/terra-tdr/run.sh @@ -62,3 +62,5 @@ if ! wait "$tdr_pid"; then cat "$tdr_log" >&2 exit 1 fi + +tail -n 20 "$tdr_log" From 70550755c674a65c893c5f83d4919173d1628853 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 7 Oct 2026 19:18:08 -0700 Subject: [PATCH 08/22] Disable unused Firestore auto configuration in TDR harness --- tests/terra-tdr/TerraDrsHarnessTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/terra-tdr/TerraDrsHarnessTest.java b/tests/terra-tdr/TerraDrsHarnessTest.java index bf547a0d..9d4e771e 100644 --- a/tests/terra-tdr/TerraDrsHarnessTest.java +++ b/tests/terra-tdr/TerraDrsHarnessTest.java @@ -51,7 +51,8 @@ */ @SpringBootTest( classes = TerraDrsHarnessTest.HarnessApplication.class, - webEnvironment = WebEnvironment.RANDOM_PORT) + webEnvironment = WebEnvironment.RANDOM_PORT, + properties = "spring.cloud.gcp.firestore.enabled=false") @ActiveProfiles({"google", "unittest"}) @Tag(Unit.TAG) class TerraDrsHarnessTest { From 50d244a3d0914beee082c83c96d8b39e6af14a9a Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 8 Oct 2026 09:17:46 -0700 Subject: [PATCH 09/22] Exercise real TDR DRS service in pull integration --- cmd/pull/terra_tdr_integration_test.go | 24 ++- tests/terra-tdr/README.md | 4 +- tests/terra-tdr/TerraDrsHarnessTest.java | 177 ++++++++++++++++++----- tests/terra-tdr/run.sh | 4 +- 4 files changed, 166 insertions(+), 43 deletions(-) diff --git a/cmd/pull/terra_tdr_integration_test.go b/cmd/pull/terra_tdr_integration_test.go index 6dc3dd6d..fab6be32 100644 --- a/cmd/pull/terra_tdr_integration_test.go +++ b/cmd/pull/terra_tdr_integration_test.go @@ -17,6 +17,7 @@ import ( "testing" "github.com/calypr/git-drs/internal/config" + "github.com/calypr/git-drs/internal/lfs" "github.com/calypr/git-drs/internal/resolver" ) @@ -73,6 +74,17 @@ func TestIntegrationPullAgainstTDRController(t *testing.T) { if _, err := config.UpdateRemote("anvil", config.RemoteSelect{Terra: &config.TerraRemote{Endpoint: proxy.URL, Mode: "read-only"}}); err != nil { t.Fatal(err) } + objectsRoot, err := lfs.ResolveObjectsRoot(context.Background()) + if err != nil { + t.Fatal(err) + } + cachePath, err := lfs.ObjectPath(objectsRoot, oid) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(cachePath); !os.IsNotExist(err) { + t.Fatalf("cache must be empty before pull: %s: %v", cachePath, err) + } resetPullFlagsForTest() includePatterns = []string{filename} requestMu.Lock() @@ -89,12 +101,22 @@ func TestIntegrationPullAgainstTDRController(t *testing.T) { if !bytes.Equal(got, payload) { t.Fatalf("hydrated content = %q, want %q", got, payload) } + resp, err := http.Get(endpoint + "/fixture-data-requests") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var fetches int + _, scanErr := fmt.Fscan(resp.Body, &fetches) + if resp.StatusCode != http.StatusOK || scanErr != nil || fetches != 1 { + t.Fatalf("fixture data downloads = %d with HTTP %d, want one", fetches, resp.StatusCode) + } requestMu.Lock() gotPaths := append([]string(nil), requestPaths...) requestMu.Unlock() wantPaths := []string{ "/ga4gh/drs/v1/objects/v2_4f770147-e372-339b-b9fa-0a7a83cf30cf", - "/ga4gh/drs/v1/objects/v2_4f770147-e372-339b-b9fa-0a7a83cf30cf/access/https-access", + "/ga4gh/drs/v1/objects/v2_4f770147-e372-339b-b9fa-0a7a83cf30cf/access/gcp-us-central1*11111111-2222-4333-8444-555555555555", } if !reflect.DeepEqual(gotPaths, wantPaths) { t.Fatalf("TDR controller requests = %v, want %v", gotPaths, wantPaths) diff --git a/tests/terra-tdr/README.md b/tests/terra-tdr/README.md index 19279ce4..7d6d9fbf 100644 --- a/tests/terra-tdr/README.md +++ b/tests/terra-tdr/README.md @@ -1,6 +1,6 @@ # TDR controller integration -`run.sh` copies `TerraDrsHarnessTest.java` into a checkout of the pinned TDR source, starts its real DRS controller and generated API routes, then runs the git-drs pull integration test against that server. The harness mocks TDR authentication and `DrsService`; it serves a small file with TDR-style CRC32C and MD5 checksums. It needs no Terra credentials or cloud storage. +`run.sh` copies `TerraDrsHarnessTest.java` into a checkout of the pinned TDR source, starts its real DRS controller, generated API routes, and `DrsService`, then runs the git-drs pull integration test against that server. The test provides fixture snapshot and file metadata and mocks authorization and Google URL signing. It checks that git-drs fetches the fixture bytes from an empty cache and verifies the TDR-produced CRC32C and MD5 checksums. It needs no Terra credentials or cloud storage. Run locally with Java 17 and Go installed: @@ -8,4 +8,4 @@ Run locally with Java 17 and Go installed: tests/terra-tdr/run.sh /path/to/jade-data-repo ``` -The CI job pins the TDR source commit in `.github/workflows/pr-checks.yaml`. This test covers the TDR HTTP controller contract and the git-drs pull path. It does not exercise TDR's database, Sam authorization, or cloud storage integrations. +The CI job pins the TDR source commit in `.github/workflows/pr-checks.yaml`. This test covers TDR's DRS object and access URL construction and the git-drs pull path. It does not exercise TDR's database, Sam authorization, or cloud storage integrations. diff --git a/tests/terra-tdr/TerraDrsHarnessTest.java b/tests/terra-tdr/TerraDrsHarnessTest.java index 9d4e771e..190c552f 100644 --- a/tests/terra-tdr/TerraDrsHarnessTest.java +++ b/tests/terra-tdr/TerraDrsHarnessTest.java @@ -1,32 +1,50 @@ -package bio.terra.app.controller; +package bio.terra.service.filedata; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import bio.terra.app.configuration.ApplicationConfiguration; +import bio.terra.app.configuration.DrsConfiguration; +import bio.terra.app.configuration.EcmConfiguration; +import bio.terra.app.controller.DataRepositoryServiceApiController; +import bio.terra.app.logging.PerformanceLogger; +import bio.terra.app.usermetrics.UserLoggingMetrics; import bio.terra.common.category.Unit; import bio.terra.common.exception.UnauthorizedException; import bio.terra.common.fixtures.AuthenticationFixtures; import bio.terra.common.iam.AuthenticatedUserRequestFactory; import bio.terra.common.iam.BearerTokenFactory; -import bio.terra.model.DRSAccessMethod; -import bio.terra.model.DRSAccessMethod.TypeEnum; -import bio.terra.model.DRSAccessURL; -import bio.terra.model.DRSChecksum; -import bio.terra.model.DRSObject; -import bio.terra.service.filedata.DrsService; +import bio.terra.model.BillingProfileModel; +import bio.terra.model.CloudPlatform; +import bio.terra.service.auth.iam.IamService; +import bio.terra.service.dataset.Dataset; +import bio.terra.service.dataset.DatasetSummary; +import bio.terra.service.filedata.google.gcs.GcsProjectFactory; +import bio.terra.service.job.JobService; +import bio.terra.service.resourcemanagement.ResourceService; +import bio.terra.service.resourcemanagement.google.GoogleProjectResource; +import bio.terra.service.snapshot.Snapshot; +import bio.terra.service.snapshot.SnapshotProject; +import bio.terra.service.snapshot.SnapshotService; +import bio.terra.service.snapshot.SnapshotSource; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BucketGetOption; import jakarta.servlet.http.HttpServletRequest; import java.io.IOException; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.time.Duration; -import java.util.List; -import java.util.zip.CRC32C; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -37,6 +55,8 @@ import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; import org.springframework.boot.test.web.server.LocalServerPort; import org.springframework.context.annotation.Import; +import org.springframework.core.task.AsyncTaskExecutor; +import org.springframework.core.task.SimpleAsyncTaskExecutor; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; @@ -45,9 +65,10 @@ import org.springframework.web.bind.annotation.RestController; /** - * A real HTTP harness for git-drs Terra DRS integration tests. The TDR API controller and its - * generated API mapping are real; storage and authentication are stubbed. The test stays alive - * until the CI driver creates GIT_DRS_TDR_STOP_FILE. + * A real HTTP harness for git-drs Terra DRS integration tests. The TDR API controller, generated + * API mapping, and DrsService are real. Snapshot lookup, authorization, file metadata, URL signing, + * and authentication are fixture-backed. The test stays alive until the CI driver creates + * GIT_DRS_TDR_STOP_FILE. */ @SpringBootTest( classes = TerraDrsHarnessTest.HarnessApplication.class, @@ -57,16 +78,29 @@ @Tag(Unit.TAG) class TerraDrsHarnessTest { private static final String OBJECT_ID = "v2_4f770147-e372-339b-b9fa-0a7a83cf30cf"; - private static final String ACCESS_ID = "https-access"; + private static final UUID SNAPSHOT_ID = UUID.fromString("11111111-2222-4333-8444-555555555555"); + private static final UUID FILE_ID = UUID.fromString("4f770147-e372-339b-b9fa-0a7a83cf30cf"); private static final byte[] FIXTURE_BYTES = "git-drs Terra DRS HTTP integration fixture\n" .getBytes(java.nio.charset.StandardCharsets.UTF_8); - private static final Duration STOP_TIMEOUT = Duration.ofMinutes(2); + private static final Duration STOP_TIMEOUT = Duration.ofMinutes(20); + private static final AtomicInteger fixtureDataRequests = new AtomicInteger(); @LocalServerPort private int port; @MockitoBean private ApplicationConfiguration applicationConfiguration; - @MockitoBean private DrsService drsService; + @MockitoBean private SnapshotService snapshotService; + @MockitoBean private FileService fileService; + @MockitoBean private DrsDao drsDao; + @MockitoBean private IamService iamService; + @MockitoBean private ResourceService resourceService; + @MockitoBean private DrsConfiguration drsConfiguration; + @MockitoBean private JobService jobService; + @MockitoBean private PerformanceLogger performanceLogger; + @MockitoBean private GcsProjectFactory gcsProjectFactory; + @MockitoBean private EcmConfiguration ecmConfiguration; + @MockitoBean private DrsMetricsService drsMetricsService; + @MockitoBean private UserLoggingMetrics userLoggingMetrics; @MockitoBean private AuthenticatedUserRequestFactory authenticatedUserRequestFactory; @MockitoBean private BearerTokenFactory bearerTokenFactory; @@ -79,20 +113,40 @@ void configureFixtureAndSignalReady() throws Exception { createParentDirectories(portFile); createParentDirectories(stopFile); + fixtureDataRequests.set(0); var testUser = AuthenticationFixtures.randomUserRequest(); when(authenticatedUserRequestFactory.from(any())) .thenAnswer( invocation -> { HttpServletRequest request = invocation.getArgument(0); + if ("/fixture-data".equals(request.getRequestURI())) { + return testUser; + } if (!"Bearer tdr-ci-fixture".equals(request.getHeader("Authorization"))) { throw new UnauthorizedException("Invalid fixture bearer token"); } return testUser; }); - when(drsService.lookupObjectByDrsId(any(), eq(OBJECT_ID), anyBoolean())) - .thenReturn(fixtureObject()); - when(drsService.getAccessUrlForObjectId(any(), eq(OBJECT_ID), eq(ACCESS_ID), isNull())) - .thenReturn(new DRSAccessURL().url("http://127.0.0.1:" + port + "/fixture-data")); + when(applicationConfiguration.getDnsName()).thenReturn("drs.anv0"); + when(drsConfiguration.maxDrsLookups()).thenReturn(10); + when(jobService.getActivePodCount()).thenReturn(1); + when(snapshotService.retrieve(SNAPSHOT_ID)).thenReturn(fixtureSnapshot()); + when(snapshotService.retrieveSnapshotProject(SNAPSHOT_ID)).thenReturn(new SnapshotProject()); + when(drsDao.retrieveReferencedSnapshotIds(any())).thenReturn(java.util.List.of(SNAPSHOT_ID)); + when(fileService.lookupSnapshotFSItem(any(), any(), anyInt())).thenReturn(fixtureFile()); + + Storage storage = mock(Storage.class); + Bucket bucket = mock(Bucket.class); + when(bucket.getLocation()).thenReturn("us-central1"); + when(storage.get(eq("fixture-bucket"), any(BucketGetOption[].class))).thenReturn(bucket); + doReturn(new URL("http://127.0.0.1:" + port + "/fixture-data")) + .when(storage) + .signUrl( + any(BlobInfo.class), + anyLong(), + any(TimeUnit.class), + any(Storage.SignUrlOption[].class)); + when(gcsProjectFactory.getStorage(any())).thenReturn(storage); Files.writeString(portFile, Integer.toString(port)); } @@ -110,22 +164,53 @@ void serveTdrApiUntilCiDriverStops() throws Exception { } } - private static DRSObject fixtureObject() throws NoSuchAlgorithmException { - CRC32C crc32c = new CRC32C(); + private static Snapshot fixtureSnapshot() { + BillingProfileModel billingProfile = + new BillingProfileModel().id(UUID.fromString("aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee")); + return new Snapshot() + .id(SNAPSHOT_ID) + .profileId(billingProfile.getId()) + .globalFileIds(true) + .projectResource(new GoogleProjectResource().googleProjectId("fixture-snapshot-project")) + .snapshotSources( + java.util.List.of( + new SnapshotSource() + .dataset( + new Dataset( + new DatasetSummary() + .selfHosted(true) + .defaultProfileId(billingProfile.getId()) + .cloudPlatform(CloudPlatform.GCP) + .billingProfiles(java.util.List.of(billingProfile))) + .id(UUID.fromString("bbbbbbbb-cccc-4ddd-8eee-ffffffffffff")) + .name("git-drs-ci-fixture") + .projectResource( + new GoogleProjectResource() + .googleProjectId("fixture-dataset-project"))))); + } + + private static FSFile fixtureFile() { + java.util.zip.CRC32C crc32c = new java.util.zip.CRC32C(); crc32c.update(FIXTURE_BYTES, 0, FIXTURE_BYTES.length); String crcHex = String.format("%08x", crc32c.getValue()); - String md5Hex = - java.util.HexFormat.of().formatHex(MessageDigest.getInstance("MD5").digest(FIXTURE_BYTES)); - - return new DRSObject() - .id(OBJECT_ID) - .selfUri("drs://localhost/" + OBJECT_ID) + String md5Hex; + try { + md5Hex = + java.util.HexFormat.of() + .formatHex(java.security.MessageDigest.getInstance("MD5").digest(FIXTURE_BYTES)); + } catch (java.security.NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + return new FSFile() + .fileId(FILE_ID) + .path("1614321.merge_output.gvcf.gz") + .cloudPath("gs://fixture-bucket/1614321.merge_output.gvcf.gz") + .cloudPlatform(CloudPlatform.GCP) + .bucketResourceId("fixture-bucket-resource") + .createdDate(java.time.Instant.parse("2026-10-07T00:00:00Z")) .size((long) FIXTURE_BYTES.length) - .checksums( - List.of( - new DRSChecksum().type("crc32c").checksum(crcHex), - new DRSChecksum().type("md5").checksum(md5Hex))) - .accessMethods(List.of(new DRSAccessMethod().type(TypeEnum.HTTPS).accessId(ACCESS_ID))); + .checksumCrc32c(crcHex) + .checksumMd5(md5Hex); } private static Path requiredPath(String envName) { @@ -145,16 +230,32 @@ private static void createParentDirectories(Path path) throws IOException { @SpringBootConfiguration @EnableAutoConfiguration(exclude = DataSourceAutoConfiguration.class) - @Import({DataRepositoryServiceApiController.class, FixtureDataController.class}) - static class HarnessApplication {} + @Import({ + DataRepositoryServiceApiController.class, + FixtureDataController.class, + DrsIdService.class, + DrsService.class + }) + static class HarnessApplication { + @org.springframework.context.annotation.Bean(name = "drsResolutionThreadpool") + AsyncTaskExecutor drsResolutionThreadpool() { + return new SimpleAsyncTaskExecutor("drs-resolution-test-"); + } + } @RestController static class FixtureDataController { @GetMapping("/fixture-data") ResponseEntity fixtureData() { + fixtureDataRequests.incrementAndGet(); return ResponseEntity.ok() .contentType(MediaType.APPLICATION_OCTET_STREAM) .body(FIXTURE_BYTES); } + + @GetMapping("/fixture-data-requests") + int fixtureDataRequestCount() { + return fixtureDataRequests.get(); + } } } diff --git a/tests/terra-tdr/run.sh b/tests/terra-tdr/run.sh index ecd860ca..a07dc749 100755 --- a/tests/terra-tdr/run.sh +++ b/tests/terra-tdr/run.sh @@ -14,12 +14,12 @@ stop_file="$coord_dir/stop" tdr_log="$coord_dir/tdr.log" cp "$git_drs_root/tests/terra-tdr/TerraDrsHarnessTest.java" \ - "$tdr_root/src/test/java/bio/terra/app/controller/TerraDrsHarnessTest.java" + "$tdr_root/src/test/java/bio/terra/service/filedata/TerraDrsHarnessTest.java" ( cd "$tdr_root" GIT_DRS_TDR_PORT_FILE="$port_file" GIT_DRS_TDR_STOP_FILE="$stop_file" \ - ./gradlew testUnit --tests bio.terra.app.controller.TerraDrsHarnessTest \ + ./gradlew testUnit --tests bio.terra.service.filedata.TerraDrsHarnessTest \ --no-daemon --console=plain ) >"$tdr_log" 2>&1 & tdr_pid=$! From 73ae942dbb6f7896931e7edf2a4867d44d98969a Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 8 Oct 2026 09:21:25 -0700 Subject: [PATCH 10/22] Exercise DRS Hub routing through real TDR service --- .github/workflows/pr-checks.yaml | 38 +++++ tests/terra-drs-hub/README.md | 13 ++ .../TerraDrsHubIntegrationTest.java | 116 +++++++++++++ tests/terra-drs-hub/run.sh | 158 ++++++++++++++++++ tests/terra-drs-hub/tdr_https_proxy.py | 81 +++++++++ 5 files changed, 406 insertions(+) create mode 100644 tests/terra-drs-hub/README.md create mode 100644 tests/terra-drs-hub/TerraDrsHubIntegrationTest.java create mode 100755 tests/terra-drs-hub/run.sh create mode 100644 tests/terra-drs-hub/tdr_https_proxy.py diff --git a/.github/workflows/pr-checks.yaml b/.github/workflows/pr-checks.yaml index 3bad64d9..402e1e7d 100644 --- a/.github/workflows/pr-checks.yaml +++ b/.github/workflows/pr-checks.yaml @@ -108,3 +108,41 @@ jobs: - name: Pull an AnVIL DRS pointer through TDR's controller run: tests/terra-tdr/run.sh "$GITHUB_WORKSPACE/tdr" + + terra-drs-hub: + name: Resolve through DRS Hub and TDR + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - name: Check out git-drs + uses: actions/checkout@v4 + + - name: Check out TDR + uses: actions/checkout@v4 + with: + repository: DataBiosphere/jade-data-repo + ref: 6433cb4ea296edf873daa30f1da29e7caf848b8b + path: tdr + persist-credentials: false + + - name: Check out DRS Hub + uses: actions/checkout@v4 + with: + repository: DataBiosphere/terra-drs-hub + ref: 618348c2efeb61d3fb50fd75c447d6ebf3f4192d + path: drs-hub + persist-credentials: false + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + cache: gradle + cache-dependency-path: | + tdr/build.gradle + drs-hub/service/build.gradle + + - name: Resolve the AnVIL compact ID through DRS Hub and TDR + run: tests/terra-drs-hub/run.sh "$GITHUB_WORKSPACE/tdr" "$GITHUB_WORKSPACE/drs-hub" diff --git a/tests/terra-drs-hub/README.md b/tests/terra-drs-hub/README.md new file mode 100644 index 00000000..c6dee548 --- /dev/null +++ b/tests/terra-drs-hub/README.md @@ -0,0 +1,13 @@ +# DRS Hub to TDR integration + +The `Resolve through DRS Hub and TDR` CI job runs the pinned public DRS Hub service and sends its real `POST /api/v4/drs/resolve` endpoint an AnVIL compact DRS ID. Hub expands `drs.anv0`, chooses its TDR provider, and makes the GA4GH DRS `OPTIONS`, object metadata, and access URL calls. Those HTTPS requests pass through a local TLS forwarder to the running TDR harness. The forwarder only translates HTTPS to the harness's local HTTP listener; it does not generate DRS responses. + +The TDR harness runs the actual DRS controller and `DrsService`. The snapshot, authorization, file record, and URL signing boundaries use test fixtures. The request uses the fixed test bearer token `tdr-ci-fixture`; authenticated metadata and access requests forward that token to TDR. Hub's DRS `OPTIONS` probe omits the bearer token, so the fixture rejects that probe and Hub exercises its configured-auth fallback before making the authenticated `GET` calls. The signed URL points to the harness's local fixture bytes, so the test needs no Google credentials or bucket access. + +This check covers Hub's AnVIL routing and TDR DRS client behavior separately from the `git-drs pull` integration check. `git-drs` currently resolves directly against TDR, so its pull test does not pass through Hub. + +To run locally, check out the pinned TDR and DRS Hub revisions used in `.github/workflows/pr-checks.yaml`, then run: + +```sh +tests/terra-drs-hub/run.sh /path/to/jade-data-repo /path/to/terra-drs-hub +``` diff --git a/tests/terra-drs-hub/TerraDrsHubIntegrationTest.java b/tests/terra-drs-hub/TerraDrsHubIntegrationTest.java new file mode 100644 index 00000000..2c216ba3 --- /dev/null +++ b/tests/terra-drs-hub/TerraDrsHubIntegrationTest.java @@ -0,0 +1,116 @@ +package bio.terra.drshub.controllers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.test.context.ActiveProfiles; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"test", "human-readable-logging", "hub-ci"}) +@Tag("Unit") +class TerraDrsHubIntegrationTest { + private static final String OBJECT_ID = "v2_4f770147-e372-339b-b9fa-0a7a83cf30cf"; + + @LocalServerPort private int port; + @Autowired private ObjectMapper objectMapper; + + @Test + void resolvesAnvilCompactIdThroughHubAndTheLiveTdrDrsApi() throws Exception { + Path proxyLog = Path.of(requiredEnv("GIT_DRS_TDR_PROXY_LOG")); + Files.deleteIfExists(proxyLog); + + String requestBody = + objectMapper.writeValueAsString( + java.util.Map.of( + "url", + "drs://drs.anv0:" + OBJECT_ID, + "fields", + List.of("size", "fileName", "hashes", "accessUrl"))); + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://127.0.0.1:" + port + "/api/v4/drs/resolve")) + .timeout(Duration.ofSeconds(30)) + .header("Authorization", "Bearer tdr-ci-fixture") + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + HttpResponse response = + HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + assertEquals(200, response.statusCode(), response.body()); + + JsonNode metadata = objectMapper.readTree(response.body()); + assertEquals(43, metadata.path("size").asLong()); + assertEquals("1614321.merge_output.gvcf.gz", metadata.path("fileName").asText()); + assertTrue(metadata.path("hashes").isObject(), response.body()); + assertEquals("d0147894", metadata.path("hashes").path("crc32c").asText()); + assertEquals("e3fbcfb74b00bf57af80060c0ac10d03", metadata.path("hashes").path("md5").asText()); + assertEquals( + "http://127.0.0.1:" + requiredEnv("GIT_DRS_TDR_HTTP_PORT") + "/fixture-data", + metadata.path("accessUrl").path("url").asText()); + + List proxiedRequests = + Files.readAllLines(proxyLog).stream().map(this::readJson).toList(); + assertTrue( + proxiedRequests.stream() + .anyMatch( + entry -> + entry.path("method").asText().equals("OPTIONS") + && entry.path("path").asText().endsWith("/objects/" + OBJECT_ID)), + proxiedRequests.toString()); + assertTrue( + proxiedRequests.stream() + .anyMatch( + entry -> + entry.path("method").asText().equals("GET") + && entry.path("path").asText().endsWith("/objects/" + OBJECT_ID)), + proxiedRequests.toString()); + assertTrue( + proxiedRequests.stream() + .anyMatch( + entry -> + entry.path("method").asText().equals("GET") + && entry + .path("path") + .asText() + .contains("/objects/" + OBJECT_ID + "/access/")), + proxiedRequests.toString()); + assertTrue( + proxiedRequests.stream() + .filter(entry -> entry.path("method").asText().equals("GET")) + .allMatch( + entry -> entry.path("authorization").asText().equals("Bearer tdr-ci-fixture")), + proxiedRequests.toString()); + } + + private JsonNode readJson(String line) { + try { + return objectMapper.readTree(line); + } catch (Exception e) { + throw new IllegalStateException("Invalid TDR proxy log entry: " + line, e); + } + } + + private static String requiredEnv(String name) { + String value = System.getenv(name); + if (value == null || value.isBlank()) { + throw new IllegalStateException(name + " must be set by the CI driver"); + } + return value; + } +} diff --git a/tests/terra-drs-hub/run.sh b/tests/terra-drs-hub/run.sh new file mode 100755 index 00000000..073f3af4 --- /dev/null +++ b/tests/terra-drs-hub/run.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 /path/to/jade-data-repo /path/to/terra-drs-hub" >&2 + exit 2 +fi + +git_drs_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +tdr_root=$(cd "$1" && pwd) +hub_root=$(cd "$2" && pwd) +coord_dir=$(mktemp -d) +port_file="$coord_dir/tdr-port" +stop_file="$coord_dir/tdr-stop" +tdr_log="$coord_dir/tdr.log" +hub_log="$coord_dir/hub.log" +proxy_log="$coord_dir/proxy.log" +hub_compile_log="$coord_dir/hub-compile.log" +certificate="$coord_dir/localhost.crt" +private_key="$coord_dir/localhost.key" +truststore="$coord_dir/truststore.p12" + +cp "$git_drs_root/tests/terra-tdr/TerraDrsHarnessTest.java" \ + "$tdr_root/src/test/java/bio/terra/service/filedata/TerraDrsHarnessTest.java" +cp "$git_drs_root/tests/terra-drs-hub/TerraDrsHubIntegrationTest.java" \ + "$hub_root/service/src/test/java/bio/terra/drshub/controllers/TerraDrsHubIntegrationTest.java" +cat >"$hub_root/service/src/test/resources/application-hub-ci.yaml" <<'YAML' +drshub: + bardEventLoggingEnabled: false + compactIdHosts: + "drs.anv0": "localhost:${GIT_DRS_TDR_TLS_PORT}" + drsProviders: + terraDataRepo: + hostRegex: "localhost:${GIT_DRS_TDR_TLS_PORT}" + accessMethodConfigs: + - type: gs + auth: current_request + fetchAccessUrl: true +YAML + +openssl req -new -newkey rsa:2048 -nodes -keyout "$private_key" -x509 -days 1 \ + -out "$certificate" -subj "/CN=localhost" \ + -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" + +java_home=${JAVA_HOME:-$(java -XshowSettings:properties -version 2>&1 | sed -n 's/^[[:space:]]*java.home = //p' | head -n 1)} +if [[ -z "$java_home" ]]; then + echo "Could not locate the active Java installation" >&2 + exit 1 +fi +cp -L "$java_home/lib/security/cacerts" "$truststore" +keytool -importcert -noprompt -alias git-drs-tdr-harness -file "$certificate" \ + -keystore "$truststore" -storepass changeit + +cd "$hub_root" +if ! JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:-} -Djavax.net.ssl.trustStore=$truststore -Djavax.net.ssl.trustStorePassword=changeit" \ + ./gradlew :service:testClasses --no-daemon --console=plain >"$hub_compile_log" 2>&1; then + cat "$hub_compile_log" >&2 + exit 1 +fi + +tls_port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') + +( + cd "$tdr_root" + GIT_DRS_TDR_PORT_FILE="$port_file" GIT_DRS_TDR_STOP_FILE="$stop_file" \ + ./gradlew testUnit --tests bio.terra.service.filedata.TerraDrsHarnessTest \ + --no-daemon --console=plain +) >"$tdr_log" 2>&1 & +tdr_pid=$! + +proxy_pid= +stop_services() { + touch "$stop_file" + if [[ -n "$proxy_pid" ]]; then + kill "$proxy_pid" 2>/dev/null || true + wait "$proxy_pid" 2>/dev/null || true + fi + wait "$tdr_pid" 2>/dev/null || true +} +trap stop_services EXIT + +for ((seconds = 0; seconds < 900; seconds++)); do + if [[ -s "$port_file" ]]; then + break + fi + if ! kill -0 "$tdr_pid" 2>/dev/null; then + cat "$tdr_log" >&2 + exit 1 + fi + sleep 1 +done + +if [[ ! -s "$port_file" ]]; then + echo "TDR DRS service did not start within 15 minutes" >&2 + cat "$tdr_log" >&2 + exit 1 +fi + +tdr_port=$(cat "$port_file") +python3 "$git_drs_root/tests/terra-drs-hub/tdr_https_proxy.py" \ + "http://127.0.0.1:$tdr_port" "$tls_port" "$certificate" "$private_key" "$proxy_log" \ + >"$coord_dir/proxy.out" 2>&1 & +proxy_pid=$! + +for ((attempt = 0; attempt < 30; attempt++)); do + if python3 - "$tls_port" <<'PY' +import socket +import ssl +import sys + +context = ssl.create_default_context() +context.check_hostname = False +context.verify_mode = ssl.CERT_NONE +try: + connection = socket.create_connection(("127.0.0.1", int(sys.argv[1])), timeout=1) + with context.wrap_socket(connection, server_hostname="localhost"): + pass +except OSError: + raise SystemExit(1) +PY + then + break + fi + if ! kill -0 "$proxy_pid" 2>/dev/null; then + cat "$coord_dir/proxy.out" >&2 + exit 1 + fi + sleep 1 +done + +cd "$hub_root" +if ! GIT_DRS_TDR_TLS_PORT="$tls_port" \ + GIT_DRS_TDR_HTTP_PORT="$tdr_port" \ + GIT_DRS_TDR_PROXY_LOG="$proxy_log" \ + JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:-} -Djavax.net.ssl.trustStore=$truststore -Djavax.net.ssl.trustStorePassword=changeit" \ + ./gradlew :service:test --tests bio.terra.drshub.controllers.TerraDrsHubIntegrationTest \ + --no-daemon --console=plain -x :service:runMinnieKenny >"$hub_log" 2>&1; then + cat "$hub_log" >&2 + cat "$tdr_log" >&2 + cat "$proxy_log" >&2 2>/dev/null || true + cat "$coord_dir/proxy.out" >&2 2>/dev/null || true + exit 1 +fi + +touch "$stop_file" +if ! wait "$tdr_pid"; then + cat "$tdr_log" >&2 + exit 1 +fi +kill "$proxy_pid" 2>/dev/null || true +wait "$proxy_pid" 2>/dev/null || true +proxy_pid= +trap - EXIT + +cat "$hub_log" +printf '\nTDR proxy requests:\n' +cat "$proxy_log" +tail -n 20 "$tdr_log" diff --git a/tests/terra-drs-hub/tdr_https_proxy.py b/tests/terra-drs-hub/tdr_https_proxy.py new file mode 100644 index 00000000..82069670 --- /dev/null +++ b/tests/terra-drs-hub/tdr_https_proxy.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Forward Hub's HTTPS DRS calls to the local HTTP TDR test server.""" + +import http.client +import json +import ssl +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlsplit + + +upstream = urlsplit(sys.argv[1]) +listen_port = int(sys.argv[2]) +certificate = sys.argv[3] +private_key = sys.argv[4] +request_log = sys.argv[5] + + +class ProxyHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_OPTIONS(self): + self.forward() + + def do_GET(self): + self.forward() + + def do_POST(self): + self.forward() + + def do_HEAD(self): + self.forward() + + def forward(self): + body = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + request = { + "method": self.command, + "path": self.path, + "authorization": self.headers.get("Authorization", ""), + } + with open(request_log, "a", encoding="utf-8") as log: + log.write(json.dumps(request) + "\n") + + connection = http.client.HTTPConnection(upstream.hostname, upstream.port, timeout=30) + try: + headers = { + name: value + for name, value in self.headers.items() + if name.lower() not in {"host", "connection", "content-length"} + } + connection.request(self.command, self.path, body=body, headers=headers) + response = connection.getresponse() + response_body = response.read() + self.send_response(response.status, response.reason) + for name, value in response.getheaders(): + if name.lower() not in {"connection", "transfer-encoding", "content-length"}: + self.send_header(name, value) + self.send_header("Content-Length", str(len(response_body))) + self.send_header("Connection", "close") + self.end_headers() + if self.command != "HEAD": + self.wfile.write(response_body) + except Exception as error: # Return the forwarding failure to the real Hub client. + response_body = str(error).encode("utf-8") + self.send_response(502) + self.send_header("Content-Length", str(len(response_body))) + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(response_body) + finally: + connection.close() + + def log_message(self, format, *args): + print(format % args, file=sys.stderr) + + +server = ThreadingHTTPServer(("127.0.0.1", listen_port), ProxyHandler) +context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +context.load_cert_chain(certificate, private_key) +server.socket = context.wrap_socket(server.socket, server_side=True) +server.serve_forever() From 037530319e8c663c583920faf1202f5e306be9b5 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 8 Oct 2026 09:22:47 -0700 Subject: [PATCH 11/22] Make Hub test truststore writable on CI runners --- tests/terra-drs-hub/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/terra-drs-hub/run.sh b/tests/terra-drs-hub/run.sh index 073f3af4..9258c5c9 100755 --- a/tests/terra-drs-hub/run.sh +++ b/tests/terra-drs-hub/run.sh @@ -48,6 +48,7 @@ if [[ -z "$java_home" ]]; then exit 1 fi cp -L "$java_home/lib/security/cacerts" "$truststore" +chmod u+w "$truststore" keytool -importcert -noprompt -alias git-drs-tdr-harness -file "$certificate" \ -keystore "$truststore" -storepass changeit From c9f0e4eb20be85b1bd9e6e3713f27e8dee3d8e83 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 8 Oct 2026 10:21:42 -0700 Subject: [PATCH 12/22] Exercise Terra pull through Hub, TDR, and local storage --- .github/workflows/pr-checks.yaml | 7 +- cmd/addref/add-ref.go | 12 +- cmd/filter/main.go | 6 +- cmd/pull/main.go | 9 +- cmd/pull/terra_gcs_url_rewrite_test.go | 37 ++ cmd/pull/terra_hub_integration_test.go | 130 +++++++ cmd/pull/terra_tdr_integration_test.go | 22 +- cmd/remote/add/init.go | 1 + cmd/remote/add/unified.go | 18 +- cmd/remote/add/unified_test.go | 36 +- cmd/smudge/main.go | 6 +- docs/anvil.md | 11 + internal/config/config.go | 55 ++- internal/config/config_test.go | 28 ++ internal/config/remote.go | 13 +- internal/remoteruntime/runtime.go | 4 +- internal/remoteruntime/runtime_test.go | 26 +- internal/resolver/resolver.go | 146 ++++++++ internal/resolver/terra_hub_test.go | 82 +++++ tests/terra-drs-hub/README.md | 10 +- .../TerraDrsHubIntegrationTest.java | 107 +----- tests/terra-drs-hub/run.sh | 70 +++- tests/terra-tdr/README.md | 12 +- tests/terra-tdr/TerraDrsHarnessTest.java | 325 ++++++++++++++---- tests/terra-tdr/firestore-emulator.sh | 52 +++ tests/terra-tdr/run.sh | 34 +- tests/terra-tdr/storage-emulator.sh | 58 ++++ 27 files changed, 1099 insertions(+), 218 deletions(-) create mode 100644 cmd/pull/terra_gcs_url_rewrite_test.go create mode 100644 cmd/pull/terra_hub_integration_test.go create mode 100644 internal/resolver/terra_hub_test.go create mode 100644 tests/terra-tdr/firestore-emulator.sh create mode 100755 tests/terra-tdr/storage-emulator.sh diff --git a/.github/workflows/pr-checks.yaml b/.github/workflows/pr-checks.yaml index 402e1e7d..a92d4f73 100644 --- a/.github/workflows/pr-checks.yaml +++ b/.github/workflows/pr-checks.yaml @@ -144,5 +144,10 @@ jobs: tdr/build.gradle drs-hub/service/build.gradle - - name: Resolve the AnVIL compact ID through DRS Hub and TDR + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.26.6' + + - name: Pull an AnVIL DRS pointer through DRS Hub and TDR run: tests/terra-drs-hub/run.sh "$GITHUB_WORKSPACE/tdr" "$GITHUB_WORKSPACE/drs-hub" diff --git a/cmd/addref/add-ref.go b/cmd/addref/add-ref.go index 03b75bbf..62a35e77 100644 --- a/cmd/addref/add-ref.go +++ b/cmd/addref/add-ref.go @@ -333,7 +333,13 @@ type drsObjectGetter interface { func resolveAddRefObject(ctx context.Context, cfg *config.Config, primaryRemote config.Remote, primary *remoteruntime.GitContext, drsURI string) (drsapi.DrsObject, error) { if primary != nil && primary.RemoteType == config.TerraServerType { - anvil, err := newAnVILResolver(ctx, primary.Endpoint) + var anvil resolver.Resolver + var err error + if primary.HubEndpoint != "" { + anvil, err = newHubResolver(ctx, primary.HubEndpoint) + } else { + anvil, err = newAnVILResolver(ctx, primary.Endpoint) + } if err != nil { return drsapi.DrsObject{}, err } @@ -376,6 +382,10 @@ var newAnVILResolver = func(ctx context.Context, endpoint string) (resolver.Reso return resolver.NewAnVIL(ctx, endpoint) } +var newHubResolver = func(ctx context.Context, endpoint string) (resolver.Resolver, error) { + return resolver.NewTerraHub(ctx, endpoint) +} + func parseDRSURIForSource(drsURI string) (objectID string, endpoint string, ok bool) { u, err := url.Parse(strings.TrimSpace(drsURI)) if err != nil || !strings.EqualFold(u.Scheme, "drs") || strings.TrimSpace(u.Host) == "" { diff --git a/cmd/filter/main.go b/cmd/filter/main.go index d33b0d0d..e0d7e621 100644 --- a/cmd/filter/main.go +++ b/cmd/filter/main.go @@ -65,7 +65,11 @@ func runFilter(cmd *cobra.Command, _ []string) error { if err != nil { logger.Info("DRS server not configured or unreachable", "err", err) } else if drsCtx.RemoteType == config.TerraServerType && !internalfilter.ShouldSkipSmudge() { - terraResolver, err = resolver.NewAnVIL(ctx, drsCtx.Endpoint) + if drsCtx.HubEndpoint != "" { + terraResolver, err = resolver.NewTerraHub(ctx, drsCtx.HubEndpoint) + } else { + terraResolver, err = resolver.NewAnVIL(ctx, drsCtx.Endpoint) + } if err != nil { return fmt.Errorf("filter: create Terra resolver: %w", err) } diff --git a/cmd/pull/main.go b/cmd/pull/main.go index 149f123d..93e9e332 100644 --- a/cmd/pull/main.go +++ b/cmd/pull/main.go @@ -46,6 +46,9 @@ var ( newAnVILResolver = func(ctx context.Context, endpoint string) (resolver.Resolver, error) { return resolver.NewAnVIL(ctx, endpoint) } + newHubResolver = func(ctx context.Context, endpoint string) (resolver.Resolver, error) { + return resolver.NewTerraHub(ctx, endpoint) + } loadWorktreeInventory = lfs.GetTrackedLfsFilesAt loadWorktreeFile = lfs.GetTrackedLfsFileAt ) @@ -158,7 +161,11 @@ var Cmd = &cobra.Command{ return fmt.Errorf("remote %q does not support resolving and downloading DRS objects", remote) } if drsCtx.IsReadOnly() { - anvil, err = newAnVILResolver(ctx, drsCtx.Endpoint) + if drsCtx.HubEndpoint != "" { + anvil, err = newHubResolver(ctx, drsCtx.HubEndpoint) + } else { + anvil, err = newAnVILResolver(ctx, drsCtx.Endpoint) + } if err != nil { return err } diff --git a/cmd/pull/terra_gcs_url_rewrite_test.go b/cmd/pull/terra_gcs_url_rewrite_test.go new file mode 100644 index 00000000..44e3292c --- /dev/null +++ b/cmd/pull/terra_gcs_url_rewrite_test.go @@ -0,0 +1,37 @@ +//go:build integration + +package pull + +import ( + "bytes" + "fmt" + "io" + "net/http" + "strings" +) + +func rewriteTDRSignedURLForEmulator(response *http.Response, emulatorEndpoint string) error { + if response == nil || response.Body == nil { + return fmt.Errorf("TDR response has no body") + } + body, err := io.ReadAll(response.Body) + if err != nil { + return fmt.Errorf("read TDR response: %w", err) + } + _ = response.Body.Close() + if !bytes.Contains(body, []byte("X-Goog-Signature=")) { + return fmt.Errorf("TDR access response is missing a V4 signed URL") + } + const sourceAuthority = "http://127.0.0.1/" + if !bytes.Contains(body, []byte(sourceAuthority)) { + return fmt.Errorf("TDR signed URL does not use the expected local storage host") + } + if !strings.HasPrefix(emulatorEndpoint, "http://127.0.0.1:") { + return fmt.Errorf("invalid local GCS emulator endpoint %q", emulatorEndpoint) + } + body = bytes.Replace(body, []byte(sourceAuthority), []byte(strings.TrimRight(emulatorEndpoint, "/")+"/"), 1) + response.Body = io.NopCloser(bytes.NewReader(body)) + response.ContentLength = int64(len(body)) + response.Header.Set("Content-Length", fmt.Sprint(len(body))) + return nil +} diff --git a/cmd/pull/terra_hub_integration_test.go b/cmd/pull/terra_hub_integration_test.go new file mode 100644 index 00000000..39cb45e1 --- /dev/null +++ b/cmd/pull/terra_hub_integration_test.go @@ -0,0 +1,130 @@ +//go:build integration + +package pull + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/calypr/git-drs/internal/config" + "github.com/calypr/git-drs/internal/lfs" + "github.com/calypr/git-drs/internal/resolver" +) + +func TestIntegrationPullThroughTerraHubAndTDR(t *testing.T) { + hubEndpoint := os.Getenv("GIT_DRS_HUB_HTTP_ENDPOINT") + proxyLog := os.Getenv("GIT_DRS_TDR_PROXY_LOG") + gcsEndpoint := os.Getenv("GIT_DRS_TDR_GCS_ENDPOINT") + if hubEndpoint == "" || proxyLog == "" || gcsEndpoint == "" { + t.Skip("GIT_DRS_HUB_HTTP_ENDPOINT, GIT_DRS_TDR_PROXY_LOG, and GIT_DRS_TDR_GCS_ENDPOINT are required") + } + parsed, err := url.Parse(hubEndpoint) + if err != nil || parsed.Scheme != "http" || parsed.Hostname() != "127.0.0.1" { + t.Fatalf("invalid local Hub endpoint %q", hubEndpoint) + } + + oldResolver := newHubResolver + t.Cleanup(func() { + newHubResolver = oldResolver + resetPullFlagsForTest() + }) + var hubCalls int + newHubResolver = func(_ context.Context, endpoint string) (resolver.Resolver, error) { + client := &http.Client{Transport: pullRoundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.Path == "/api/v4/drs/resolve" { + hubCalls++ + } + copy := req.Clone(req.Context()) + copy.Header.Set("Authorization", "Bearer tdr-ci-fixture") + response, err := http.DefaultTransport.RoundTrip(copy) + if err != nil { + return nil, err + } + if req.URL.Path == "/api/v4/drs/resolve" { + if err := rewriteTDRSignedURLForEmulator(response, gcsEndpoint); err != nil { + return nil, err + } + } + return response, nil + })} + return resolver.NewTerraHubWithClient(endpoint, client) + } + + repo := t.TempDir() + runGitCmdTest(t, repo, "init", "-q") + t.Chdir(repo) + const filename = "1614321.merge_output.gvcf.gz" + const oid = "drs://drs.anv0:v2_4f770147-e372-339b-b9fa-0a7a83cf30cf" + payload := []byte("git-drs Terra DRS HTTP integration fixture\n") + if err := os.WriteFile(".gitattributes", []byte(filename+" filter=drs diff=drs merge=drs -text\n"), 0o644); err != nil { + t.Fatal(err) + } + pointer := fmt.Sprintf("version https://calypr.github.io/spec/v1\noid %s\nsize %d\n", oid, len(payload)) + if err := os.WriteFile(filename, []byte(pointer), 0o644); err != nil { + t.Fatal(err) + } + runGitCmdTest(t, repo, "add", filename) + if _, err := config.UpdateRemote("anvil", config.RemoteSelect{Terra: &config.TerraRemote{ + Endpoint: "https://data.terra.bio", HubEndpoint: hubEndpoint, Mode: "read-only", + }}); err != nil { + t.Fatal(err) + } + objectsRoot, err := lfs.ResolveObjectsRoot(context.Background()) + if err != nil { + t.Fatal(err) + } + cachePath, err := lfs.ObjectPath(objectsRoot, oid) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(cachePath); !os.IsNotExist(err) { + t.Fatalf("cache must be empty before pull: %s: %v", cachePath, err) + } + resetPullFlagsForTest() + includePatterns = []string{filename} + if err := Cmd.RunE(Cmd, nil); err != nil { + t.Fatalf("pull through Hub and TDR: %v", err) + } + got, err := os.ReadFile(filepath.Join(repo, filename)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, payload) { + t.Fatalf("hydrated content = %q, want %q", got, payload) + } + if hubCalls != 1 { + t.Fatalf("Hub resolve calls = %d, want one", hubCalls) + } + data, err := os.ReadFile(proxyLog) + if err != nil { + t.Fatal(err) + } + var metadata, access bool + for _, line := range bytes.Split(bytes.TrimSpace(data), []byte("\n")) { + var request struct { + Method string `json:"method"` + Path string `json:"path"` + Authorization string `json:"authorization"` + } + if err := json.Unmarshal(line, &request); err != nil { + t.Fatalf("invalid TDR proxy log line %q: %v", line, err) + } + if request.Method == "GET" && strings.HasSuffix(request.Path, "/objects/v2_4f770147-e372-339b-b9fa-0a7a83cf30cf") { + metadata = request.Authorization == "Bearer tdr-ci-fixture" + } + if request.Method == "GET" && strings.Contains(request.Path, "/objects/v2_4f770147-e372-339b-b9fa-0a7a83cf30cf/access/") { + access = request.Authorization == "Bearer tdr-ci-fixture" + } + } + if !metadata || !access { + t.Fatalf("Hub did not make authenticated TDR metadata and access requests: %s", data) + } +} diff --git a/cmd/pull/terra_tdr_integration_test.go b/cmd/pull/terra_tdr_integration_test.go index fab6be32..cca198ea 100644 --- a/cmd/pull/terra_tdr_integration_test.go +++ b/cmd/pull/terra_tdr_integration_test.go @@ -13,6 +13,7 @@ import ( "os" "path/filepath" "reflect" + "strings" "sync" "testing" @@ -23,8 +24,9 @@ import ( func TestIntegrationPullAgainstTDRController(t *testing.T) { endpoint := os.Getenv("GIT_DRS_TDR_HTTP_ENDPOINT") - if endpoint == "" { - t.Skip("GIT_DRS_TDR_HTTP_ENDPOINT is required") + gcsEndpoint := os.Getenv("GIT_DRS_TDR_GCS_ENDPOINT") + if endpoint == "" || gcsEndpoint == "" { + t.Skip("GIT_DRS_TDR_HTTP_ENDPOINT and GIT_DRS_TDR_GCS_ENDPOINT are required") } tdrURL, err := url.Parse(endpoint) if err != nil || tdrURL.Scheme != "http" || tdrURL.Host == "" { @@ -32,6 +34,12 @@ func TestIntegrationPullAgainstTDRController(t *testing.T) { } reverseProxy := httputil.NewSingleHostReverseProxy(tdrURL) + reverseProxy.ModifyResponse = func(response *http.Response) error { + if strings.Contains(response.Request.URL.Path, "/access/") { + return rewriteTDRSignedURLForEmulator(response, gcsEndpoint) + } + return nil + } var requestMu sync.Mutex var requestPaths []string proxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -101,16 +109,6 @@ func TestIntegrationPullAgainstTDRController(t *testing.T) { if !bytes.Equal(got, payload) { t.Fatalf("hydrated content = %q, want %q", got, payload) } - resp, err := http.Get(endpoint + "/fixture-data-requests") - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - var fetches int - _, scanErr := fmt.Fscan(resp.Body, &fetches) - if resp.StatusCode != http.StatusOK || scanErr != nil || fetches != 1 { - t.Fatalf("fixture data downloads = %d with HTTP %d, want one", fetches, resp.StatusCode) - } requestMu.Lock() gotPaths := append([]string(nil), requestPaths...) requestMu.Unlock() diff --git a/cmd/remote/add/init.go b/cmd/remote/add/init.go index 6658b1d8..f5929403 100644 --- a/cmd/remote/add/init.go +++ b/cmd/remote/add/init.go @@ -29,6 +29,7 @@ func init() { Cmd.PersistentFlags().StringVar(&providerFlag, "provider", "auto", "DRS provider adapter") Cmd.PersistentFlags().StringVar(&storageFlag, "storage", "", "Advanced publishing bucket/prefix") Cmd.PersistentFlags().StringVar(&checkoutFlag, "checkout", "", "Checkout mode: pointers or hydrate") + Cmd.PersistentFlags().StringVar(&hubEndpointFlag, "hub-endpoint", "", "Terra DRS Hub resolver base URL (for example https://drshub.dsde-prod.broadinstitute.org)") Gen3Cmd.Flags().StringVar(&credFile, "cred", "", "[gen3] Import a Gen3 credential file into this profile") Gen3Cmd.Flags().StringVar(&fenceToken, "token", "", "[gen3] Use a temporary bearer token issued from fence") Gen3Cmd.Flags().StringVar(&selectedBucket, "bucket", "", "[gen3] Select a specific visible bucket when multiple buckets match the scope") diff --git a/cmd/remote/add/unified.go b/cmd/remote/add/unified.go index ee1afd2f..68f435f7 100644 --- a/cmd/remote/add/unified.go +++ b/cmd/remote/add/unified.go @@ -16,7 +16,7 @@ import ( var ( scopeFlag, authFlag, credentialFlag, providerFlag string - storageFlag, checkoutFlag string + storageFlag, checkoutFlag, hubEndpointFlag string ) func runUnified(cmd *cobra.Command, args []string) error { @@ -58,6 +58,17 @@ func runUnified(cmd *cobra.Command, args []string) error { if err := validateProviderAuth(provider, auth); err != nil { return err } + hubEndpoint := strings.TrimSpace(hubEndpointFlag) + if hubEndpoint != "" { + hubURL, parseErr := url.ParseRequestURI(hubEndpoint) + if provider != "terra" { + return fmt.Errorf("--hub-endpoint is only supported for the Terra provider") + } + if parseErr != nil || hubURL.Scheme != "https" || hubURL.Host == "" || hubURL.User != nil || hubURL.RawQuery != "" || hubURL.Fragment != "" { + return fmt.Errorf("--hub-endpoint must be an HTTPS URL without credentials") + } + hubEndpoint = hubURL.String() + } u, err := url.ParseRequestURI(strings.TrimSpace(endpoint)) if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil { return fmt.Errorf("endpoint must be an HTTPS URL without credentials, or a built-in alias") @@ -91,8 +102,11 @@ func runUnified(cmd *cobra.Command, args []string) error { return fmt.Errorf("remote %q already exists; choose an explicit name or remove it first", name) } } - r := &config.GenericRemote{Endpoint: u.String(), Provider: provider, Auth: auth, Credential: credentialFlag, Scope: scopeFlag, Storage: storageFlag, Checkout: checkoutFlag, Preset: presetName, PresetVersion: version, RegistryServiceID: registryID} + r := &config.GenericRemote{Endpoint: u.String(), HubEndpoint: hubEndpoint, Provider: provider, Auth: auth, Credential: credentialFlag, Scope: scopeFlag, Storage: storageFlag, Checkout: checkoutFlag, Preset: presetName, PresetVersion: version, RegistryServiceID: registryID} fmt.Fprintf(cmd.OutOrStdout(), "Remote: %s\nEndpoint: %s\nProvider: %s\nAuth: %s\n", name, r.Endpoint, r.Provider, r.Auth) + if r.HubEndpoint != "" { + fmt.Fprintf(cmd.OutOrStdout(), "DRS resolver: %s\n", r.HubEndpoint) + } if r.Preset != "" { fmt.Fprintf(cmd.OutOrStdout(), "Preset: %s (built-in catalog v%d)\n", r.Preset, r.PresetVersion) } diff --git a/cmd/remote/add/unified_test.go b/cmd/remote/add/unified_test.go index ece26d26..219f82d1 100644 --- a/cmd/remote/add/unified_test.go +++ b/cmd/remote/add/unified_test.go @@ -31,6 +31,36 @@ func TestUnifiedAddExpandsPreset(t *testing.T) { } } +func TestUnifiedAddOptsTerraRemoteIntoDrsHub(t *testing.T) { + testutils.SetupTestGitRepo(t) + resetUnifiedFlags(t) + hubEndpointFlag = "https://drshub.dsde-prod.broadinstitute.org" + if err := runUnified(Cmd, []string{"anvil", "terra"}); err != nil { + t.Fatal(err) + } + cfg, err := config.LoadConfig() + if err != nil { + t.Fatal(err) + } + r := cfg.Remotes["anvil"].Generic + if r == nil || r.Endpoint != "https://data.terra.bio" || r.HubEndpoint != hubEndpointFlag || r.Provider != "terra" { + t.Fatalf("unexpected Terra Hub remote: %+v", r) + } +} + +func TestUnifiedAddRejectsHubEndpointForNonTerraProvider(t *testing.T) { + repo := testutils.SetupTestGitRepo(t) + resetUnifiedFlags(t) + providerFlag = "ga4gh" + hubEndpointFlag = "https://drshub.dsde-prod.broadinstitute.org" + if err := runUnified(Cmd, []string{"https://drs.example.org"}); err == nil { + t.Fatal("expected Hub endpoint to be restricted to Terra") + } + if _, err := os.Stat(repo + "/.git-drs"); !os.IsNotExist(err) { + t.Fatalf("invalid remote modified repository state: %v", err) + } +} + func TestUnifiedAddDerivesURLNameAndRejectsHTTP(t *testing.T) { testutils.SetupTestGitRepo(t) resetUnifiedFlags(t) @@ -127,9 +157,9 @@ func TestUnifiedAddRejectsStdinCredentialBeforePersisting(t *testing.T) { } func resetUnifiedFlags(t *testing.T) { - old := []string{scopeFlag, authFlag, credentialFlag, providerFlag, storageFlag, checkoutFlag} - scopeFlag, authFlag, credentialFlag, providerFlag, storageFlag, checkoutFlag = "", "auto", "", "auto", "", "" + old := []string{scopeFlag, authFlag, credentialFlag, providerFlag, storageFlag, checkoutFlag, hubEndpointFlag} + scopeFlag, authFlag, credentialFlag, providerFlag, storageFlag, checkoutFlag, hubEndpointFlag = "", "auto", "", "auto", "", "", "" t.Cleanup(func() { - scopeFlag, authFlag, credentialFlag, providerFlag, storageFlag, checkoutFlag = old[0], old[1], old[2], old[3], old[4], old[5] + scopeFlag, authFlag, credentialFlag, providerFlag, storageFlag, checkoutFlag, hubEndpointFlag = old[0], old[1], old[2], old[3], old[4], old[5], old[6] }) } diff --git a/cmd/smudge/main.go b/cmd/smudge/main.go index 1e799e5e..93bad8e2 100644 --- a/cmd/smudge/main.go +++ b/cmd/smudge/main.go @@ -79,7 +79,11 @@ func runSmudge(cmd *cobra.Command, args []string) error { if !internalfilter.ShouldSkipSmudge() { var terraResolver resolver.Resolver if drsCtx.RemoteType == config.TerraServerType { - terraResolver, err = resolver.NewAnVIL(ctx, drsCtx.Endpoint) + if drsCtx.HubEndpoint != "" { + terraResolver, err = resolver.NewTerraHub(ctx, drsCtx.HubEndpoint) + } else { + terraResolver, err = resolver.NewAnVIL(ctx, drsCtx.Endpoint) + } if err != nil { return fmt.Errorf("smudge: create Terra resolver: %w", err) } diff --git a/docs/anvil.md b/docs/anvil.md index 836f39ff..aeb0174b 100644 --- a/docs/anvil.md +++ b/docs/anvil.md @@ -32,6 +32,17 @@ flag is needed. The command stores only public remote metadata in `.git/config`. Google ADC remains local user state. Each clone must run the command because `.git/config` is not tracked. +To resolve DRS identifiers through Terra DRS Hub, add `--hub-endpoint` to the +remote creation command: + +```bash +git drs remote add anvil terra --checkout hydrate \ + --hub-endpoint https://drshub.dsde-prod.broadinstitute.org +``` + +The `--hub-endpoint` option changes DRS resolution to Hub's `/api/v4/drs/resolve` +API. The Terra `Endpoint` remains the TDR service URL used for service checks. + ## Publish one reference Use the configured remote and choose the path that the data should occupy: diff --git a/internal/config/config.go b/internal/config/config.go index 7b900875..8b0d8278 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -255,6 +255,9 @@ func UpdateRemote(name Remote, remote RemoteSelect) (*Config, error) { } else if remote.Terra != nil { remoteSubsection.SetOption("type", "terra") remoteSubsection.SetOption("endpoint", remote.Terra.Endpoint) + if remote.Terra.HubEndpoint != "" { + remoteSubsection.SetOption("hub-endpoint", remote.Terra.HubEndpoint) + } if remote.Terra.Auth != "" { remoteSubsection.SetOption("auth", remote.Terra.Auth) } @@ -282,7 +285,7 @@ func UpdateRemote(name Remote, remote RemoteSelect) (*Config, error) { remoteSubsection.SetOption("endpoint", r.Endpoint) remoteSubsection.SetOption("provider", r.Provider) remoteSubsection.SetOption("auth", r.Auth) - for key, value := range map[string]string{"credential": r.Credential, "scope": r.Scope, "storage": r.Storage, "checkout": r.Checkout, "preset": r.Preset, "registry-service-id": r.RegistryServiceID} { + for key, value := range map[string]string{"hub-endpoint": r.HubEndpoint, "credential": r.Credential, "scope": r.Scope, "storage": r.Storage, "checkout": r.Checkout, "preset": r.Preset, "registry-service-id": r.RegistryServiceID} { if value != "" { remoteSubsection.SetOption(key, value) } @@ -307,7 +310,7 @@ func UpdateRemote(name Remote, remote RemoteSelect) (*Config, error) { return LoadConfig() } -func parseAndAddRemote(cfg *Config, subsectionName string, remoteType string, endpoint string, project string, bucket string, organization string, storagePrefix string, auth string, mode string) { +func parseAndAddRemote(cfg *Config, subsectionName string, remoteType string, endpoint string, project string, bucket string, organization string, storagePrefix string, auth string, mode string, hubEndpoint string) { if !strings.HasPrefix(subsectionName, remoteSubsectionPrefix) { return } @@ -325,9 +328,10 @@ func parseAndAddRemote(cfg *Config, subsectionName string, remoteType string, en } } else if remoteType == "terra" { rs.Terra = &TerraRemote{ - Endpoint: endpoint, - Auth: auth, - Mode: mode, + Endpoint: endpoint, + HubEndpoint: hubEndpoint, + Auth: auth, + Mode: mode, } } else if remoteType == "local" { rs.Local = &LocalRemote{ @@ -345,7 +349,7 @@ func parseAndAddRemote(cfg *Config, subsectionName string, remoteType string, en func addGenericRemote(cfg *Config, name Remote, opts map[string]string) { version, _ := strconv.Atoi(opts["preset-version"]) cfg.Remotes[name] = RemoteSelect{AccessMethod: opts["access-method"], Generic: &GenericRemote{ - Endpoint: opts["endpoint"], Provider: opts["provider"], Auth: opts["auth"], + Endpoint: opts["endpoint"], HubEndpoint: opts["hub-endpoint"], Provider: opts["provider"], Auth: opts["auth"], Credential: opts["credential"], Scope: opts["scope"], Storage: opts["storage"], Checkout: opts["checkout"], Preset: opts["preset"], PresetVersion: version, RegistryServiceID: opts["registry-service-id"], @@ -359,6 +363,13 @@ func withGenericEndpoint(remote RemoteSelect, endpoint string) RemoteSelect { return remote } +func withGenericHubEndpoint(remote RemoteSelect, endpoint string) RemoteSelect { + generic := *remote.Generic + generic.HubEndpoint = endpoint + remote.Generic = &generic + return remote +} + func configMapEntries(values map[string]string) []string { keys := make([]string, 0, len(values)) for key := range values { @@ -459,12 +470,18 @@ func loadGitConfigOverrides(cfg *Config) error { scalars[key] = last(values) } shared := cfg.Remotes[name] - localEndpoint := scalars["type"] != "" || scalars["endpoint"] != "" + localEndpoint := scalars["type"] != "" || scalars["endpoint"] != "" || scalars["hub-endpoint"] != "" if scalars["type"] == "ga4gh" { addGenericRemote(cfg, name, scalars) - } else if scalars["type"] == "" && scalars["endpoint"] != "" && shared.Generic != nil { - cfg.Remotes[name] = withGenericEndpoint(shared, scalars["endpoint"]) - } else if scalars["type"] != "" || scalars["endpoint"] != "" { + } else if scalars["type"] == "" && shared.Generic != nil && (scalars["endpoint"] != "" || scalars["hub-endpoint"] != "") { + cfg.Remotes[name] = shared + if scalars["endpoint"] != "" { + cfg.Remotes[name] = withGenericEndpoint(cfg.Remotes[name], scalars["endpoint"]) + } + if scalars["hub-endpoint"] != "" { + cfg.Remotes[name] = withGenericHubEndpoint(cfg.Remotes[name], scalars["hub-endpoint"]) + } + } else if scalars["type"] != "" || scalars["endpoint"] != "" || scalars["hub-endpoint"] != "" { parseAndAddRemote( cfg, remoteSubsectionPrefix+string(name), @@ -476,6 +493,7 @@ func loadGitConfigOverrides(cfg *Config) error { scalars["storage-prefix"], scalars["auth"], scalars["mode"], + scalars["hub-endpoint"], ) } remote := cfg.Remotes[name] @@ -534,16 +552,22 @@ func LoadConfig() (*Config, error) { } name := Remote(strings.TrimPrefix(subsection.Name, remoteSubsectionPrefix)) shared := cfg.Remotes[name] - localEndpoint := subsection.Option("type") != "" || subsection.Option("endpoint") != "" + localEndpoint := subsection.Option("type") != "" || subsection.Option("endpoint") != "" || subsection.Option("hub-endpoint") != "" if subsection.Option("type") == "ga4gh" { opts := make(map[string]string) - for _, key := range []string{"endpoint", "provider", "auth", "credential", "scope", "storage", "checkout", "preset", "preset-version", "registry-service-id", "access-method"} { + for _, key := range []string{"endpoint", "hub-endpoint", "provider", "auth", "credential", "scope", "storage", "checkout", "preset", "preset-version", "registry-service-id", "access-method"} { opts[key] = subsection.Option(key) } addGenericRemote(cfg, Remote(strings.TrimPrefix(subsection.Name, remoteSubsectionPrefix)), opts) - } else if subsection.Option("type") == "" && subsection.Option("endpoint") != "" && shared.Generic != nil { - cfg.Remotes[name] = withGenericEndpoint(shared, subsection.Option("endpoint")) - } else if subsection.Option("type") != "" || subsection.Option("endpoint") != "" { + } else if subsection.Option("type") == "" && shared.Generic != nil && (subsection.Option("endpoint") != "" || subsection.Option("hub-endpoint") != "") { + cfg.Remotes[name] = shared + if subsection.Option("endpoint") != "" { + cfg.Remotes[name] = withGenericEndpoint(cfg.Remotes[name], subsection.Option("endpoint")) + } + if subsection.Option("hub-endpoint") != "" { + cfg.Remotes[name] = withGenericHubEndpoint(cfg.Remotes[name], subsection.Option("hub-endpoint")) + } + } else if subsection.Option("type") != "" || subsection.Option("endpoint") != "" || subsection.Option("hub-endpoint") != "" { parseAndAddRemote( cfg, subsection.Name, @@ -555,6 +579,7 @@ func LoadConfig() (*Config, error) { subsection.Option("storage-prefix"), subsection.Option("auth"), subsection.Option("mode"), + subsection.Option("hub-endpoint"), ) } remote := cfg.Remotes[name] diff --git a/internal/config/config_test.go b/internal/config/config_test.go index b3226729..ab5e8fd0 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -562,3 +562,31 @@ func TestRemoveRemote_TerraCleansAuthAndMode(t *testing.T) { } } } + +func TestTerraHubEndpointRoundTripsWithoutChangingTdrEndpoint(t *testing.T) { + setupTestRepo(t) + remoteName := Remote("anvil") + _, err := UpdateRemote(remoteName, RemoteSelect{Terra: &TerraRemote{ + Endpoint: "https://data.terra.bio", + HubEndpoint: "https://drshub.dsde-prod.broadinstitute.org", + Mode: "read-only", + }}) + if err != nil { + t.Fatal(err) + } + + loaded, err := LoadConfig() + if err != nil { + t.Fatal(err) + } + remote := loaded.Remotes[remoteName].Terra + if remote == nil { + t.Fatal("Terra remote was not loaded") + } + if remote.Endpoint != "https://data.terra.bio" { + t.Errorf("TDR endpoint = %q", remote.Endpoint) + } + if remote.HubEndpoint != "https://drshub.dsde-prod.broadinstitute.org" { + t.Errorf("Hub endpoint = %q", remote.HubEndpoint) + } +} diff --git a/internal/config/remote.go b/internal/config/remote.go index b62b090d..9ed303ee 100644 --- a/internal/config/remote.go +++ b/internal/config/remote.go @@ -29,9 +29,9 @@ type RemoteSelect struct { // GenericRemote is the compositional configuration produced by the unified // remote-add command. Credential contains a source identifier, never a secret. type GenericRemote struct { - Endpoint, Provider, Auth, Credential, Scope, Storage, Checkout string - Preset, RegistryServiceID string - PresetVersion int + Endpoint, HubEndpoint, Provider, Auth, Credential, Scope, Storage, Checkout string + Preset, RegistryServiceID string + PresetVersion int } func (r GenericRemote) GetProjectId() string { _, p, _ := strings.Cut(r.Scope, "/"); return p } @@ -55,9 +55,10 @@ func (s Gen3Remote) GetBucketName() string { return s.Bucket } func (s Gen3Remote) GetStoragePrefix() string { return s.StoragePrefix } type TerraRemote struct { - Endpoint string `yaml:"endpoint"` - Auth string `yaml:"auth"` - Mode string `yaml:"mode"` + Endpoint string `yaml:"endpoint"` + HubEndpoint string `yaml:"hub_endpoint"` + Auth string `yaml:"auth"` + Mode string `yaml:"mode"` } func (t TerraRemote) GetProjectId() string { return "" } diff --git a/internal/remoteruntime/runtime.go b/internal/remoteruntime/runtime.go index 4714025d..76898179 100644 --- a/internal/remoteruntime/runtime.go +++ b/internal/remoteruntime/runtime.go @@ -23,6 +23,7 @@ type GitContext struct { Client *syclient.Client RemoteType config.RemoteType Endpoint string + HubEndpoint string Organization string ProjectId string BucketName string @@ -102,7 +103,7 @@ func New(cfg *config.Config, remote config.Remote, logger *slog.Logger) (ctx *Gi } switch x.Generic.Provider { case "terra": - return terraClient(config.TerraRemote{Endpoint: x.Generic.Endpoint, Auth: x.Generic.Auth, Mode: "read-only"}, logger) + return terraClient(config.TerraRemote{Endpoint: x.Generic.Endpoint, HubEndpoint: x.Generic.HubEndpoint, Auth: x.Generic.Auth, Mode: "read-only"}, logger) case "gen3": switch x.Generic.Auth { case "bearer", "provider-helper", "provider-helper:gen3-profile": @@ -133,6 +134,7 @@ func terraClient(remote config.TerraRemote, logger *slog.Logger) (*GitContext, e return &GitContext{ RemoteType: config.TerraServerType, Endpoint: remote.Endpoint, + HubEndpoint: remote.HubEndpoint, Logger: logger, Credential: &syconf.Credential{APIEndpoint: remote.Endpoint}, Capabilities: Capabilities{Resolve: true, Download: true, ReadOnly: true}, diff --git a/internal/remoteruntime/runtime_test.go b/internal/remoteruntime/runtime_test.go index c8f13dd5..cfdcaa1b 100644 --- a/internal/remoteruntime/runtime_test.go +++ b/internal/remoteruntime/runtime_test.go @@ -135,10 +135,31 @@ func TestNewTerraRemoteContext(t *testing.T) { } } +func TestNewTerraRemoteContextPreservesExplicitHubEndpoint(t *testing.T) { + setupTestRepo(t) + cfg, err := config.UpdateRemote(config.Remote("anvil"), config.RemoteSelect{ + Terra: &config.TerraRemote{ + Endpoint: "https://data.terra.bio", + HubEndpoint: "https://drshub.dsde-prod.broadinstitute.org", + Mode: "read-only", + }, + }) + if err != nil { + t.Fatal(err) + } + gitCtx, err := New(cfg, config.Remote("anvil"), drslog.GetLogger()) + if err != nil { + t.Fatal(err) + } + if gitCtx.Endpoint != "https://data.terra.bio" || gitCtx.HubEndpoint != "https://drshub.dsde-prod.broadinstitute.org" { + t.Fatalf("Terra endpoints = TDR:%q Hub:%q", gitCtx.Endpoint, gitCtx.HubEndpoint) + } +} + func TestNewGenericTerraUsesProviderAdapter(t *testing.T) { setupTestRepo(t) cfg := &config.Config{Remotes: map[config.Remote]config.RemoteSelect{ - "anvil": {Generic: &config.GenericRemote{Endpoint: "https://data.terra.bio", Provider: "terra", Auth: "google-adc"}}, + "anvil": {Generic: &config.GenericRemote{Endpoint: "https://data.terra.bio", HubEndpoint: "https://drshub.dsde-prod.broadinstitute.org", Provider: "terra", Auth: "google-adc"}}, }} gitCtx, err := New(cfg, "anvil", drslog.GetLogger()) if err != nil { @@ -147,6 +168,9 @@ func TestNewGenericTerraUsesProviderAdapter(t *testing.T) { if gitCtx.RemoteType != config.TerraServerType || !gitCtx.IsReadOnly() || !gitCtx.CanDownload() { t.Fatalf("generic Terra did not resolve through Terra adapter: %+v", gitCtx) } + if gitCtx.Endpoint != "https://data.terra.bio" || gitCtx.HubEndpoint != "https://drshub.dsde-prod.broadinstitute.org" { + t.Fatalf("generic Terra endpoints = TDR:%q Hub:%q", gitCtx.Endpoint, gitCtx.HubEndpoint) + } } func TestResolveGen3CredentialUsesSelectedProfile(t *testing.T) { diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index 61db9a89..b7855a53 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -2,6 +2,7 @@ package resolver import ( + "bytes" "context" "crypto/md5" "crypto/sha256" @@ -18,6 +19,7 @@ import ( "os" "path" "path/filepath" + "sort" "strconv" "strings" @@ -243,6 +245,11 @@ type AnVILResolver struct { client *http.Client } +type TerraHubResolver struct { + endpoint *url.URL + client *http.Client +} + func NewAnVIL(ctx context.Context, endpoint string) (*AnVILResolver, error) { if _, err := trustedEndpoint(endpoint); err != nil { return nil, err @@ -254,6 +261,43 @@ func NewAnVIL(ctx context.Context, endpoint string) (*AnVILResolver, error) { return NewAnVILWithClient(endpoint, &http.Client{Transport: &oauth2.Transport{Base: http.DefaultTransport, Source: creds.TokenSource}}) } +// NewTerraHub creates a resolver for Terra DRS Hub using Google ADC for its +// bearer token. Hub then forwards that token to the selected DRS provider. +func NewTerraHub(ctx context.Context, endpoint string) (*TerraHubResolver, error) { + if _, err := trustedTerraHubEndpoint(endpoint); err != nil { + return nil, err + } + creds, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/cloud-platform") + if err != nil { + return nil, fmt.Errorf("%w: run `gcloud auth application-default login`: %v", ErrCredentials, err) + } + return NewTerraHubWithClient(endpoint, &http.Client{Transport: &oauth2.Transport{Base: http.DefaultTransport, Source: creds.TokenSource}}) +} + +// NewTerraHubWithClient uses a caller-supplied client. Plain HTTP is accepted +// only for loopback endpoints so tests can exercise a local Hub instance. +func NewTerraHubWithClient(endpoint string, client *http.Client) (*TerraHubResolver, error) { + u, err := trustedTerraHubEndpoint(endpoint) + if err != nil { + u, err = trustedLoopbackEndpoint(endpoint) + } + if err != nil { + return nil, err + } + if client == nil { + return nil, fmt.Errorf("Terra DRS Hub HTTP client is required") + } + return &TerraHubResolver{endpoint: u, client: client}, nil +} + +func trustedTerraHubEndpoint(endpoint string) (*url.URL, error) { + u, err := url.Parse(strings.TrimSpace(endpoint)) + if err != nil || u.Host == "" || u.Scheme != "https" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return nil, fmt.Errorf("invalid Terra DRS Hub endpoint") + } + return u, nil +} + // NewAnVILWithClient uses a caller-supplied HTTP client for a trusted HTTPS // resolver endpoint. The caller is responsible for attaching authentication. func NewAnVILWithClient(endpoint string, client *http.Client) (*AnVILResolver, error) { @@ -275,6 +319,18 @@ func trustedEndpoint(endpoint string) (*url.URL, error) { return u, nil } +func trustedLoopbackEndpoint(endpoint string) (*url.URL, error) { + u, err := url.Parse(strings.TrimSpace(endpoint)) + if err != nil || u.Host == "" || u.Scheme != "http" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return nil, fmt.Errorf("invalid Terra DRS Hub endpoint") + } + host := u.Hostname() + if host != "localhost" && host != "127.0.0.1" && host != "::1" { + return nil, fmt.Errorf("invalid Terra DRS Hub endpoint") + } + return u, nil +} + func NormalizeDRSURI(raw string) (string, string, error) { trimmed := strings.TrimSpace(raw) // DRS compact identifiers use a colon between the authority and object ID @@ -376,3 +432,93 @@ func (r *AnVILResolver) getJSON(ctx context.Context, endpoint string, dst any) e } return nil } + +type terraHubRequest struct { + URL string `json:"url"` + Fields []string `json:"fields"` +} + +type terraHubResponse struct { + Name string `json:"name"` + FileName string `json:"fileName"` + Size int64 `json:"size"` + Hashes map[string]string `json:"hashes"` + AccessURL *struct { + URL string `json:"url"` + Headers map[string]string `json:"headers"` + } `json:"accessUrl"` +} + +// GetObject resolves an AnVIL DRS identifier through Terra DRS Hub. Hub's +// resolve response includes the ephemeral access URL, so the caller can +// download it without a second resolver request. +func (r *TerraHubResolver) GetObject(ctx context.Context, drsURI string) (*ResolvedObject, error) { + canonical, id, err := NormalizeDRSURI(drsURI) + if err != nil { + return nil, err + } + body, err := json.Marshal(terraHubRequest{ + URL: canonical, + Fields: []string{"size", "fileName", "hashes", "accessUrl"}, + }) + if err != nil { + return nil, err + } + u := *r.endpoint + u.Path = path.Join(u.Path, "api", "v4", "drs", "resolve") + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + resp, err := r.client.Do(req) + if err != nil { + return nil, fmt.Errorf("Terra DRS Hub unavailable: %w", err) + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusUnauthorized, http.StatusForbidden: + return nil, ErrUnauthorized + case http.StatusNotFound: + return nil, ErrNotFound + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) + return nil, fmt.Errorf("Terra DRS Hub returned HTTP %d", resp.StatusCode) + } + var metadata terraHubResponse + if err := json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&metadata); err != nil { + return nil, fmt.Errorf("decode Terra DRS Hub response: %w", err) + } + if metadata.Size < 0 { + return nil, fmt.Errorf("Terra DRS Hub returned an invalid object size") + } + checksums := make([]Checksum, 0, len(metadata.Hashes)) + for kind, checksum := range metadata.Hashes { + checksums = append(checksums, Checksum{Type: kind, Checksum: checksum}) + } + sort.Slice(checksums, func(i, j int) bool { return checksums[i].Type < checksums[j].Type }) + name := metadata.FileName + if name == "" { + name = metadata.Name + } + obj := &ResolvedObject{DRSURI: canonical, ID: id, Name: name, Size: metadata.Size, Checksums: checksums} + if metadata.AccessURL != nil && strings.TrimSpace(metadata.AccessURL.URL) != "" { + headerNames := make([]string, 0, len(metadata.AccessURL.Headers)) + for key := range metadata.AccessURL.Headers { + headerNames = append(headerNames, key) + } + sort.Strings(headerNames) + headers := make([]string, 0, len(headerNames)) + for _, key := range headerNames { + headers = append(headers, key+": "+metadata.AccessURL.Headers[key]) + } + obj.AccessMethods = []AccessMethod{{AccessURL: &ResolvedAccess{URL: metadata.AccessURL.URL, Headers: headers}}} + } + return obj, nil +} + +func (r *TerraHubResolver) GetAccess(context.Context, string, string) (*ResolvedAccess, error) { + return nil, fmt.Errorf("Terra DRS Hub resolves access URLs with object metadata") +} diff --git a/internal/resolver/terra_hub_test.go b/internal/resolver/terra_hub_test.go new file mode 100644 index 00000000..666385c3 --- /dev/null +++ b/internal/resolver/terra_hub_test.go @@ -0,0 +1,82 @@ +package resolver + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestTerraHubResolverContract(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/v4/drs/resolve" { + t.Errorf("request = %s %s, want POST /api/v4/drs/resolve", r.Method, r.URL.Path) + } + if got := r.Header.Get("Authorization"); got != "Bearer hub-test-token" { + t.Errorf("Authorization = %q, want test bearer", got) + } + var request terraHubRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Errorf("decode request: %v", err) + } + if request.URL != "drs://drs.anv0:v2_fixture" { + t.Errorf("request URL = %q", request.URL) + } + if strings.Join(request.Fields, ",") != "size,fileName,hashes,accessUrl" { + t.Errorf("request fields = %v", request.Fields) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"size":43,"fileName":"fixture.vcf.gz","hashes":{"md5":"abc","crc32c":"def"},"accessUrl":{"url":"https://storage.example/object","headers":{"X-Fixture":"fixture-token","Accept":"application/octet-stream"}}}`)) + })) + defer server.Close() + + client := server.Client() + baseTransport := client.Transport + client.Transport = roundTripperFunc(func(request *http.Request) (*http.Response, error) { + request.Header.Set("Authorization", "Bearer hub-test-token") + return baseTransport.RoundTrip(request) + }) + r, err := NewTerraHubWithClient(server.URL, client) + if err != nil { + t.Fatal(err) + } + object, err := r.GetObject(context.Background(), "drs://drs.anv0:v2_fixture") + if err != nil { + t.Fatal(err) + } + if object.ID != "v2_fixture" || object.DRSURI != "drs://drs.anv0:v2_fixture" || object.Name != "fixture.vcf.gz" || object.Size != 43 { + t.Fatalf("object metadata = %+v", object) + } + if len(object.Checksums) != 2 || object.Checksums[0] != (Checksum{Type: "crc32c", Checksum: "def"}) || object.Checksums[1] != (Checksum{Type: "md5", Checksum: "abc"}) { + t.Fatalf("object checksums = %+v", object.Checksums) + } + if len(object.AccessMethods) != 1 || object.AccessMethods[0].AccessURL == nil { + t.Fatalf("access methods = %+v", object.AccessMethods) + } + access := object.AccessMethods[0].AccessURL + if access.URL != "https://storage.example/object" || strings.Join(access.Headers, ",") != "Accept: application/octet-stream,X-Fixture: fixture-token" { + t.Fatalf("access URL = %+v", access) + } +} + +func TestTerraHubHTTPClientAllowsOnlyLoopbackHTTP(t *testing.T) { + client := &http.Client{} + for _, endpoint := range []string{"http://localhost:8080", "http://127.0.0.1:8080", "http://[::1]:8080"} { + if _, err := NewTerraHubWithClient(endpoint, client); err != nil { + t.Errorf("NewTerraHubWithClient(%q): %v", endpoint, err) + } + } + for _, endpoint := range []string{"http://hub.example", "https://hub.example/path?token=x", "https://user:pass@hub.example"} { + if _, err := NewTerraHubWithClient(endpoint, client); err == nil { + t.Errorf("NewTerraHubWithClient(%q) unexpectedly succeeded", endpoint) + } + } +} + +type roundTripperFunc func(*http.Request) (*http.Response, error) + +func (f roundTripperFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} diff --git a/tests/terra-drs-hub/README.md b/tests/terra-drs-hub/README.md index c6dee548..6ecfd591 100644 --- a/tests/terra-drs-hub/README.md +++ b/tests/terra-drs-hub/README.md @@ -1,12 +1,10 @@ -# DRS Hub to TDR integration +# Terra pull through DRS Hub and TDR -The `Resolve through DRS Hub and TDR` CI job runs the pinned public DRS Hub service and sends its real `POST /api/v4/drs/resolve` endpoint an AnVIL compact DRS ID. Hub expands `drs.anv0`, chooses its TDR provider, and makes the GA4GH DRS `OPTIONS`, object metadata, and access URL calls. Those HTTPS requests pass through a local TLS forwarder to the running TDR harness. The forwarder only translates HTTPS to the harness's local HTTP listener; it does not generate DRS responses. +The `Resolve through DRS Hub and TDR` CI job runs a single `git-drs pull` against an AnVIL pointer. The CLI sends the DRS ID to a locally running DRS Hub. Hub expands `drs.anv0`, selects its TDR provider, and calls TDR's DRS API. TDR resolves the object through its real DRS controller, `DrsService`, PostgreSQL DRS ID and snapshot records, and Firestore file record. The CLI downloads the returned bytes and hydrates the worktree file. The test checks the hydrated bytes, a Hub resolve call, and authenticated TDR metadata and access requests. -The TDR harness runs the actual DRS controller and `DrsService`. The snapshot, authorization, file record, and URL signing boundaries use test fixtures. The request uses the fixed test bearer token `tdr-ci-fixture`; authenticated metadata and access requests forward that token to TDR. Hub's DRS `OPTIONS` probe omits the bearer token, so the fixture rejects that probe and Hub exercises its configured-auth fallback before making the authenticated `GET` calls. The signed URL points to the harness's local fixture bytes, so the test needs no Google credentials or bucket access. +The external authorization boundary uses a fixed test bearer token (`tdr-ci-fixture`) instead of Google credentials or Sam. Hub's unauthenticated DRS `OPTIONS` probe is rejected by that boundary, exercising Hub's configured-auth fallback. The local TLS proxy forwards Hub's requests to TDR and records them; it does not fabricate DRS responses. TDR's real Google Cloud Storage client looks up the seeded bucket and signs the access URL with a generated test-only key. A local token endpoint supplies an access token, and the GCS-compatible emulator serves the object bytes. The CLI-side test bridge changes the signed URL authority to the emulator's local address. The emulator does not validate signed URL signatures or expiry. The runner asserts one GCS object download. -This check covers Hub's AnVIL routing and TDR DRS client behavior separately from the `git-drs pull` integration check. `git-drs` currently resolves directly against TDR, so its pull test does not pass through Hub. - -To run locally, check out the pinned TDR and DRS Hub revisions used in `.github/workflows/pr-checks.yaml`, then run: +Run locally using the pinned TDR and DRS Hub revisions in `.github/workflows/pr-checks.yaml`: ```sh tests/terra-drs-hub/run.sh /path/to/jade-data-repo /path/to/terra-drs-hub diff --git a/tests/terra-drs-hub/TerraDrsHubIntegrationTest.java b/tests/terra-drs-hub/TerraDrsHubIntegrationTest.java index 2c216ba3..c72b8d39 100644 --- a/tests/terra-drs-hub/TerraDrsHubIntegrationTest.java +++ b/tests/terra-drs-hub/TerraDrsHubIntegrationTest.java @@ -1,21 +1,10 @@ package bio.terra.drshub.controllers; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; -import java.util.List; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.web.server.LocalServerPort; import org.springframework.test.context.ActiveProfiles; @@ -24,93 +13,31 @@ @ActiveProfiles({"test", "human-readable-logging", "hub-ci"}) @Tag("Unit") class TerraDrsHubIntegrationTest { - private static final String OBJECT_ID = "v2_4f770147-e372-339b-b9fa-0a7a83cf30cf"; + private static final Duration STOP_TIMEOUT = Duration.ofMinutes(20); @LocalServerPort private int port; - @Autowired private ObjectMapper objectMapper; @Test - void resolvesAnvilCompactIdThroughHubAndTheLiveTdrDrsApi() throws Exception { - Path proxyLog = Path.of(requiredEnv("GIT_DRS_TDR_PROXY_LOG")); - Files.deleteIfExists(proxyLog); - - String requestBody = - objectMapper.writeValueAsString( - java.util.Map.of( - "url", - "drs://drs.anv0:" + OBJECT_ID, - "fields", - List.of("size", "fileName", "hashes", "accessUrl"))); - HttpRequest request = - HttpRequest.newBuilder() - .uri(URI.create("http://127.0.0.1:" + port + "/api/v4/drs/resolve")) - .timeout(Duration.ofSeconds(30)) - .header("Authorization", "Bearer tdr-ci-fixture") - .header("Content-Type", "application/json") - .POST(HttpRequest.BodyPublishers.ofString(requestBody)) - .build(); - - HttpResponse response = - HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - assertEquals(200, response.statusCode(), response.body()); - - JsonNode metadata = objectMapper.readTree(response.body()); - assertEquals(43, metadata.path("size").asLong()); - assertEquals("1614321.merge_output.gvcf.gz", metadata.path("fileName").asText()); - assertTrue(metadata.path("hashes").isObject(), response.body()); - assertEquals("d0147894", metadata.path("hashes").path("crc32c").asText()); - assertEquals("e3fbcfb74b00bf57af80060c0ac10d03", metadata.path("hashes").path("md5").asText()); - assertEquals( - "http://127.0.0.1:" + requiredEnv("GIT_DRS_TDR_HTTP_PORT") + "/fixture-data", - metadata.path("accessUrl").path("url").asText()); - - List proxiedRequests = - Files.readAllLines(proxyLog).stream().map(this::readJson).toList(); - assertTrue( - proxiedRequests.stream() - .anyMatch( - entry -> - entry.path("method").asText().equals("OPTIONS") - && entry.path("path").asText().endsWith("/objects/" + OBJECT_ID)), - proxiedRequests.toString()); - assertTrue( - proxiedRequests.stream() - .anyMatch( - entry -> - entry.path("method").asText().equals("GET") - && entry.path("path").asText().endsWith("/objects/" + OBJECT_ID)), - proxiedRequests.toString()); - assertTrue( - proxiedRequests.stream() - .anyMatch( - entry -> - entry.path("method").asText().equals("GET") - && entry - .path("path") - .asText() - .contains("/objects/" + OBJECT_ID + "/access/")), - proxiedRequests.toString()); - assertTrue( - proxiedRequests.stream() - .filter(entry -> entry.path("method").asText().equals("GET")) - .allMatch( - entry -> entry.path("authorization").asText().equals("Bearer tdr-ci-fixture")), - proxiedRequests.toString()); - } - - private JsonNode readJson(String line) { - try { - return objectMapper.readTree(line); - } catch (Exception e) { - throw new IllegalStateException("Invalid TDR proxy log entry: " + line, e); + void serveHubUntilGitDrsPullFinishes() throws Exception { + Path portFile = requiredPath("GIT_DRS_HUB_PORT_FILE"); + Path stopFile = requiredPath("GIT_DRS_HUB_STOP_FILE"); + Files.deleteIfExists(stopFile); + Files.writeString(portFile, Integer.toString(port)); + + long deadline = System.nanoTime() + STOP_TIMEOUT.toNanos(); + while (!Files.exists(stopFile) && System.nanoTime() < deadline) { + Thread.sleep(200); + } + if (!Files.exists(stopFile)) { + throw new IllegalStateException("Timed out waiting for git-drs pull to finish"); } } - private static String requiredEnv(String name) { - String value = System.getenv(name); + private static Path requiredPath(String envName) { + String value = System.getenv(envName); if (value == null || value.isBlank()) { - throw new IllegalStateException(name + " must be set by the CI driver"); + throw new IllegalStateException(envName + " must name a coordination file"); } - return value; + return Path.of(value); } } diff --git a/tests/terra-drs-hub/run.sh b/tests/terra-drs-hub/run.sh index 9258c5c9..f53598fe 100755 --- a/tests/terra-drs-hub/run.sh +++ b/tests/terra-drs-hub/run.sh @@ -9,9 +9,13 @@ fi git_drs_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) tdr_root=$(cd "$1" && pwd) hub_root=$(cd "$2" && pwd) +source "$git_drs_root/tests/terra-tdr/firestore-emulator.sh" +source "$git_drs_root/tests/terra-tdr/storage-emulator.sh" coord_dir=$(mktemp -d) port_file="$coord_dir/tdr-port" stop_file="$coord_dir/tdr-stop" +hub_port_file="$coord_dir/hub-port" +hub_stop_file="$coord_dir/hub-stop" tdr_log="$coord_dir/tdr.log" hub_log="$coord_dir/hub.log" proxy_log="$coord_dir/proxy.log" @@ -60,23 +64,36 @@ if ! JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:-} -Djavax.net.ssl.trustStore=$trust fi tls_port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') +FIRESTORE_CONTAINER= +start_firestore_emulator +STORAGE_CONTAINER= +start_storage_emulator "$coord_dir/gcs-seed" ( cd "$tdr_root" GIT_DRS_TDR_PORT_FILE="$port_file" GIT_DRS_TDR_STOP_FILE="$stop_file" \ + FIRESTORE_EMULATOR_HOST="$FIRESTORE_EMULATOR_HOST" \ + GIT_DRS_TDR_GCS_ENDPOINT="$STORAGE_EMULATOR_HTTP_ENDPOINT" \ ./gradlew testUnit --tests bio.terra.service.filedata.TerraDrsHarnessTest \ - --no-daemon --console=plain + --no-daemon --console=plain --rerun-tasks ) >"$tdr_log" 2>&1 & tdr_pid=$! proxy_pid= +hub_pid= stop_services() { + touch "$hub_stop_file" touch "$stop_file" + if [[ -n "$hub_pid" ]]; then + wait "$hub_pid" 2>/dev/null || true + fi if [[ -n "$proxy_pid" ]]; then kill "$proxy_pid" 2>/dev/null || true wait "$proxy_pid" 2>/dev/null || true fi wait "$tdr_pid" 2>/dev/null || true + stop_firestore_emulator + stop_storage_emulator } trap stop_services EXIT @@ -129,13 +146,41 @@ PY sleep 1 done -cd "$hub_root" -if ! GIT_DRS_TDR_TLS_PORT="$tls_port" \ +( + cd "$hub_root" + GIT_DRS_HUB_PORT_FILE="$hub_port_file" GIT_DRS_HUB_STOP_FILE="$hub_stop_file" \ + GIT_DRS_TDR_TLS_PORT="$tls_port" \ GIT_DRS_TDR_HTTP_PORT="$tdr_port" \ GIT_DRS_TDR_PROXY_LOG="$proxy_log" \ JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:-} -Djavax.net.ssl.trustStore=$truststore -Djavax.net.ssl.trustStorePassword=changeit" \ ./gradlew :service:test --tests bio.terra.drshub.controllers.TerraDrsHubIntegrationTest \ - --no-daemon --console=plain -x :service:runMinnieKenny >"$hub_log" 2>&1; then + --no-daemon --console=plain --rerun-tasks -x :service:runMinnieKenny +) >"$hub_log" 2>&1 & +hub_pid=$! + +for ((seconds = 0; seconds < 900; seconds++)); do + if [[ -s "$hub_port_file" ]]; then + break + fi + if ! kill -0 "$hub_pid" 2>/dev/null; then + cat "$hub_log" >&2 + exit 1 + fi + sleep 1 +done +if [[ ! -s "$hub_port_file" ]]; then + echo "DRS Hub did not start within 15 minutes" >&2 + cat "$hub_log" >&2 + exit 1 +fi + +hub_port=$(cat "$hub_port_file") +cd "$git_drs_root" +if ! GIT_DRS_HUB_HTTP_ENDPOINT="http://127.0.0.1:$hub_port" \ + GIT_DRS_TDR_PROXY_LOG="$proxy_log" \ + GIT_DRS_TDR_GCS_ENDPOINT="$STORAGE_EMULATOR_HTTP_ENDPOINT" \ + go test -race -tags=integration -count=1 ./cmd/pull \ + -run '^TestIntegrationPullThroughTerraHubAndTDR$' -v; then cat "$hub_log" >&2 cat "$tdr_log" >&2 cat "$proxy_log" >&2 2>/dev/null || true @@ -143,6 +188,20 @@ if ! GIT_DRS_TDR_TLS_PORT="$tls_port" \ exit 1 fi +downloads=$(storage_object_download_count) +if [[ "$downloads" != 1 ]]; then + echo "GCS emulator served $downloads object downloads, want exactly one" >&2 + docker logs "$STORAGE_CONTAINER" >&2 || true + exit 1 +fi + +touch "$hub_stop_file" +if ! wait "$hub_pid"; then + cat "$hub_log" >&2 + exit 1 +fi +hub_pid= + touch "$stop_file" if ! wait "$tdr_pid"; then cat "$tdr_log" >&2 @@ -151,6 +210,9 @@ fi kill "$proxy_pid" 2>/dev/null || true wait "$proxy_pid" 2>/dev/null || true proxy_pid= +tdr_pid= +stop_firestore_emulator +stop_storage_emulator trap - EXIT cat "$hub_log" diff --git a/tests/terra-tdr/README.md b/tests/terra-tdr/README.md index 7d6d9fbf..eb443d2d 100644 --- a/tests/terra-tdr/README.md +++ b/tests/terra-tdr/README.md @@ -1,11 +1,15 @@ -# TDR controller integration +# TDR pull integration -`run.sh` copies `TerraDrsHarnessTest.java` into a checkout of the pinned TDR source, starts its real DRS controller, generated API routes, and `DrsService`, then runs the git-drs pull integration test against that server. The test provides fixture snapshot and file metadata and mocks authorization and Google URL signing. It checks that git-drs fetches the fixture bytes from an empty cache and verifies the TDR-produced CRC32C and MD5 checksums. It needs no Terra credentials or cloud storage. +`run.sh` starts TDR's real DRS controller, generated API routes, and `DrsService`, then runs the `git-drs pull` integration test against that server. TDR stores the compact DRS ID mapping and the snapshot, source, and project records in an ephemeral PostgreSQL Testcontainer. The real `SnapshotService` and `SnapshotDao` load that metadata; the DAO's dataset materialization and empty table/relationship collaborators remain fixture-backed. TDR reads the seeded file record through its real `FileService` and `FireStoreDao` against the Firestore emulator. A GCS-compatible emulator serves the fixture object over HTTP. -Run locally with Java 17 and Go installed: +Authorization remains fixture-backed. TDR uses the Google Cloud Storage Java client to look up the bucket and create a V4 signed URL with a generated test-only service-account key. A local token endpoint supplies its access token; no Google credentials are used. The test proxy changes only the signed URL's authority to point at the local GCS emulator. The emulator serves the bytes but does not validate the signature or expiry. + +The pull starts with an empty cache, checks the hydrated bytes and TDR-provided checksums, and the runner asserts that the emulator served exactly one object download. + +Run locally with Java 17, Go, and Docker installed: ```bash tests/terra-tdr/run.sh /path/to/jade-data-repo ``` -The CI job pins the TDR source commit in `.github/workflows/pr-checks.yaml`. This test covers TDR's DRS object and access URL construction and the git-drs pull path. It does not exercise TDR's database, Sam authorization, or cloud storage integrations. +The CI workflow pins the TDR source revision. The test covers TDR's controller and `DrsService`, PostgreSQL DRS and snapshot lookup, Firestore file lookup, GCS client calls, the returned signed URL shape, and the CLI download path. It does not exercise Sam authorization or TDR's full `DatasetDao` materialization. The local token endpoint also does not validate Google-issued credentials. diff --git a/tests/terra-tdr/TerraDrsHarnessTest.java b/tests/terra-tdr/TerraDrsHarnessTest.java index 190c552f..9c97b433 100644 --- a/tests/terra-tdr/TerraDrsHarnessTest.java +++ b/tests/terra-tdr/TerraDrsHarnessTest.java @@ -1,19 +1,16 @@ package bio.terra.service.filedata; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import bio.terra.app.configuration.ApplicationConfiguration; +import bio.terra.app.configuration.DataRepoJdbcConfiguration; import bio.terra.app.configuration.DrsConfiguration; import bio.terra.app.configuration.EcmConfiguration; import bio.terra.app.controller.DataRepositoryServiceApiController; import bio.terra.app.logging.PerformanceLogger; import bio.terra.app.usermetrics.UserLoggingMetrics; +import bio.terra.common.EmbeddedDatabaseTest; import bio.terra.common.category.Unit; import bio.terra.common.exception.UnauthorizedException; import bio.terra.common.fixtures.AuthenticationFixtures; @@ -22,53 +19,86 @@ import bio.terra.model.BillingProfileModel; import bio.terra.model.CloudPlatform; import bio.terra.service.auth.iam.IamService; +import bio.terra.service.auth.ras.EcmService; +import bio.terra.service.configuration.ConfigEnum; +import bio.terra.service.configuration.ConfigurationService; import bio.terra.service.dataset.Dataset; +import bio.terra.service.dataset.DatasetDao; +import bio.terra.service.dataset.DatasetService; import bio.terra.service.dataset.DatasetSummary; +import bio.terra.service.duos.DuosClient; +import bio.terra.service.duos.DuosDao; +import bio.terra.service.filedata.google.firestore.EncodeFixture; +import bio.terra.service.filedata.google.firestore.FireStoreDao; +import bio.terra.service.filedata.google.firestore.FireStoreDependencyDao; +import bio.terra.service.filedata.google.firestore.FireStoreDirectoryEntry; +import bio.terra.service.filedata.google.firestore.FireStoreFile; import bio.terra.service.filedata.google.gcs.GcsProjectFactory; import bio.terra.service.job.JobService; +import bio.terra.service.journal.JournalService; +import bio.terra.service.load.LoadService; +import bio.terra.service.profile.ProfileService; +import bio.terra.service.rawls.RawlsService; +import bio.terra.service.resourcemanagement.MetadataDataAccessUtils; import bio.terra.service.resourcemanagement.ResourceService; import bio.terra.service.resourcemanagement.google.GoogleProjectResource; import bio.terra.service.snapshot.Snapshot; -import bio.terra.service.snapshot.SnapshotProject; +import bio.terra.service.snapshot.SnapshotDao; +import bio.terra.service.snapshot.SnapshotMapTableDao; +import bio.terra.service.snapshot.SnapshotRelationshipDao; import bio.terra.service.snapshot.SnapshotService; import bio.terra.service.snapshot.SnapshotSource; -import com.google.cloud.storage.BlobInfo; -import com.google.cloud.storage.Bucket; +import bio.terra.service.snapshot.SnapshotTableDao; +import bio.terra.service.snapshotbuilder.SnapshotBuilderSettingsDao; +import bio.terra.service.snapshotbuilder.SnapshotRequestDao; +import bio.terra.service.tabulardata.google.bigquery.BigQuerySnapshotPdao; +import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.storage.Storage; -import com.google.cloud.storage.Storage.BucketGetOption; +import com.google.cloud.storage.StorageOptions; +import com.sun.net.httpserver.HttpServer; import jakarta.servlet.http.HttpServletRequest; import java.io.IOException; -import java.net.URL; +import java.net.InetSocketAddress; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; +import java.security.KeyPairGenerator; import java.time.Duration; +import java.util.Base64; +import java.util.List; import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import javax.sql.DataSource; +import liquibase.Contexts; +import liquibase.Liquibase; +import liquibase.database.jvm.JdbcConnection; +import liquibase.resource.ClassLoaderResourceAccessor; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.Import; import org.springframework.core.task.AsyncTaskExecutor; import org.springframework.core.task.SimpleAsyncTaskExecutor; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.bean.override.mockito.MockitoBean; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RestController; /** * A real HTTP harness for git-drs Terra DRS integration tests. The TDR API controller, generated - * API mapping, and DrsService are real. Snapshot lookup, authorization, file metadata, URL signing, - * and authentication are fixture-backed. The test stays alive until the CI driver creates - * GIT_DRS_TDR_STOP_FILE. + * API mapping, DrsService, SnapshotService, SnapshotDao, FireStoreDao, and the GCS client are real. + * The test stays alive until the CI driver creates GIT_DRS_TDR_STOP_FILE. */ @SpringBootTest( classes = TerraDrsHarnessTest.HarnessApplication.class, @@ -76,24 +106,45 @@ properties = "spring.cloud.gcp.firestore.enabled=false") @ActiveProfiles({"google", "unittest"}) @Tag(Unit.TAG) +@EmbeddedDatabaseTest class TerraDrsHarnessTest { private static final String OBJECT_ID = "v2_4f770147-e372-339b-b9fa-0a7a83cf30cf"; private static final UUID SNAPSHOT_ID = UUID.fromString("11111111-2222-4333-8444-555555555555"); private static final UUID FILE_ID = UUID.fromString("4f770147-e372-339b-b9fa-0a7a83cf30cf"); + private static final UUID BILLING_PROFILE_ID = + UUID.fromString("aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"); + private static final UUID DATASET_ID = UUID.fromString("bbbbbbbb-cccc-4ddd-8eee-ffffffffffff"); + private static final UUID SNAPSHOT_PROJECT_RESOURCE_ID = + UUID.fromString("cccccccc-dddd-4eee-8fff-000000000001"); + private static final UUID DATASET_PROJECT_RESOURCE_ID = + UUID.fromString("cccccccc-dddd-4eee-8fff-000000000002"); private static final byte[] FIXTURE_BYTES = "git-drs Terra DRS HTTP integration fixture\n" .getBytes(java.nio.charset.StandardCharsets.UTF_8); private static final Duration STOP_TIMEOUT = Duration.ofMinutes(20); - private static final AtomicInteger fixtureDataRequests = new AtomicInteger(); - @LocalServerPort private int port; + private HttpServer tokenServer; - @MockitoBean private ApplicationConfiguration applicationConfiguration; - @MockitoBean private SnapshotService snapshotService; - @MockitoBean private FileService fileService; - @MockitoBean private DrsDao drsDao; @MockitoBean private IamService iamService; @MockitoBean private ResourceService resourceService; + @MockitoBean private ConfigurationService configurationService; + @MockitoBean private DatasetService datasetService; + @MockitoBean private LoadService loadService; + @MockitoBean private ProfileService profileService; + @MockitoBean private DatasetDao datasetDao; + @MockitoBean private JournalService journalService; + @MockitoBean private DuosDao duosDao; + @MockitoBean private FireStoreDependencyDao dependencyDao; + @MockitoBean private BigQuerySnapshotPdao bigQuerySnapshotPdao; + @MockitoBean private SnapshotRequestDao snapshotRequestDao; + @MockitoBean private SnapshotTableDao snapshotTableDao; + @MockitoBean private SnapshotRelationshipDao snapshotRelationshipDao; + @MockitoBean private MetadataDataAccessUtils metadataDataAccessUtils; + @MockitoBean private EcmService ecmService; + @MockitoBean private RawlsService rawlsService; + @MockitoBean private DuosClient duosClient; + @MockitoBean private SnapshotBuilderSettingsDao snapshotBuilderSettingsDao; + @MockitoBean private DataRepoJdbcConfiguration dataRepoJdbcConfiguration; @MockitoBean private DrsConfiguration drsConfiguration; @MockitoBean private JobService jobService; @MockitoBean private PerformanceLogger performanceLogger; @@ -103,6 +154,10 @@ class TerraDrsHarnessTest { @MockitoBean private UserLoggingMetrics userLoggingMetrics; @MockitoBean private AuthenticatedUserRequestFactory authenticatedUserRequestFactory; @MockitoBean private BearerTokenFactory bearerTokenFactory; + @Autowired private DrsDao drsDao; + @Autowired private DrsIdService drsIdService; + @Autowired private FireStoreDao fireStoreDao; + @Autowired private JdbcTemplate jdbcTemplate; @BeforeEach void configureFixtureAndSignalReady() throws Exception { @@ -113,44 +168,165 @@ void configureFixtureAndSignalReady() throws Exception { createParentDirectories(portFile); createParentDirectories(stopFile); - fixtureDataRequests.set(0); var testUser = AuthenticationFixtures.randomUserRequest(); when(authenticatedUserRequestFactory.from(any())) .thenAnswer( invocation -> { HttpServletRequest request = invocation.getArgument(0); - if ("/fixture-data".equals(request.getRequestURI())) { - return testUser; - } if (!"Bearer tdr-ci-fixture".equals(request.getHeader("Authorization"))) { throw new UnauthorizedException("Invalid fixture bearer token"); } return testUser; }); - when(applicationConfiguration.getDnsName()).thenReturn("drs.anv0"); when(drsConfiguration.maxDrsLookups()).thenReturn(10); when(jobService.getActivePodCount()).thenReturn(1); - when(snapshotService.retrieve(SNAPSHOT_ID)).thenReturn(fixtureSnapshot()); - when(snapshotService.retrieveSnapshotProject(SNAPSHOT_ID)).thenReturn(new SnapshotProject()); - when(drsDao.retrieveReferencedSnapshotIds(any())).thenReturn(java.util.List.of(SNAPSHOT_ID)); - when(fileService.lookupSnapshotFSItem(any(), any(), anyInt())).thenReturn(fixtureFile()); + when(datasetDao.retrieve(DATASET_ID)) + .thenReturn(fixtureSnapshot().getFirstSnapshotSource().getDataset()); + when(snapshotTableDao.retrieveTables(SNAPSHOT_ID)).thenReturn(java.util.List.of()); + when(resourceService.getProjectResource(SNAPSHOT_PROJECT_RESOURCE_ID)) + .thenReturn( + new GoogleProjectResource() + .id(SNAPSHOT_PROJECT_RESOURCE_ID) + .googleProjectId("fixture-snapshot-project")); + when(dataRepoJdbcConfiguration.getDataSource()) + .thenReturn(org.mockito.Mockito.mock(org.apache.commons.dbcp2.PoolingDataSource.class)); + when(configurationService.getParameterValue(any())) + .thenAnswer( + invocation -> + switch (invocation.getArgument(0, ConfigEnum.class)) { + case FIRESTORE_RETRIES -> 1; + default -> 100; + }); + migrateAndSeedDrsRecord(); + seedFirestoreFile(); - Storage storage = mock(Storage.class); - Bucket bucket = mock(Bucket.class); - when(bucket.getLocation()).thenReturn("us-central1"); - when(storage.get(eq("fixture-bucket"), any(BucketGetOption[].class))).thenReturn(bucket); - doReturn(new URL("http://127.0.0.1:" + port + "/fixture-data")) - .when(storage) - .signUrl( - any(BlobInfo.class), - anyLong(), - any(TimeUnit.class), - any(Storage.SignUrlOption[].class)); + tokenServer = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + tokenServer.createContext( + "/token", + exchange -> { + byte[] response = + "{\"access_token\":\"tdr-ci-token\",\"expires_in\":3600,\"token_type\":\"Bearer\"}" + .getBytes(java.nio.charset.StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, response.length); + exchange.getResponseBody().write(response); + exchange.close(); + }); + tokenServer.start(); + var keyPairGenerator = KeyPairGenerator.getInstance("RSA"); + keyPairGenerator.initialize(2048); + String privateKeyPem = + "-----BEGIN PRIVATE KEY-----\n" + + Base64.getMimeEncoder(64, new byte[] {'\n'}) + .encodeToString(keyPairGenerator.generateKeyPair().getPrivate().getEncoded()) + + "\n-----END PRIVATE KEY-----\n"; + var storageCredentials = + ServiceAccountCredentials.fromPkcs8( + "git-drs-ci@fixture.invalid", + "git-drs-ci-key", + privateKeyPem, + null, + List.of("https://www.googleapis.com/auth/devstorage.read_only"), + null, + URI.create("http://127.0.0.1:" + tokenServer.getAddress().getPort() + "/token")); + String storageEndpoint = System.getenv("GIT_DRS_TDR_GCS_ENDPOINT"); + if (storageEndpoint == null || storageEndpoint.isBlank()) { + throw new IllegalStateException("GIT_DRS_TDR_GCS_ENDPOINT must name the local GCS emulator"); + } + Storage storage = + StorageOptions.newBuilder() + .setProjectId("fixture") + .setHost(storageEndpoint) + .setCredentials(storageCredentials) + .build() + .getService(); when(gcsProjectFactory.getStorage(any())).thenReturn(storage); Files.writeString(portFile, Integer.toString(port)); } + @org.junit.jupiter.api.AfterEach + void stopFixtureTokenServer() { + if (tokenServer != null) { + tokenServer.stop(0); + } + } + + private void seedFirestoreFile() throws Exception { + Snapshot snapshot = fixtureSnapshot(); + Dataset dataset = snapshot.getFirstSnapshotSource().getDataset(); + fireStoreDao.upsertFileMetadata( + dataset, + new FireStoreFile() + .fileId(FILE_ID.toString()) + .fileCreatedDate("2026-10-07T00:00:00Z") + .gspath("gs://fixture-bucket/1614321.merge_output.gvcf.gz") + .checksumCrc32c(fixtureFile().getChecksumCrc32c()) + .checksumMd5(fixtureFile().getChecksumMd5()) + .size((long) FIXTURE_BYTES.length) + .mimeType("application/gzip") + .description("CI file fixture") + .bucketResourceId("fixture-bucket-resource") + .loadTag("git-drs-ci")); + fireStoreDao.createDirectoryEntry( + dataset, + new FireStoreDirectoryEntry() + .fileId(FILE_ID.toString()) + .isFileRef(true) + .path("/") + .name("1614321.merge_output.gvcf.gz") + .datasetId(dataset.getId().toString()) + .fileCreatedDate("2026-10-07T00:00:00Z") + .loadTag("git-drs-ci")); + fireStoreDao.addFilesToSnapshot(dataset, snapshot, java.util.List.of(FILE_ID.toString())); + } + + private void migrateAndSeedDrsRecord() throws Exception { + jdbcTemplate.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto"); + try (var connection = jdbcTemplate.getDataSource().getConnection()) { + new Liquibase( + "db/changelog.xml", new ClassLoaderResourceAccessor(), new JdbcConnection(connection)) + .update(new Contexts()); + } + jdbcTemplate.update( + "INSERT INTO billing_profile (id, name, billing_account_id) VALUES (?, ?, ?)", + BILLING_PROFILE_ID, + "git-drs-ci-profile", + "fixture-billing-account"); + jdbcTemplate.update( + "INSERT INTO project_resource (id, google_project_id, google_project_number, profile_id) VALUES (?, ?, ?, ?)", + SNAPSHOT_PROJECT_RESOURCE_ID, + "fixture-snapshot-project", + "100000000001", + BILLING_PROFILE_ID); + jdbcTemplate.update( + "INSERT INTO project_resource (id, google_project_id, google_project_number, profile_id) VALUES (?, ?, ?, ?)", + DATASET_PROJECT_RESOURCE_ID, + "fixture-dataset-project", + "100000000002", + BILLING_PROFILE_ID); + jdbcTemplate.update( + "INSERT INTO dataset (id, name, default_profile_id, project_resource_id, sharedlock, tags) VALUES (?, ?, ?, ?, '{}', ARRAY[]::TEXT[])", + DATASET_ID, + "git-drs-ci-fixture", + BILLING_PROFILE_ID, + DATASET_PROJECT_RESOURCE_ID); + jdbcTemplate.update( + "INSERT INTO snapshot (id, name, profile_id, project_resource_id) VALUES (?, ?, ?, ?)", + SNAPSHOT_ID, + "git-drs-ci-fixture", + BILLING_PROFILE_ID, + SNAPSHOT_PROJECT_RESOURCE_ID); + jdbcTemplate.update( + "INSERT INTO snapshot_source (snapshot_id, dataset_id) VALUES (?, ?)", + SNAPSHOT_ID, + DATASET_ID); + DrsId objectId = drsIdService.fromObjectId(OBJECT_ID); + if (drsDao.recordDrsIdToSnapshot(SNAPSHOT_ID, java.util.List.of(objectId)) != 1) { + throw new IllegalStateException("TDR DrsDao did not insert the fixture DRS mapping"); + } + } + @Test void serveTdrApiUntilCiDriverStops() throws Exception { Path stopFile = requiredPath("GIT_DRS_TDR_STOP_FILE"); @@ -165,8 +341,7 @@ void serveTdrApiUntilCiDriverStops() throws Exception { } private static Snapshot fixtureSnapshot() { - BillingProfileModel billingProfile = - new BillingProfileModel().id(UUID.fromString("aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee")); + BillingProfileModel billingProfile = new BillingProfileModel().id(BILLING_PROFILE_ID); return new Snapshot() .id(SNAPSHOT_ID) .profileId(billingProfile.getId()) @@ -230,32 +405,54 @@ private static void createParentDirectories(Path path) throws IOException { @SpringBootConfiguration @EnableAutoConfiguration(exclude = DataSourceAutoConfiguration.class) + @ComponentScan( + basePackages = "bio.terra.service.filedata.google.firestore", + excludeFilters = + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = EncodeFixture.class)) @Import({ DataRepositoryServiceApiController.class, - FixtureDataController.class, DrsIdService.class, - DrsService.class + DrsService.class, + FileService.class, + SnapshotDao.class, + SnapshotService.class, + SnapshotMapTableDao.class }) static class HarnessApplication { - @org.springframework.context.annotation.Bean(name = "drsResolutionThreadpool") - AsyncTaskExecutor drsResolutionThreadpool() { - return new SimpleAsyncTaskExecutor("drs-resolution-test-"); + @org.springframework.context.annotation.Bean + ApplicationConfiguration applicationConfiguration() { + ApplicationConfiguration configuration = new ApplicationConfiguration(); + configuration.setDnsName("drs.anv0"); + configuration.setFirestoreFutureTimeoutSeconds(30); + return configuration; } - } - @RestController - static class FixtureDataController { - @GetMapping("/fixture-data") - ResponseEntity fixtureData() { - fixtureDataRequests.incrementAndGet(); - return ResponseEntity.ok() - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .body(FIXTURE_BYTES); + @org.springframework.context.annotation.Bean( + name = "performanceThreadpool", + destroyMethod = "shutdownNow") + ExecutorService performanceThreadpool() { + return Executors.newSingleThreadExecutor(); } - @GetMapping("/fixture-data-requests") - int fixtureDataRequestCount() { - return fixtureDataRequests.get(); + @org.springframework.context.annotation.Bean + NamedParameterJdbcTemplate namedParameterJdbcTemplate( + @Qualifier("embeddedDataSource") DataSource dataSource) { + return new NamedParameterJdbcTemplate(dataSource); + } + + @org.springframework.context.annotation.Bean + JdbcTemplate jdbcTemplate(@Qualifier("embeddedDataSource") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @org.springframework.context.annotation.Bean + DrsDao drsDao(NamedParameterJdbcTemplate jdbcTemplate, DrsIdService drsIdService) { + return new DrsDao(jdbcTemplate, drsIdService); + } + + @org.springframework.context.annotation.Bean(name = "drsResolutionThreadpool") + AsyncTaskExecutor drsResolutionThreadpool() { + return new SimpleAsyncTaskExecutor("drs-resolution-test-"); } } } diff --git a/tests/terra-tdr/firestore-emulator.sh b/tests/terra-tdr/firestore-emulator.sh new file mode 100644 index 00000000..198024be --- /dev/null +++ b/tests/terra-tdr/firestore-emulator.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +start_firestore_emulator() { + local requested_port + requested_port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') + FIRESTORE_CONTAINER="git-drs-firestore-${PPID}-$$" + FIRESTORE_EMULATOR_HOST="127.0.0.1:$requested_port" + export FIRESTORE_EMULATOR_HOST + + docker run --detach --rm \ + --name "$FIRESTORE_CONTAINER" \ + --publish "127.0.0.1:$requested_port:8080" \ + --env CLOUDSDK_CORE_PROJECT=git-drs-ci-fixture \ + --entrypoint gcloud \ + gcr.io/google.com/cloudsdktool/google-cloud-cli@sha256:be2f0e582a94c9e06e7a384fe2b3b39d590eeb8da65a110eda991598c100fe80 \ + emulators firestore start --host-port=0.0.0.0:8080 >/dev/null + + for ((attempt = 0; attempt < 60; attempt++)); do + if python3 - "$requested_port" <<'PY' +import socket +import sys + +try: + with socket.create_connection(("127.0.0.1", int(sys.argv[1])), timeout=1): + pass +except OSError: + raise SystemExit(1) +PY + then + return 0 + fi + local container_running + container_running=$(docker inspect --format '{{.State.Running}}' "$FIRESTORE_CONTAINER" 2>/dev/null || true) + if [[ "$container_running" != true ]]; then + docker logs "$FIRESTORE_CONTAINER" >&2 || true + stop_firestore_emulator + return 1 + fi + sleep 1 + done + + echo "Firestore emulator did not start on $FIRESTORE_EMULATOR_HOST" >&2 + docker logs "$FIRESTORE_CONTAINER" >&2 || true + stop_firestore_emulator + return 1 +} + +stop_firestore_emulator() { + if [[ -n "${FIRESTORE_CONTAINER:-}" ]]; then + docker rm --force "$FIRESTORE_CONTAINER" >/dev/null 2>&1 || true + fi +} diff --git a/tests/terra-tdr/run.sh b/tests/terra-tdr/run.sh index a07dc749..4d6ebdd0 100755 --- a/tests/terra-tdr/run.sh +++ b/tests/terra-tdr/run.sh @@ -13,22 +13,35 @@ port_file="$coord_dir/port" stop_file="$coord_dir/stop" tdr_log="$coord_dir/tdr.log" +source "$git_drs_root/tests/terra-tdr/firestore-emulator.sh" +source "$git_drs_root/tests/terra-tdr/storage-emulator.sh" +FIRESTORE_CONTAINER= +STORAGE_CONTAINER= +start_firestore_emulator +start_storage_emulator "$coord_dir/gcs-seed" + cp "$git_drs_root/tests/terra-tdr/TerraDrsHarnessTest.java" \ "$tdr_root/src/test/java/bio/terra/service/filedata/TerraDrsHarnessTest.java" ( cd "$tdr_root" GIT_DRS_TDR_PORT_FILE="$port_file" GIT_DRS_TDR_STOP_FILE="$stop_file" \ + FIRESTORE_EMULATOR_HOST="$FIRESTORE_EMULATOR_HOST" \ + GIT_DRS_TDR_GCS_ENDPOINT="$STORAGE_EMULATOR_HTTP_ENDPOINT" \ ./gradlew testUnit --tests bio.terra.service.filedata.TerraDrsHarnessTest \ - --no-daemon --console=plain + --no-daemon --console=plain --rerun-tasks ) >"$tdr_log" 2>&1 & tdr_pid=$! -stop_tdr() { +stop_services() { touch "$stop_file" - wait "$tdr_pid" || true + if [[ -n "${tdr_pid:-}" ]]; then + wait "$tdr_pid" || true + fi + stop_firestore_emulator + stop_storage_emulator } -trap stop_tdr EXIT +trap stop_services EXIT for ((seconds=0; seconds<900; seconds++)); do if [[ -s "$port_file" ]]; then @@ -50,17 +63,28 @@ fi port=$(cat "$port_file") cd "$git_drs_root" if ! GIT_DRS_TDR_HTTP_ENDPOINT="http://127.0.0.1:$port" \ + GIT_DRS_TDR_GCS_ENDPOINT="$STORAGE_EMULATOR_HTTP_ENDPOINT" \ go test -race -tags=integration -count=1 ./cmd/pull \ -run '^TestIntegrationPullAgainstTDRController$' -v; then cat "$tdr_log" >&2 exit 1 fi +downloads=$(storage_object_download_count) +if [[ "$downloads" != 1 ]]; then + echo "GCS emulator served $downloads object downloads, want exactly one" >&2 + docker logs "$STORAGE_CONTAINER" >&2 || true + exit 1 +fi + touch "$stop_file" -trap - EXIT if ! wait "$tdr_pid"; then cat "$tdr_log" >&2 exit 1 fi +tdr_pid= +stop_firestore_emulator +stop_storage_emulator +trap - EXIT tail -n 20 "$tdr_log" diff --git a/tests/terra-tdr/storage-emulator.sh b/tests/terra-tdr/storage-emulator.sh new file mode 100755 index 00000000..58fd97d6 --- /dev/null +++ b/tests/terra-tdr/storage-emulator.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +start_storage_emulator() { + local seed_dir=$1 + local requested_port + requested_port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') + STORAGE_CONTAINER="git-drs-storage-${PPID}-$$" + STORAGE_EMULATOR_HOST="127.0.0.1:$requested_port" + STORAGE_EMULATOR_HTTP_ENDPOINT="http://$STORAGE_EMULATOR_HOST" + export STORAGE_EMULATOR_HOST + export STORAGE_EMULATOR_HTTP_ENDPOINT + + mkdir -p "$seed_dir/fixture-bucket" + printf 'git-drs Terra DRS HTTP integration fixture\n' \ + >"$seed_dir/fixture-bucket/1614321.merge_output.gvcf.gz" + + docker run --detach --rm \ + --name "$STORAGE_CONTAINER" \ + --publish "127.0.0.1:$requested_port:8000" \ + --volume "$seed_dir:/seed:ro" \ + fsouza/fake-gcs-server@sha256:797ce226d62f947c009dc40246b30cfb456b8473d8241407f9d6f2c04e4d69ef \ + -backend filesystem \ + -data /seed \ + -filesystem-root /storage \ + -scheme http \ + -port 8000 \ + -public-host "$STORAGE_EMULATOR_HOST" >/dev/null + + for ((attempt = 0; attempt < 60; attempt++)); do + if curl --fail --silent "$STORAGE_EMULATOR_HTTP_ENDPOINT/_internal/healthcheck" >/dev/null; then + return 0 + fi + local running + running=$(docker inspect --format '{{.State.Running}}' "$STORAGE_CONTAINER" 2>/dev/null || true) + if [[ "$running" != true ]]; then + docker logs "$STORAGE_CONTAINER" >&2 || true + stop_storage_emulator + return 1 + fi + sleep 1 + done + + echo "GCS emulator did not start on $STORAGE_EMULATOR_HTTP_ENDPOINT" >&2 + docker logs "$STORAGE_CONTAINER" >&2 || true + stop_storage_emulator + return 1 +} + +stop_storage_emulator() { + if [[ -n "${STORAGE_CONTAINER:-}" ]]; then + docker rm --force "$STORAGE_CONTAINER" >/dev/null 2>&1 || true + fi +} + +storage_object_download_count() { + docker logs "$STORAGE_CONTAINER" 2>&1 | + python3 -c 'import sys; marker = "GET /fixture-bucket/1614321.merge_output.gvcf.gz?X-Goog-Algorithm="; print(sum(marker in line for line in sys.stdin))' +} From afae32c9e03c55225b38ff5680f230b19458ca08 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 8 Oct 2026 11:35:07 -0700 Subject: [PATCH 13/22] Exercise Terra pull with CLI, ADC, and signed storage URL --- .../terra_gcs_url_rewrite_integration_test.go | 112 ++++++ cmd/pull/terra_gcs_url_rewrite_test.go | 183 ++++++++- cmd/pull/terra_hub_integration_test.go | 349 +++++++++++++++--- cmd/pull/terra_tdr_integration_test.go | 15 +- tests/terra-drs-hub/README.md | 8 +- tests/terra-drs-hub/run.sh | 10 +- tests/terra-drs-hub/tdr_https_proxy.py | 7 +- tests/terra-tdr/README.md | 6 +- tests/terra-tdr/TerraDrsHarnessTest.java | 62 +++- tests/terra-tdr/run.sh | 3 + tests/terra-tdr/storage-emulator.sh | 4 +- 11 files changed, 666 insertions(+), 93 deletions(-) create mode 100644 cmd/pull/terra_gcs_url_rewrite_integration_test.go diff --git a/cmd/pull/terra_gcs_url_rewrite_integration_test.go b/cmd/pull/terra_gcs_url_rewrite_integration_test.go new file mode 100644 index 00000000..11479f5c --- /dev/null +++ b/cmd/pull/terra_gcs_url_rewrite_integration_test.go @@ -0,0 +1,112 @@ +//go:build integration + +package pull + +import ( + "bytes" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/hex" + "encoding/json" + "encoding/pem" + "io" + "net/http" + "net/url" + "strings" + "testing" + "time" +) + +func TestRewriteTDRSignedURLVerifiesV4SignatureAndExpiry(t *testing.T) { + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + publicKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: mustPKIXPublicKey(t, &privateKey.PublicKey)}) + now := time.Now().UTC().Truncate(time.Second) + date := now.Format("20060102T150405Z") + query := url.Values{ + "X-Goog-Algorithm": {gcsSigningAlgorithm}, + "X-Goog-Credential": {"git-drs-ci@fixture.invalid/" + now.Format("20060102") + "/auto/storage/goog4_request"}, + "X-Goog-Date": {date}, + "X-Goog-Expires": {"900"}, + "X-Goog-SignedHeaders": {"host"}, + } + signedURL := &url.URL{Scheme: "http", Host: "127.0.0.1", Path: "/fixture-bucket/object"} + signedURL.RawQuery = query.Encode() + canonical := strings.Join([]string{ + "GET", + signedURL.EscapedPath(), + canonicalGCSQuery(query), + "host:" + gcsSigningHost + "\n", + "host", + "UNSIGNED-PAYLOAD", + }, "\n") + canonicalHash := sha256.Sum256([]byte(canonical)) + stringToSign := strings.Join([]string{ + gcsSigningAlgorithm, + date, + now.Format("20060102") + "/auto/storage/goog4_request", + hex.EncodeToString(canonicalHash[:]), + }, "\n") + digest := sha256.Sum256([]byte(stringToSign)) + signature, err := rsa.SignPKCS1v15(rand.Reader, privateKey, crypto.SHA256, digest[:]) + if err != nil { + t.Fatal(err) + } + query.Set("X-Goog-Signature", hex.EncodeToString(signature)) + signedURL.RawQuery = query.Encode() + responseBody, err := json.Marshal(map[string]any{"accessUrl": map[string]any{"url": signedURL.String()}}) + if err != nil { + t.Fatal(err) + } + response := &http.Response{Body: io.NopCloser(bytes.NewReader(responseBody))} + if err := rewriteTDRSignedURLForEmulator(response, "http://127.0.0.1:12345", publicKeyPEM); err != nil { + t.Fatalf("valid signature should be accepted: %v", err) + } + rewrittenBody, err := io.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(rewrittenBody), "http://127.0.0.1:12345/fixture-bucket/object?") { + t.Fatalf("signed URL was not rewritten to emulator: %s", rewrittenBody) + } + + for name, tamper := range map[string]func(*url.URL){ + "path": func(value *url.URL) { value.Path = "/fixture-bucket/other" }, + "signature": func(value *url.URL) { + query := value.Query() + signature := []byte(query.Get("X-Goog-Signature")) + if signature[0] == '0' { + signature[0] = '1' + } else { + signature[0] = '0' + } + query.Set("X-Goog-Signature", string(signature)) + value.RawQuery = query.Encode() + }, + } { + t.Run(name, func(t *testing.T) { + mutated := *signedURL + tamper(&mutated) + if err := verifyGCSV4SignedURL(&mutated, publicKeyPEM, gcsSigningHost, now); err == nil { + t.Fatal("tampered signed URL was accepted") + } + }) + } + if err := verifyGCSV4SignedURL(signedURL, publicKeyPEM, gcsSigningHost, now.Add(16*time.Minute)); err == nil { + t.Fatal("expired signed URL was accepted") + } +} + +func mustPKIXPublicKey(t *testing.T, publicKey *rsa.PublicKey) []byte { + t.Helper() + der, err := x509.MarshalPKIXPublicKey(publicKey) + if err != nil { + t.Fatal(err) + } + return der +} diff --git a/cmd/pull/terra_gcs_url_rewrite_test.go b/cmd/pull/terra_gcs_url_rewrite_test.go index 44e3292c..b1039954 100644 --- a/cmd/pull/terra_gcs_url_rewrite_test.go +++ b/cmd/pull/terra_gcs_url_rewrite_test.go @@ -4,34 +4,199 @@ package pull import ( "bytes" + "crypto" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/hex" + "encoding/json" + "encoding/pem" "fmt" "io" "net/http" + "net/url" + "sort" + "strconv" "strings" + "time" ) -func rewriteTDRSignedURLForEmulator(response *http.Response, emulatorEndpoint string) error { +const ( + gcsSigningAlgorithm = "GOOG4-RSA-SHA256" + gcsSignedAuthority = "http://127.0.0.1/" + gcsSigningHost = "storage.googleapis.com" +) + +func rewriteTDRSignedURLForEmulator(response *http.Response, emulatorEndpoint string, publicKeyPEM []byte) error { if response == nil || response.Body == nil { return fmt.Errorf("TDR response has no body") } + if !strings.HasPrefix(emulatorEndpoint, "http://127.0.0.1:") && !strings.HasPrefix(emulatorEndpoint, "http://host.docker.internal:") { + return fmt.Errorf("invalid local GCS emulator endpoint %q", emulatorEndpoint) + } body, err := io.ReadAll(response.Body) if err != nil { return fmt.Errorf("read TDR response: %w", err) } _ = response.Body.Close() - if !bytes.Contains(body, []byte("X-Goog-Signature=")) { - return fmt.Errorf("TDR access response is missing a V4 signed URL") + + signedURL, err := signedURLFromJSON(body) + if err != nil { + return err } - const sourceAuthority = "http://127.0.0.1/" - if !bytes.Contains(body, []byte(sourceAuthority)) { - return fmt.Errorf("TDR signed URL does not use the expected local storage host") + if signedURL.Scheme+"://"+signedURL.Host+"/" != gcsSignedAuthority { + return fmt.Errorf("TDR signed URL authority is %q, want %q", signedURL.Scheme+"://"+signedURL.Host+"/", gcsSignedAuthority) } - if !strings.HasPrefix(emulatorEndpoint, "http://127.0.0.1:") { - return fmt.Errorf("invalid local GCS emulator endpoint %q", emulatorEndpoint) + if err := verifyGCSV4SignedURL(signedURL, publicKeyPEM, gcsSigningHost, time.Now()); err != nil { + return fmt.Errorf("verify TDR GCS signed URL: %w", err) + } + oldAuthority := signedURL.Scheme + "://" + signedURL.Host + newAuthority := strings.TrimRight(emulatorEndpoint, "/") + if bytes.Count(body, []byte(oldAuthority)) != 1 { + return fmt.Errorf("TDR response must contain its signed URL authority exactly once") } - body = bytes.Replace(body, []byte(sourceAuthority), []byte(strings.TrimRight(emulatorEndpoint, "/")+"/"), 1) + body = bytes.Replace(body, []byte(oldAuthority), []byte(newAuthority), 1) response.Body = io.NopCloser(bytes.NewReader(body)) response.ContentLength = int64(len(body)) + if response.Header == nil { + response.Header = make(http.Header) + } response.Header.Set("Content-Length", fmt.Sprint(len(body))) return nil } + +func signedURLFromJSON(body []byte) (*url.URL, error) { + var document any + if err := json.Unmarshal(body, &document); err != nil { + return nil, fmt.Errorf("decode TDR access response: %w", err) + } + var signedURLs []*url.URL + var visit func(any) + visit = func(value any) { + switch value := value.(type) { + case map[string]any: + for _, child := range value { + visit(child) + } + case []any: + for _, child := range value { + visit(child) + } + case string: + parsed, err := url.Parse(value) + if err == nil && parsed.IsAbs() && parsed.Query().Get("X-Goog-Algorithm") != "" { + signedURLs = append(signedURLs, parsed) + } + } + } + visit(document) + if len(signedURLs) != 1 { + return nil, fmt.Errorf("TDR response contains %d V4 signed URLs, want exactly one", len(signedURLs)) + } + return signedURLs[0], nil +} + +func verifyGCSV4SignedURL(signedURL *url.URL, publicKeyPEM []byte, signingHost string, now time.Time) error { + if signedURL == nil || signedURL.Scheme != "http" || signedURL.Host == "" || signedURL.Path == "" { + return fmt.Errorf("signed URL must be an absolute HTTP URL with a host and path") + } + query := signedURL.Query() + if query.Get("X-Goog-Algorithm") != gcsSigningAlgorithm { + return fmt.Errorf("unsupported signing algorithm %q", query.Get("X-Goog-Algorithm")) + } + credential := query.Get("X-Goog-Credential") + credentialParts := strings.SplitN(credential, "/", 2) + if len(credentialParts) != 2 || credentialParts[0] == "" || credentialParts[1] == "" { + return fmt.Errorf("signed URL has invalid credential scope %q", credential) + } + signedAt, err := time.Parse("20060102T150405Z", query.Get("X-Goog-Date")) + if err != nil { + return fmt.Errorf("signed URL has invalid signing time: %w", err) + } + expires, err := strconv.ParseInt(query.Get("X-Goog-Expires"), 10, 64) + if err != nil || expires < 1 || expires > 604800 { + return fmt.Errorf("signed URL has invalid expiration %q", query.Get("X-Goog-Expires")) + } + if now.Before(signedAt.Add(-5*time.Minute)) || !now.Before(signedAt.Add(time.Duration(expires)*time.Second)) { + return fmt.Errorf("signed URL is not currently valid (signed %s, expires after %ds)", signedAt.UTC().Format(time.RFC3339), expires) + } + if query.Get("X-Goog-SignedHeaders") != "host" { + return fmt.Errorf("signed URL signed headers are %q, want host", query.Get("X-Goog-SignedHeaders")) + } + signature, err := hex.DecodeString(query.Get("X-Goog-Signature")) + if err != nil || len(signature) == 0 { + return fmt.Errorf("signed URL has an invalid signature encoding") + } + publicKey, err := parseRSAPublicKey(publicKeyPEM) + if err != nil { + return err + } + if len(signature) != publicKey.Size() { + return fmt.Errorf("signed URL signature is %d bytes, RSA key requires %d", len(signature), publicKey.Size()) + } + + canonicalRequest := strings.Join([]string{ + "GET", + signedURL.EscapedPath(), + canonicalGCSQuery(query), + "host:" + signingHost + "\n", + "host", + "UNSIGNED-PAYLOAD", + }, "\n") + canonicalHash := sha256.Sum256([]byte(canonicalRequest)) + stringToSign := strings.Join([]string{ + gcsSigningAlgorithm, + query.Get("X-Goog-Date"), + credentialParts[1], + hex.EncodeToString(canonicalHash[:]), + }, "\n") + digest := sha256.Sum256([]byte(stringToSign)) + if err := rsa.VerifyPKCS1v15(publicKey, crypto.SHA256, digest[:], signature); err != nil { + return fmt.Errorf("RSA-SHA256 signature does not match the canonical request: %w", err) + } + return nil +} + +func canonicalGCSQuery(query url.Values) string { + type pair struct{ key, value string } + pairs := make([]pair, 0, len(query)) + for key, values := range query { + if key == "X-Goog-Signature" { + continue + } + for _, value := range values { + pairs = append(pairs, pair{rfc3986Escape(key), rfc3986Escape(value)}) + } + } + sort.Slice(pairs, func(i, j int) bool { + if pairs[i].key == pairs[j].key { + return pairs[i].value < pairs[j].value + } + return pairs[i].key < pairs[j].key + }) + parts := make([]string, len(pairs)) + for i, pair := range pairs { + parts[i] = pair.key + "=" + pair.value + } + return strings.Join(parts, "&") +} + +func rfc3986Escape(value string) string { + return strings.ReplaceAll(url.QueryEscape(value), "+", "%20") +} + +func parseRSAPublicKey(publicKeyPEM []byte) (*rsa.PublicKey, error) { + block, _ := pem.Decode(publicKeyPEM) + if block == nil { + return nil, fmt.Errorf("TDR signing public key is not PEM encoded") + } + parsed, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("parse TDR signing public key: %w", err) + } + publicKey, ok := parsed.(*rsa.PublicKey) + if !ok { + return nil, fmt.Errorf("TDR signing public key is %T, want RSA", parsed) + } + return publicKey, nil +} diff --git a/cmd/pull/terra_hub_integration_test.go b/cmd/pull/terra_hub_integration_test.go index 39cb45e1..d4683131 100644 --- a/cmd/pull/terra_hub_integration_test.go +++ b/cmd/pull/terra_hub_integration_test.go @@ -5,78 +5,209 @@ package pull import ( "bytes" "context" + "crypto" + "crypto/md5" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/base64" "encoding/json" + "encoding/pem" "fmt" + "hash/crc32" + "io" + "math/big" + "net" "net/http" + "net/http/httptest" "net/url" "os" + "os/exec" "path/filepath" "strings" + "sync/atomic" "testing" + "time" - "github.com/calypr/git-drs/internal/config" "github.com/calypr/git-drs/internal/lfs" - "github.com/calypr/git-drs/internal/resolver" ) func TestIntegrationPullThroughTerraHubAndTDR(t *testing.T) { + payload := []byte("git-drs Terra DRS HTTP integration fixture\n") + binary := os.Getenv("GIT_DRS_BINARY") hubEndpoint := os.Getenv("GIT_DRS_HUB_HTTP_ENDPOINT") proxyLog := os.Getenv("GIT_DRS_TDR_PROXY_LOG") gcsEndpoint := os.Getenv("GIT_DRS_TDR_GCS_ENDPOINT") - if hubEndpoint == "" || proxyLog == "" || gcsEndpoint == "" { - t.Skip("GIT_DRS_HUB_HTTP_ENDPOINT, GIT_DRS_TDR_PROXY_LOG, and GIT_DRS_TDR_GCS_ENDPOINT are required") + keyPath := os.Getenv("GIT_DRS_TDR_SIGNING_PUBLIC_KEY_FILE") + if binary == "" || hubEndpoint == "" || proxyLog == "" || gcsEndpoint == "" || keyPath == "" { + t.Skip("GIT_DRS_BINARY and the local Hub/TDR/GCS fixture endpoints are required") + } + publicKeyPEM, err := os.ReadFile(keyPath) + if err != nil { + t.Fatalf("read TDR signing public key: %v", err) } - parsed, err := url.Parse(hubEndpoint) - if err != nil || parsed.Scheme != "http" || parsed.Hostname() != "127.0.0.1" { + upstream, err := url.Parse(hubEndpoint) + if err != nil || upstream.Scheme != "http" || upstream.Hostname() != "127.0.0.1" { t.Fatalf("invalid local Hub endpoint %q", hubEndpoint) } - oldResolver := newHubResolver - t.Cleanup(func() { - newHubResolver = oldResolver - resetPullFlagsForTest() - }) - var hubCalls int - newHubResolver = func(_ context.Context, endpoint string) (resolver.Resolver, error) { - client := &http.Client{Transport: pullRoundTripFunc(func(req *http.Request) (*http.Response, error) { - if req.URL.Path == "/api/v4/drs/resolve" { - hubCalls++ + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("generate mock ADC key: %v", err) + } + var tokenRequests atomic.Int32 + tokenListener, err := net.Listen("tcp", "0.0.0.0:0") + if err != nil { + t.Fatalf("listen for mock OAuth token endpoint: %v", err) + } + tokenEndpoint := fmt.Sprintf("http://host.docker.internal:%d", tokenListener.Addr().(*net.TCPAddr).Port) + tokenServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/token" { + http.Error(w, "unexpected OAuth request", http.StatusNotFound) + return + } + if err := r.ParseForm(); err != nil { + http.Error(w, "bad form", http.StatusBadRequest) + return + } + if r.Form.Get("grant_type") != "urn:ietf:params:oauth:grant-type:jwt-bearer" { + http.Error(w, "unexpected grant type", http.StatusBadRequest) + return + } + parts := strings.Split(r.Form.Get("assertion"), ".") + if len(parts) != 3 { + http.Error(w, "invalid service-account JWT", http.StatusBadRequest) + return + } + claimsBytes, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + http.Error(w, "invalid JWT claims", http.StatusBadRequest) + return + } + var claims struct { + Issuer string `json:"iss"` + Audience string `json:"aud"` + Scope string `json:"scope"` + } + if err := json.Unmarshal(claimsBytes, &claims); err != nil || claims.Issuer != "git-drs-ci@example.invalid" || claims.Audience != tokenEndpoint+"/token" || !strings.Contains(claims.Scope, "cloud-platform") { + http.Error(w, "unexpected service-account claims", http.StatusBadRequest) + return + } + signature, err := base64.RawURLEncoding.DecodeString(parts[2]) + if err != nil { + http.Error(w, "invalid JWT signature encoding", http.StatusBadRequest) + return + } + digest := sha256.Sum256([]byte(parts[0] + "." + parts[1])) + if err := rsa.VerifyPKCS1v15(&key.PublicKey, crypto.SHA256, digest[:], signature); err != nil { + http.Error(w, "service-account JWT signature invalid", http.StatusBadRequest) + return + } + tokenRequests.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"access_token":"tdr-ci-fixture","token_type":"Bearer","expires_in":3600}`) + })) + tokenServer.Listener = tokenListener + tokenServer.Start() + defer tokenServer.Close() + + var resolveRequests atomic.Int32 + var hubBearer atomic.Value + frontListener, err := net.Listen("tcp", "0.0.0.0:0") + if err != nil { + t.Fatalf("listen for local Hub proxy: %v", err) + } + frontCertificate, frontCertificatePEM, err := makeDockerHostCertificate() + if err != nil { + t.Fatalf("make local Hub TLS certificate: %v", err) + } + frontProxy := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + forwarded, err := http.NewRequestWithContext(r.Context(), r.Method, "http://"+upstream.Host+r.URL.RequestURI(), r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + forwarded.Header = r.Header.Clone() + forwarded.Host = upstream.Host + response, err := http.DefaultTransport.RoundTrip(forwarded) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + defer response.Body.Close() + if r.URL.Path == "/api/v4/drs/resolve" { + resolveRequests.Add(1) + hubBearer.Store(r.Header.Get("Authorization")) + if err := verifyTerraHubMetadata(response, payload); err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return } - copy := req.Clone(req.Context()) - copy.Header.Set("Authorization", "Bearer tdr-ci-fixture") - response, err := http.DefaultTransport.RoundTrip(copy) - if err != nil { - return nil, err + if err := rewriteTDRSignedURLForEmulator(response, gcsEndpoint, publicKeyPEM); err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return } - if req.URL.Path == "/api/v4/drs/resolve" { - if err := rewriteTDRSignedURLForEmulator(response, gcsEndpoint); err != nil { - return nil, err - } + } + for name, values := range response.Header { + if strings.EqualFold(name, "Connection") || strings.EqualFold(name, "Transfer-Encoding") { + continue + } + for _, value := range values { + w.Header().Add(name, value) } - return response, nil - })} - return resolver.NewTerraHubWithClient(endpoint, client) + } + w.WriteHeader(response.StatusCode) + _, _ = io.Copy(w, response.Body) + })) + frontProxy.Listener = frontListener + frontProxy.TLS = &tls.Config{Certificates: []tls.Certificate{frontCertificate}, MinVersion: tls.VersionTLS12} + frontProxy.StartTLS() + defer frontProxy.Close() + frontProxyURL := fmt.Sprintf("https://host.docker.internal:%d", frontListener.Addr().(*net.TCPAddr).Port) + + credentialsPath := filepath.Join(t.TempDir(), "service-account.json") + privateKeyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("marshal ADC key: %v", err) + } + credentials, err := json.Marshal(map[string]string{ + "type": "service_account", "project_id": "git-drs-ci", "private_key_id": "git-drs-ci-key", + "private_key": string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateKeyDER})), + "client_email": "git-drs-ci@example.invalid", "client_id": "1234567890", + "token_uri": tokenEndpoint + "/token", "universe_domain": "googleapis.com", + }) + if err != nil { + t.Fatalf("encode ADC credentials: %v", err) + } + if err := os.WriteFile(credentialsPath, credentials, 0o600); err != nil { + t.Fatalf("write ADC credentials: %v", err) + } + certPath := filepath.Join(t.TempDir(), "hub-proxy-ca.pem") + if err := os.WriteFile(certPath, frontCertificatePEM, 0o600); err != nil { + t.Fatalf("write Hub proxy TLS certificate: %v", err) } repo := t.TempDir() runGitCmdTest(t, repo, "init", "-q") - t.Chdir(repo) + runGitCmdTest(t, repo, "config", "user.name", "git-drs CI fixture") + runGitCmdTest(t, repo, "config", "user.email", "git-drs-ci@example.invalid") + runGitCmdTest(t, repo, "config", "drs.default-remote", "anvil") + runGitCmdTest(t, repo, "config", "drs.remote.anvil.type", "terra") + runGitCmdTest(t, repo, "config", "drs.remote.anvil.endpoint", "https://data.terra.bio") + runGitCmdTest(t, repo, "config", "drs.remote.anvil.hub-endpoint", frontProxyURL) + runGitCmdTest(t, repo, "config", "drs.remote.anvil.mode", "read-only") const filename = "1614321.merge_output.gvcf.gz" const oid = "drs://drs.anv0:v2_4f770147-e372-339b-b9fa-0a7a83cf30cf" - payload := []byte("git-drs Terra DRS HTTP integration fixture\n") - if err := os.WriteFile(".gitattributes", []byte(filename+" filter=drs diff=drs merge=drs -text\n"), 0o644); err != nil { - t.Fatal(err) - } pointer := fmt.Sprintf("version https://calypr.github.io/spec/v1\noid %s\nsize %d\n", oid, len(payload)) - if err := os.WriteFile(filename, []byte(pointer), 0o644); err != nil { + if err := os.WriteFile(filepath.Join(repo, ".gitattributes"), []byte(filename+" filter=drs diff=drs merge=drs -text\n"), 0o644); err != nil { t.Fatal(err) } - runGitCmdTest(t, repo, "add", filename) - if _, err := config.UpdateRemote("anvil", config.RemoteSelect{Terra: &config.TerraRemote{ - Endpoint: "https://data.terra.bio", HubEndpoint: hubEndpoint, Mode: "read-only", - }}); err != nil { + if err := os.WriteFile(filepath.Join(repo, filename), []byte(pointer), 0o644); err != nil { t.Fatal(err) } + runGitCmdTest(t, repo, "add", filename) + t.Chdir(repo) objectsRoot, err := lfs.ResolveObjectsRoot(context.Background()) if err != nil { t.Fatal(err) @@ -88,43 +219,149 @@ func TestIntegrationPullThroughTerraHubAndTDR(t *testing.T) { if _, err := os.Stat(cachePath); !os.IsNotExist(err) { t.Fatalf("cache must be empty before pull: %s: %v", cachePath, err) } - resetPullFlagsForTest() - includePatterns = []string{filename} - if err := Cmd.RunE(Cmd, nil); err != nil { - t.Fatalf("pull through Hub and TDR: %v", err) + + runPull := func() []byte { + command := exec.Command("docker", "run", "--rm", + "--add-host", "host.docker.internal:host-gateway", + "--volume", binary+":/usr/local/bin/git-drs:ro", + "--volume", repo+":/repo", + "--volume", credentialsPath+":/ci/service-account.json:ro", + "--volume", certPath+":/ci/hub-proxy-ca.pem:ro", + "--workdir", "/repo", + "--env", "HOME=/tmp/git-drs-ci-home", + "--env", "GIT_CONFIG_NOSYSTEM=1", + "--env", "GOOGLE_APPLICATION_CREDENTIALS=/ci/service-account.json", + "--env", "SSL_CERT_FILE=/ci/hub-proxy-ca.pem", + "--entrypoint", "/bin/sh", "alpine/git@sha256:062a01ad7a0eb17cff382bc5e26086b4d710e56dfdfdf001109a49b6d9bd378c", "-c", + "mkdir -p /tmp/git-drs-ci-home && git config --global --add safe.directory /repo && git config --global filter.drs.clean \"/usr/local/bin/git-drs clean -- %f\" && git config --global filter.drs.smudge \"/usr/local/bin/git-drs smudge -- %f\" && git config --global filter.drs.process \"/usr/local/bin/git-drs filter\" && git config --global filter.drs.required true && exec /usr/local/bin/git-drs pull -I "+filename) + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("git-drs pull exited with %v:\n%s", err, output) + } + return output } - got, err := os.ReadFile(filepath.Join(repo, filename)) - if err != nil { - t.Fatal(err) + output := runPull() + if got, err := os.ReadFile(filepath.Join(repo, filename)); err != nil || !bytes.Equal(got, payload) { + t.Fatalf("CLI worktree bytes = %q, want %q (error %v)\n%s", got, payload, err, output) + } + if got := tokenRequests.Load(); got != 1 { + t.Fatalf("ADC token exchanges = %d, want one", got) } - if !bytes.Equal(got, payload) { - t.Fatalf("hydrated content = %q, want %q", got, payload) + if got := resolveRequests.Load(); got != 1 { + t.Fatalf("Hub resolve requests = %d, want one", got) } - if hubCalls != 1 { - t.Fatalf("Hub resolve calls = %d, want one", hubCalls) + if got := hubBearer.Load(); got != "Bearer tdr-ci-fixture" { + t.Fatalf("Hub received Authorization %v, want ADC fixture token", got) } - data, err := os.ReadFile(proxyLog) + if got := runGitOutputTest(t, repo, "show", ":"+filename); got != pointer { + t.Fatalf("staged index bytes = %q, want original pointer %q", got, pointer) + } + cached, err := os.ReadFile(cachePath) + if err != nil || !bytes.Equal(cached, payload) { + t.Fatalf("cache bytes do not match fixture: %d bytes, %v", len(cached), err) + } + proxyLogBytes, err := os.ReadFile(proxyLog) if err != nil { t.Fatal(err) } - var metadata, access bool - for _, line := range bytes.Split(bytes.TrimSpace(data), []byte("\n")) { + var metadataRequests, accessRequests int + var unauthenticatedProbe bool + for _, line := range bytes.Split(bytes.TrimSpace(proxyLogBytes), []byte("\n")) { var request struct { Method string `json:"method"` Path string `json:"path"` Authorization string `json:"authorization"` + Status int `json:"status"` } if err := json.Unmarshal(line, &request); err != nil { t.Fatalf("invalid TDR proxy log line %q: %v", line, err) } + if request.Method == "OPTIONS" && strings.HasSuffix(request.Path, "/objects/v2_4f770147-e372-339b-b9fa-0a7a83cf30cf") && request.Authorization == "" && request.Status == http.StatusOK { + unauthenticatedProbe = true + } if request.Method == "GET" && strings.HasSuffix(request.Path, "/objects/v2_4f770147-e372-339b-b9fa-0a7a83cf30cf") { - metadata = request.Authorization == "Bearer tdr-ci-fixture" + if request.Authorization != "Bearer tdr-ci-fixture" || request.Status != http.StatusOK { + t.Fatalf("invalid TDR metadata request: %s", line) + } + metadataRequests++ } if request.Method == "GET" && strings.Contains(request.Path, "/objects/v2_4f770147-e372-339b-b9fa-0a7a83cf30cf/access/") { - access = request.Authorization == "Bearer tdr-ci-fixture" + if request.Authorization != "Bearer tdr-ci-fixture" || request.Status != http.StatusOK { + t.Fatalf("invalid TDR access request: %s", line) + } + accessRequests++ } } - if !metadata || !access { - t.Fatalf("Hub did not make authenticated TDR metadata and access requests: %s", data) + if !unauthenticatedProbe || metadataRequests != 1 || accessRequests != 1 { + t.Fatalf("Hub did not complete the unauthenticated probe and authenticated TDR requests: %s", proxyLogBytes) + } + + secondOutput := runPull() + if got, err := os.ReadFile(filepath.Join(repo, filename)); err != nil || !bytes.Equal(got, payload) { + t.Fatalf("cached pull changed worktree file: %d bytes, %v\n%s", len(got), err, secondOutput) + } + if got := tokenRequests.Load(); got != 1 { + t.Fatalf("ADC token exchanges after cached pull = %d, want one", got) + } + if got := resolveRequests.Load(); got != 1 { + t.Fatalf("Hub resolve requests after cached pull = %d, want one", got) + } + secondProxyLog, err := os.ReadFile(proxyLog) + if err != nil || !bytes.Equal(secondProxyLog, proxyLogBytes) { + t.Fatalf("cached pull contacted TDR: before %d bytes, after %d bytes (error %v)", len(proxyLogBytes), len(secondProxyLog), err) + } +} + +func makeDockerHostCertificate() (tls.Certificate, []byte, error) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return tls.Certificate{}, nil, err + } + now := time.Now() + template := &x509.Certificate{ + SerialNumber: big.NewInt(now.UnixNano()), + NotBefore: now.Add(-time.Minute), NotAfter: now.Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, IsCA: true, DNSNames: []string{"host.docker.internal"}, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + return tls.Certificate{}, nil, err + } + certificatePEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + privateKeyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return tls.Certificate{}, nil, err + } + privateKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateKeyDER}) + certificate, err := tls.X509KeyPair(certificatePEM, privateKeyPEM) + return certificate, certificatePEM, err +} + +func verifyTerraHubMetadata(response *http.Response, payload []byte) error { + if response.StatusCode != http.StatusOK { + return fmt.Errorf("Hub returned HTTP %d", response.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(response.Body, 4<<20)) + if err != nil { + return fmt.Errorf("read Hub response: %w", err) + } + _ = response.Body.Close() + response.Body = io.NopCloser(bytes.NewReader(body)) + var metadata struct { + FileName string `json:"fileName"` + Size int64 `json:"size"` + Hashes map[string]string `json:"hashes"` + } + if err := json.Unmarshal(body, &metadata); err != nil { + return fmt.Errorf("decode Hub metadata: %w", err) + } + crc := crc32.Checksum(payload, crc32.MakeTable(crc32.Castagnoli)) + if metadata.FileName != "1614321.merge_output.gvcf.gz" || metadata.Size != int64(len(payload)) || + metadata.Hashes["md5"] != fmt.Sprintf("%x", md5.Sum(payload)) || + metadata.Hashes["crc32c"] != fmt.Sprintf("%08x", crc) { + return fmt.Errorf("Hub returned inconsistent file metadata: %s", body) } + return nil } diff --git a/cmd/pull/terra_tdr_integration_test.go b/cmd/pull/terra_tdr_integration_test.go index cca198ea..1b177996 100644 --- a/cmd/pull/terra_tdr_integration_test.go +++ b/cmd/pull/terra_tdr_integration_test.go @@ -25,23 +25,28 @@ import ( func TestIntegrationPullAgainstTDRController(t *testing.T) { endpoint := os.Getenv("GIT_DRS_TDR_HTTP_ENDPOINT") gcsEndpoint := os.Getenv("GIT_DRS_TDR_GCS_ENDPOINT") - if endpoint == "" || gcsEndpoint == "" { - t.Skip("GIT_DRS_TDR_HTTP_ENDPOINT and GIT_DRS_TDR_GCS_ENDPOINT are required") + keyPath := os.Getenv("GIT_DRS_TDR_SIGNING_PUBLIC_KEY_FILE") + if endpoint == "" || gcsEndpoint == "" || keyPath == "" { + t.Skip("GIT_DRS_TDR_HTTP_ENDPOINT, GIT_DRS_TDR_GCS_ENDPOINT, and GIT_DRS_TDR_SIGNING_PUBLIC_KEY_FILE are required") + } + publicKeyPEM, err := os.ReadFile(keyPath) + if err != nil { + t.Fatalf("read TDR signing public key: %v", err) } tdrURL, err := url.Parse(endpoint) if err != nil || tdrURL.Scheme != "http" || tdrURL.Host == "" { t.Fatalf("invalid local TDR endpoint %q", endpoint) } + var requestMu sync.Mutex + var requestPaths []string reverseProxy := httputil.NewSingleHostReverseProxy(tdrURL) reverseProxy.ModifyResponse = func(response *http.Response) error { if strings.Contains(response.Request.URL.Path, "/access/") { - return rewriteTDRSignedURLForEmulator(response, gcsEndpoint) + return rewriteTDRSignedURLForEmulator(response, gcsEndpoint, publicKeyPEM) } return nil } - var requestMu sync.Mutex - var requestPaths []string proxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requestMu.Lock() requestPaths = append(requestPaths, r.URL.Path) diff --git a/tests/terra-drs-hub/README.md b/tests/terra-drs-hub/README.md index 6ecfd591..e831748f 100644 --- a/tests/terra-drs-hub/README.md +++ b/tests/terra-drs-hub/README.md @@ -1,8 +1,12 @@ # Terra pull through DRS Hub and TDR -The `Resolve through DRS Hub and TDR` CI job runs a single `git-drs pull` against an AnVIL pointer. The CLI sends the DRS ID to a locally running DRS Hub. Hub expands `drs.anv0`, selects its TDR provider, and calls TDR's DRS API. TDR resolves the object through its real DRS controller, `DrsService`, PostgreSQL DRS ID and snapshot records, and Firestore file record. The CLI downloads the returned bytes and hydrates the worktree file. The test checks the hydrated bytes, a Hub resolve call, and authenticated TDR metadata and access requests. +The `Resolve through DRS Hub and TDR` CI job starts the pinned public TDR and DRS Hub source revisions and runs the real `git-drs` CLI binary against them. Hub expands the `drs.anv0` compact ID, selects its Terra provider, and calls TDR's DRS API. TDR's controller and `DrsService` resolve the PostgreSQL DRS-to-snapshot mapping, snapshot metadata, and Firestore file metadata, then return a Google Cloud Storage access URL. -The external authorization boundary uses a fixed test bearer token (`tdr-ci-fixture`) instead of Google credentials or Sam. Hub's unauthenticated DRS `OPTIONS` probe is rejected by that boundary, exercising Hub's configured-auth fallback. The local TLS proxy forwards Hub's requests to TDR and records them; it does not fabricate DRS responses. TDR's real Google Cloud Storage client looks up the seeded bucket and signs the access URL with a generated test-only key. A local token endpoint supplies an access token, and the GCS-compatible emulator serves the object bytes. The CLI-side test bridge changes the signed URL authority to the emulator's local address. The emulator does not validate signed URL signatures or expiry. The runner asserts one GCS object download. +The test uses a generated service-account ADC JSON key and a local OAuth token endpoint. The token endpoint validates the signed JWT grant before returning a fixture bearer token. Hub's initial unauthenticated `OPTIONS` probe reaches TDR, followed by authenticated metadata and access requests. The test checks the request methods, statuses, and bearer token. TDR authorization and some snapshot/provider dependencies remain fixture-backed; Google credentials and external Sam are not needed. + +TDR's real Google Storage client generates a V4 RSA signed URL using a generated test key. Before the CLI receives the Hub response, a local HTTPS bridge verifies the returned metadata and validates the signed URL's canonical request, RSA signature, and expiry. The bridge then changes only the signed URL authority so the CLI can reach the local GCS-compatible emulator from its Linux container. The emulator serves the fixture bytes but does not validate signed URLs itself. + +The runner builds a Linux `git-drs` binary for the host architecture and runs it inside a pinned multi-architecture Alpine Git image. It configures the actual clean, smudge, and process filters, checks that the worktree contains the downloaded bytes while the index still contains the pointer, verifies the cache, and runs a second pull to confirm it makes no additional Hub, OAuth, or TDR requests. It also asserts that the emulator served exactly one object download. Run locally using the pinned TDR and DRS Hub revisions in `.github/workflows/pr-checks.yaml`: diff --git a/tests/terra-drs-hub/run.sh b/tests/terra-drs-hub/run.sh index f53598fe..67c2e561 100755 --- a/tests/terra-drs-hub/run.sh +++ b/tests/terra-drs-hub/run.sh @@ -23,6 +23,7 @@ hub_compile_log="$coord_dir/hub-compile.log" certificate="$coord_dir/localhost.crt" private_key="$coord_dir/localhost.key" truststore="$coord_dir/truststore.p12" +signing_public_key_file="$coord_dir/tdr-signing-public-key.pem" cp "$git_drs_root/tests/terra-tdr/TerraDrsHarnessTest.java" \ "$tdr_root/src/test/java/bio/terra/service/filedata/TerraDrsHarnessTest.java" @@ -72,6 +73,7 @@ start_storage_emulator "$coord_dir/gcs-seed" ( cd "$tdr_root" GIT_DRS_TDR_PORT_FILE="$port_file" GIT_DRS_TDR_STOP_FILE="$stop_file" \ + GIT_DRS_TDR_SIGNING_PUBLIC_KEY_FILE="$signing_public_key_file" \ FIRESTORE_EMULATOR_HOST="$FIRESTORE_EMULATOR_HOST" \ GIT_DRS_TDR_GCS_ENDPOINT="$STORAGE_EMULATOR_HTTP_ENDPOINT" \ ./gradlew testUnit --tests bio.terra.service.filedata.TerraDrsHarnessTest \ @@ -176,9 +178,13 @@ fi hub_port=$(cat "$hub_port_file") cd "$git_drs_root" -if ! GIT_DRS_HUB_HTTP_ENDPOINT="http://127.0.0.1:$hub_port" \ +cli_binary="$coord_dir/git-drs-linux" +GOOS=linux GOARCH="$(go env GOARCH)" CGO_ENABLED=0 go build -o "$cli_binary" . +if ! GIT_DRS_BINARY="$cli_binary" \ + GIT_DRS_HUB_HTTP_ENDPOINT="http://127.0.0.1:$hub_port" \ GIT_DRS_TDR_PROXY_LOG="$proxy_log" \ - GIT_DRS_TDR_GCS_ENDPOINT="$STORAGE_EMULATOR_HTTP_ENDPOINT" \ + GIT_DRS_TDR_GCS_ENDPOINT="${STORAGE_EMULATOR_HTTP_ENDPOINT/127.0.0.1/host.docker.internal}" \ + GIT_DRS_TDR_SIGNING_PUBLIC_KEY_FILE="$signing_public_key_file" \ go test -race -tags=integration -count=1 ./cmd/pull \ -run '^TestIntegrationPullThroughTerraHubAndTDR$' -v; then cat "$hub_log" >&2 diff --git a/tests/terra-drs-hub/tdr_https_proxy.py b/tests/terra-drs-hub/tdr_https_proxy.py index 82069670..aeb880dc 100644 --- a/tests/terra-drs-hub/tdr_https_proxy.py +++ b/tests/terra-drs-hub/tdr_https_proxy.py @@ -38,9 +38,6 @@ def forward(self): "path": self.path, "authorization": self.headers.get("Authorization", ""), } - with open(request_log, "a", encoding="utf-8") as log: - log.write(json.dumps(request) + "\n") - connection = http.client.HTTPConnection(upstream.hostname, upstream.port, timeout=30) try: headers = { @@ -50,6 +47,7 @@ def forward(self): } connection.request(self.command, self.path, body=body, headers=headers) response = connection.getresponse() + request["status"] = response.status response_body = response.read() self.send_response(response.status, response.reason) for name, value in response.getheaders(): @@ -61,6 +59,7 @@ def forward(self): if self.command != "HEAD": self.wfile.write(response_body) except Exception as error: # Return the forwarding failure to the real Hub client. + request["status"] = 502 response_body = str(error).encode("utf-8") self.send_response(502) self.send_header("Content-Length", str(len(response_body))) @@ -68,6 +67,8 @@ def forward(self): self.end_headers() self.wfile.write(response_body) finally: + with open(request_log, "a", encoding="utf-8") as log: + log.write(json.dumps(request) + "\n") connection.close() def log_message(self, format, *args): diff --git a/tests/terra-tdr/README.md b/tests/terra-tdr/README.md index eb443d2d..6f891eb2 100644 --- a/tests/terra-tdr/README.md +++ b/tests/terra-tdr/README.md @@ -1,8 +1,8 @@ # TDR pull integration -`run.sh` starts TDR's real DRS controller, generated API routes, and `DrsService`, then runs the `git-drs pull` integration test against that server. TDR stores the compact DRS ID mapping and the snapshot, source, and project records in an ephemeral PostgreSQL Testcontainer. The real `SnapshotService` and `SnapshotDao` load that metadata; the DAO's dataset materialization and empty table/relationship collaborators remain fixture-backed. TDR reads the seeded file record through its real `FileService` and `FireStoreDao` against the Firestore emulator. A GCS-compatible emulator serves the fixture object over HTTP. +`run.sh` starts TDR's real DRS controller, generated API routes, and `DrsService`, then runs the `git-drs pull` integration test against that server. TDR stores the compact DRS ID mapping and the snapshot, source, project, and dataset records in an ephemeral PostgreSQL Testcontainer. The real `SnapshotService`, `SnapshotDao`, `DatasetDao`, `SnapshotTableDao`, `DatasetTableDao`, `DatasetRelationshipDao`, `AssetDao`, and `StorageResourceDao` load that metadata through TDR's production SQL. The fixture has a GCP Firestore storage resource, but no tables, relationships, or assets. Project resource lookup remains fixture-backed. TDR reads the seeded file record through its real `FileService` and `FireStoreDao` against the Firestore emulator. A GCS-compatible emulator serves the fixture object over HTTP. -Authorization remains fixture-backed. TDR uses the Google Cloud Storage Java client to look up the bucket and create a V4 signed URL with a generated test-only service-account key. A local token endpoint supplies its access token; no Google credentials are used. The test proxy changes only the signed URL's authority to point at the local GCS emulator. The emulator serves the bytes but does not validate the signature or expiry. +Authorization remains fixture-backed. TDR uses the Google Cloud Storage Java client to look up the bucket and create a V4 signed URL with a generated test-only service-account key. A local token endpoint supplies its access token; no Google credentials are used. The test verifies the URL's RSA signature and expiry using the generated public key before changing its authority to point at the local GCS emulator. The emulator serves the bytes but does not validate signed URLs itself. The pull starts with an empty cache, checks the hydrated bytes and TDR-provided checksums, and the runner asserts that the emulator served exactly one object download. @@ -12,4 +12,4 @@ Run locally with Java 17, Go, and Docker installed: tests/terra-tdr/run.sh /path/to/jade-data-repo ``` -The CI workflow pins the TDR source revision. The test covers TDR's controller and `DrsService`, PostgreSQL DRS and snapshot lookup, Firestore file lookup, GCS client calls, the returned signed URL shape, and the CLI download path. It does not exercise Sam authorization or TDR's full `DatasetDao` materialization. The local token endpoint also does not validate Google-issued credentials. +The CI workflow pins the TDR source revision. The test covers TDR's controller and `DrsService`, PostgreSQL DRS/snapshot/dataset lookup, Firestore file lookup, GCS client calls, the returned signed URL shape, and the CLI download path. It does not exercise Sam authorization or Google project resource lookup; the local token endpoint supplies the generated test identity's access token without validating Google-issued credentials. diff --git a/tests/terra-tdr/TerraDrsHarnessTest.java b/tests/terra-tdr/TerraDrsHarnessTest.java index 9c97b433..b5e3c807 100644 --- a/tests/terra-tdr/TerraDrsHarnessTest.java +++ b/tests/terra-tdr/TerraDrsHarnessTest.java @@ -24,8 +24,12 @@ import bio.terra.service.configuration.ConfigurationService; import bio.terra.service.dataset.Dataset; import bio.terra.service.dataset.DatasetDao; +import bio.terra.service.dataset.DatasetRelationshipDao; import bio.terra.service.dataset.DatasetService; import bio.terra.service.dataset.DatasetSummary; +import bio.terra.service.dataset.DatasetTableDao; +import bio.terra.service.dataset.AssetDao; +import bio.terra.service.dataset.StorageResourceDao; import bio.terra.service.duos.DuosClient; import bio.terra.service.duos.DuosDao; import bio.terra.service.filedata.google.firestore.EncodeFixture; @@ -52,6 +56,11 @@ import bio.terra.service.snapshotbuilder.SnapshotBuilderSettingsDao; import bio.terra.service.snapshotbuilder.SnapshotRequestDao; import bio.terra.service.tabulardata.google.bigquery.BigQuerySnapshotPdao; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.paramnames.ParameterNamesModule; import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; @@ -97,7 +106,8 @@ /** * A real HTTP harness for git-drs Terra DRS integration tests. The TDR API controller, generated - * API mapping, DrsService, SnapshotService, SnapshotDao, FireStoreDao, and the GCS client are real. + * API mapping, DrsService, SnapshotService, SnapshotDao, DatasetDao, FireStoreDao, and the GCS + * client are real. * The test stays alive until the CI driver creates GIT_DRS_TDR_STOP_FILE. */ @SpringBootTest( @@ -131,13 +141,11 @@ class TerraDrsHarnessTest { @MockitoBean private DatasetService datasetService; @MockitoBean private LoadService loadService; @MockitoBean private ProfileService profileService; - @MockitoBean private DatasetDao datasetDao; @MockitoBean private JournalService journalService; @MockitoBean private DuosDao duosDao; @MockitoBean private FireStoreDependencyDao dependencyDao; @MockitoBean private BigQuerySnapshotPdao bigQuerySnapshotPdao; @MockitoBean private SnapshotRequestDao snapshotRequestDao; - @MockitoBean private SnapshotTableDao snapshotTableDao; @MockitoBean private SnapshotRelationshipDao snapshotRelationshipDao; @MockitoBean private MetadataDataAccessUtils metadataDataAccessUtils; @MockitoBean private EcmService ecmService; @@ -180,16 +188,16 @@ void configureFixtureAndSignalReady() throws Exception { }); when(drsConfiguration.maxDrsLookups()).thenReturn(10); when(jobService.getActivePodCount()).thenReturn(1); - when(datasetDao.retrieve(DATASET_ID)) - .thenReturn(fixtureSnapshot().getFirstSnapshotSource().getDataset()); - when(snapshotTableDao.retrieveTables(SNAPSHOT_ID)).thenReturn(java.util.List.of()); when(resourceService.getProjectResource(SNAPSHOT_PROJECT_RESOURCE_ID)) .thenReturn( new GoogleProjectResource() .id(SNAPSHOT_PROJECT_RESOURCE_ID) .googleProjectId("fixture-snapshot-project")); - when(dataRepoJdbcConfiguration.getDataSource()) - .thenReturn(org.mockito.Mockito.mock(org.apache.commons.dbcp2.PoolingDataSource.class)); + when(resourceService.getProjectResource(DATASET_PROJECT_RESOURCE_ID)) + .thenReturn( + new GoogleProjectResource() + .id(DATASET_PROJECT_RESOURCE_ID) + .googleProjectId("fixture-dataset-project")); when(configurationService.getParameterValue(any())) .thenAnswer( invocation -> @@ -215,11 +223,22 @@ void configureFixtureAndSignalReady() throws Exception { tokenServer.start(); var keyPairGenerator = KeyPairGenerator.getInstance("RSA"); keyPairGenerator.initialize(2048); + var signingKeyPair = keyPairGenerator.generateKeyPair(); String privateKeyPem = "-----BEGIN PRIVATE KEY-----\n" + Base64.getMimeEncoder(64, new byte[] {'\n'}) - .encodeToString(keyPairGenerator.generateKeyPair().getPrivate().getEncoded()) + .encodeToString(signingKeyPair.getPrivate().getEncoded()) + "\n-----END PRIVATE KEY-----\n"; + String publicKeyPath = System.getenv("GIT_DRS_TDR_SIGNING_PUBLIC_KEY_FILE"); + if (publicKeyPath == null || publicKeyPath.isBlank()) { + throw new IllegalStateException("GIT_DRS_TDR_SIGNING_PUBLIC_KEY_FILE must be set"); + } + Files.writeString( + Path.of(publicKeyPath), + "-----BEGIN PUBLIC KEY-----\n" + + Base64.getMimeEncoder(64, new byte[] {'\n'}) + .encodeToString(signingKeyPair.getPublic().getEncoded()) + + "\n-----END PUBLIC KEY-----\n"); var storageCredentials = ServiceAccountCredentials.fromPkcs8( "git-drs-ci@fixture.invalid", @@ -306,11 +325,17 @@ private void migrateAndSeedDrsRecord() throws Exception { "100000000002", BILLING_PROFILE_ID); jdbcTemplate.update( - "INSERT INTO dataset (id, name, default_profile_id, project_resource_id, sharedlock, tags) VALUES (?, ?, ?, ?, '{}', ARRAY[]::TEXT[])", + "INSERT INTO dataset (id, name, default_profile_id, project_resource_id, self_hosted, sharedlock, tags) VALUES (?, ?, ?, ?, TRUE, '{}', ARRAY[]::TEXT[])", DATASET_ID, "git-drs-ci-fixture", BILLING_PROFILE_ID, DATASET_PROJECT_RESOURCE_ID); + jdbcTemplate.update( + "INSERT INTO storage_resource (dataset_id, region, cloud_resource, cloud_platform) VALUES (?, ?, ?, ?)", + DATASET_ID, + "US_CENTRAL1", + "FIRESTORE", + "GCP"); jdbcTemplate.update( "INSERT INTO snapshot (id, name, profile_id, project_resource_id) VALUES (?, ?, ?, ?)", SNAPSHOT_ID, @@ -414,11 +439,26 @@ private static void createParentDirectories(Path path) throws IOException { DrsIdService.class, DrsService.class, FileService.class, + DatasetDao.class, + DatasetTableDao.class, + AssetDao.class, + DatasetRelationshipDao.class, + StorageResourceDao.class, SnapshotDao.class, SnapshotService.class, - SnapshotMapTableDao.class + SnapshotMapTableDao.class, + SnapshotTableDao.class }) static class HarnessApplication { + @org.springframework.context.annotation.Bean(name = "daoObjectMapper") + ObjectMapper daoObjectMapper() { + return new ObjectMapper() + .registerModule(new ParameterNamesModule()) + .registerModule(new Jdk8Module()) + .registerModule(new JavaTimeModule()) + .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_VALUES, true); + } + @org.springframework.context.annotation.Bean ApplicationConfiguration applicationConfiguration() { ApplicationConfiguration configuration = new ApplicationConfiguration(); diff --git a/tests/terra-tdr/run.sh b/tests/terra-tdr/run.sh index 4d6ebdd0..053ab555 100755 --- a/tests/terra-tdr/run.sh +++ b/tests/terra-tdr/run.sh @@ -12,6 +12,7 @@ coord_dir=$(mktemp -d) port_file="$coord_dir/port" stop_file="$coord_dir/stop" tdr_log="$coord_dir/tdr.log" +signing_public_key_file="$coord_dir/tdr-signing-public-key.pem" source "$git_drs_root/tests/terra-tdr/firestore-emulator.sh" source "$git_drs_root/tests/terra-tdr/storage-emulator.sh" @@ -26,6 +27,7 @@ cp "$git_drs_root/tests/terra-tdr/TerraDrsHarnessTest.java" \ ( cd "$tdr_root" GIT_DRS_TDR_PORT_FILE="$port_file" GIT_DRS_TDR_STOP_FILE="$stop_file" \ + GIT_DRS_TDR_SIGNING_PUBLIC_KEY_FILE="$signing_public_key_file" \ FIRESTORE_EMULATOR_HOST="$FIRESTORE_EMULATOR_HOST" \ GIT_DRS_TDR_GCS_ENDPOINT="$STORAGE_EMULATOR_HTTP_ENDPOINT" \ ./gradlew testUnit --tests bio.terra.service.filedata.TerraDrsHarnessTest \ @@ -64,6 +66,7 @@ port=$(cat "$port_file") cd "$git_drs_root" if ! GIT_DRS_TDR_HTTP_ENDPOINT="http://127.0.0.1:$port" \ GIT_DRS_TDR_GCS_ENDPOINT="$STORAGE_EMULATOR_HTTP_ENDPOINT" \ + GIT_DRS_TDR_SIGNING_PUBLIC_KEY_FILE="$signing_public_key_file" \ go test -race -tags=integration -count=1 ./cmd/pull \ -run '^TestIntegrationPullAgainstTDRController$' -v; then cat "$tdr_log" >&2 diff --git a/tests/terra-tdr/storage-emulator.sh b/tests/terra-tdr/storage-emulator.sh index 58fd97d6..2e379fb2 100755 --- a/tests/terra-tdr/storage-emulator.sh +++ b/tests/terra-tdr/storage-emulator.sh @@ -16,7 +16,7 @@ start_storage_emulator() { docker run --detach --rm \ --name "$STORAGE_CONTAINER" \ - --publish "127.0.0.1:$requested_port:8000" \ + --publish "$requested_port:8000" \ --volume "$seed_dir:/seed:ro" \ fsouza/fake-gcs-server@sha256:797ce226d62f947c009dc40246b30cfb456b8473d8241407f9d6f2c04e4d69ef \ -backend filesystem \ @@ -24,7 +24,7 @@ start_storage_emulator() { -filesystem-root /storage \ -scheme http \ -port 8000 \ - -public-host "$STORAGE_EMULATOR_HOST" >/dev/null + -public-host "host.docker.internal:$requested_port" >/dev/null for ((attempt = 0; attempt < 60; attempt++)); do if curl --fail --silent "$STORAGE_EMULATOR_HTTP_ENDPOINT/_internal/healthcheck" >/dev/null; then From 1fac3d2d30c921e42c33279681abe2d5edd14138 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 8 Oct 2026 11:40:49 -0700 Subject: [PATCH 14/22] Match TDR formatter in CI harness --- tests/terra-tdr/TerraDrsHarnessTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/terra-tdr/TerraDrsHarnessTest.java b/tests/terra-tdr/TerraDrsHarnessTest.java index b5e3c807..34c17c08 100644 --- a/tests/terra-tdr/TerraDrsHarnessTest.java +++ b/tests/terra-tdr/TerraDrsHarnessTest.java @@ -22,13 +22,13 @@ import bio.terra.service.auth.ras.EcmService; import bio.terra.service.configuration.ConfigEnum; import bio.terra.service.configuration.ConfigurationService; +import bio.terra.service.dataset.AssetDao; import bio.terra.service.dataset.Dataset; import bio.terra.service.dataset.DatasetDao; import bio.terra.service.dataset.DatasetRelationshipDao; import bio.terra.service.dataset.DatasetService; import bio.terra.service.dataset.DatasetSummary; import bio.terra.service.dataset.DatasetTableDao; -import bio.terra.service.dataset.AssetDao; import bio.terra.service.dataset.StorageResourceDao; import bio.terra.service.duos.DuosClient; import bio.terra.service.duos.DuosDao; @@ -107,8 +107,7 @@ /** * A real HTTP harness for git-drs Terra DRS integration tests. The TDR API controller, generated * API mapping, DrsService, SnapshotService, SnapshotDao, DatasetDao, FireStoreDao, and the GCS - * client are real. - * The test stays alive until the CI driver creates GIT_DRS_TDR_STOP_FILE. + * client are real. The test stays alive until the CI driver creates GIT_DRS_TDR_STOP_FILE. */ @SpringBootTest( classes = TerraDrsHarnessTest.HarnessApplication.class, From f393b273db1b37d795e3c65c23763f5aa78357de Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 8 Oct 2026 11:51:07 -0700 Subject: [PATCH 15/22] Fix Terra CI storage host and cache ownership --- cmd/pull/terra_hub_integration_test.go | 1 + tests/terra-drs-hub/run.sh | 2 +- tests/terra-tdr/storage-emulator.sh | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/pull/terra_hub_integration_test.go b/cmd/pull/terra_hub_integration_test.go index d4683131..4f972e0f 100644 --- a/cmd/pull/terra_hub_integration_test.go +++ b/cmd/pull/terra_hub_integration_test.go @@ -223,6 +223,7 @@ func TestIntegrationPullThroughTerraHubAndTDR(t *testing.T) { runPull := func() []byte { command := exec.Command("docker", "run", "--rm", "--add-host", "host.docker.internal:host-gateway", + "--user", fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()), "--volume", binary+":/usr/local/bin/git-drs:ro", "--volume", repo+":/repo", "--volume", credentialsPath+":/ci/service-account.json:ro", diff --git a/tests/terra-drs-hub/run.sh b/tests/terra-drs-hub/run.sh index 67c2e561..45a4099e 100755 --- a/tests/terra-drs-hub/run.sh +++ b/tests/terra-drs-hub/run.sh @@ -68,7 +68,7 @@ tls_port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0) FIRESTORE_CONTAINER= start_firestore_emulator STORAGE_CONTAINER= -start_storage_emulator "$coord_dir/gcs-seed" +start_storage_emulator "$coord_dir/gcs-seed" host.docker.internal ( cd "$tdr_root" diff --git a/tests/terra-tdr/storage-emulator.sh b/tests/terra-tdr/storage-emulator.sh index 2e379fb2..65238708 100755 --- a/tests/terra-tdr/storage-emulator.sh +++ b/tests/terra-tdr/storage-emulator.sh @@ -2,6 +2,7 @@ start_storage_emulator() { local seed_dir=$1 + local public_host=${2:-127.0.0.1} local requested_port requested_port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') STORAGE_CONTAINER="git-drs-storage-${PPID}-$$" @@ -24,7 +25,7 @@ start_storage_emulator() { -filesystem-root /storage \ -scheme http \ -port 8000 \ - -public-host "host.docker.internal:$requested_port" >/dev/null + -public-host "$public_host:$requested_port" >/dev/null for ((attempt = 0; attempt < 60; attempt++)); do if curl --fail --silent "$STORAGE_EMULATOR_HTTP_ENDPOINT/_internal/healthcheck" >/dev/null; then From 20c5ced5207600bf64f3cba7fb1cdfe9192428f6 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 8 Oct 2026 11:58:26 -0700 Subject: [PATCH 16/22] Remove redundant direct TDR CI test --- .github/workflows/pr-checks.yaml | 33 ------- cmd/pull/terra_tdr_integration_test.go | 127 ------------------------- tests/terra-tdr/README.md | 15 --- tests/terra-tdr/run.sh | 93 ------------------ 4 files changed, 268 deletions(-) delete mode 100644 cmd/pull/terra_tdr_integration_test.go delete mode 100644 tests/terra-tdr/README.md delete mode 100755 tests/terra-tdr/run.sh diff --git a/.github/workflows/pr-checks.yaml b/.github/workflows/pr-checks.yaml index a92d4f73..2a58ab2c 100644 --- a/.github/workflows/pr-checks.yaml +++ b/.github/workflows/pr-checks.yaml @@ -76,39 +76,6 @@ jobs: - name: Run tests run: go test -v -race ./... - tdr-controller: - name: Pull through TDR controller - runs-on: ubuntu-latest - timeout-minutes: 30 - - steps: - - name: Check out git-drs - uses: actions/checkout@v4 - - - name: Check out TDR - uses: actions/checkout@v4 - with: - repository: DataBiosphere/jade-data-repo - ref: 6433cb4ea296edf873daa30f1da29e7caf848b8b - path: tdr - persist-credentials: false - - - name: Set up Java - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: '17' - cache: gradle - cache-dependency-path: tdr/build.gradle - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.26.6' - - - name: Pull an AnVIL DRS pointer through TDR's controller - run: tests/terra-tdr/run.sh "$GITHUB_WORKSPACE/tdr" - terra-drs-hub: name: Resolve through DRS Hub and TDR runs-on: ubuntu-latest diff --git a/cmd/pull/terra_tdr_integration_test.go b/cmd/pull/terra_tdr_integration_test.go deleted file mode 100644 index 1b177996..00000000 --- a/cmd/pull/terra_tdr_integration_test.go +++ /dev/null @@ -1,127 +0,0 @@ -//go:build integration - -package pull - -import ( - "bytes" - "context" - "fmt" - "net/http" - "net/http/httptest" - "net/http/httputil" - "net/url" - "os" - "path/filepath" - "reflect" - "strings" - "sync" - "testing" - - "github.com/calypr/git-drs/internal/config" - "github.com/calypr/git-drs/internal/lfs" - "github.com/calypr/git-drs/internal/resolver" -) - -func TestIntegrationPullAgainstTDRController(t *testing.T) { - endpoint := os.Getenv("GIT_DRS_TDR_HTTP_ENDPOINT") - gcsEndpoint := os.Getenv("GIT_DRS_TDR_GCS_ENDPOINT") - keyPath := os.Getenv("GIT_DRS_TDR_SIGNING_PUBLIC_KEY_FILE") - if endpoint == "" || gcsEndpoint == "" || keyPath == "" { - t.Skip("GIT_DRS_TDR_HTTP_ENDPOINT, GIT_DRS_TDR_GCS_ENDPOINT, and GIT_DRS_TDR_SIGNING_PUBLIC_KEY_FILE are required") - } - publicKeyPEM, err := os.ReadFile(keyPath) - if err != nil { - t.Fatalf("read TDR signing public key: %v", err) - } - tdrURL, err := url.Parse(endpoint) - if err != nil || tdrURL.Scheme != "http" || tdrURL.Host == "" { - t.Fatalf("invalid local TDR endpoint %q", endpoint) - } - - var requestMu sync.Mutex - var requestPaths []string - reverseProxy := httputil.NewSingleHostReverseProxy(tdrURL) - reverseProxy.ModifyResponse = func(response *http.Response) error { - if strings.Contains(response.Request.URL.Path, "/access/") { - return rewriteTDRSignedURLForEmulator(response, gcsEndpoint, publicKeyPEM) - } - return nil - } - proxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requestMu.Lock() - requestPaths = append(requestPaths, r.URL.Path) - requestMu.Unlock() - reverseProxy.ServeHTTP(w, r) - })) - t.Cleanup(proxy.Close) - oldResolver := newAnVILResolver - t.Cleanup(func() { - newAnVILResolver = oldResolver - resetPullFlagsForTest() - }) - newAnVILResolver = func(_ context.Context, endpoint string) (resolver.Resolver, error) { - client := proxy.Client() - base := client.Transport - client.Transport = pullRoundTripFunc(func(req *http.Request) (*http.Response, error) { - authorized := req.Clone(req.Context()) - authorized.Header.Set("Authorization", "Bearer tdr-ci-fixture") - return base.RoundTrip(authorized) - }) - return resolver.NewAnVILWithClient(endpoint, client) - } - - repo := t.TempDir() - runGitCmdTest(t, repo, "init", "-q") - t.Chdir(repo) - const filename = "1614321.merge_output.gvcf.gz" - const oid = "drs://drs.anv0:v2_4f770147-e372-339b-b9fa-0a7a83cf30cf" - payload := []byte("git-drs Terra DRS HTTP integration fixture\n") - if err := os.WriteFile(".gitattributes", []byte(filename+" filter=drs diff=drs merge=drs -text\n"), 0o644); err != nil { - t.Fatal(err) - } - pointer := fmt.Sprintf("version https://calypr.github.io/spec/v1\noid %s\nsize %d\n", oid, len(payload)) - if err := os.WriteFile(filename, []byte(pointer), 0o644); err != nil { - t.Fatal(err) - } - runGitCmdTest(t, repo, "add", filename) - if _, err := config.UpdateRemote("anvil", config.RemoteSelect{Terra: &config.TerraRemote{Endpoint: proxy.URL, Mode: "read-only"}}); err != nil { - t.Fatal(err) - } - objectsRoot, err := lfs.ResolveObjectsRoot(context.Background()) - if err != nil { - t.Fatal(err) - } - cachePath, err := lfs.ObjectPath(objectsRoot, oid) - if err != nil { - t.Fatal(err) - } - if _, err := os.Stat(cachePath); !os.IsNotExist(err) { - t.Fatalf("cache must be empty before pull: %s: %v", cachePath, err) - } - resetPullFlagsForTest() - includePatterns = []string{filename} - requestMu.Lock() - requestPaths = nil - requestMu.Unlock() - - if err := Cmd.RunE(Cmd, nil); err != nil { - t.Fatalf("pull through TDR controller: %v", err) - } - got, err := os.ReadFile(filepath.Join(repo, filename)) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(got, payload) { - t.Fatalf("hydrated content = %q, want %q", got, payload) - } - requestMu.Lock() - gotPaths := append([]string(nil), requestPaths...) - requestMu.Unlock() - wantPaths := []string{ - "/ga4gh/drs/v1/objects/v2_4f770147-e372-339b-b9fa-0a7a83cf30cf", - "/ga4gh/drs/v1/objects/v2_4f770147-e372-339b-b9fa-0a7a83cf30cf/access/gcp-us-central1*11111111-2222-4333-8444-555555555555", - } - if !reflect.DeepEqual(gotPaths, wantPaths) { - t.Fatalf("TDR controller requests = %v, want %v", gotPaths, wantPaths) - } -} diff --git a/tests/terra-tdr/README.md b/tests/terra-tdr/README.md deleted file mode 100644 index 6f891eb2..00000000 --- a/tests/terra-tdr/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# TDR pull integration - -`run.sh` starts TDR's real DRS controller, generated API routes, and `DrsService`, then runs the `git-drs pull` integration test against that server. TDR stores the compact DRS ID mapping and the snapshot, source, project, and dataset records in an ephemeral PostgreSQL Testcontainer. The real `SnapshotService`, `SnapshotDao`, `DatasetDao`, `SnapshotTableDao`, `DatasetTableDao`, `DatasetRelationshipDao`, `AssetDao`, and `StorageResourceDao` load that metadata through TDR's production SQL. The fixture has a GCP Firestore storage resource, but no tables, relationships, or assets. Project resource lookup remains fixture-backed. TDR reads the seeded file record through its real `FileService` and `FireStoreDao` against the Firestore emulator. A GCS-compatible emulator serves the fixture object over HTTP. - -Authorization remains fixture-backed. TDR uses the Google Cloud Storage Java client to look up the bucket and create a V4 signed URL with a generated test-only service-account key. A local token endpoint supplies its access token; no Google credentials are used. The test verifies the URL's RSA signature and expiry using the generated public key before changing its authority to point at the local GCS emulator. The emulator serves the bytes but does not validate signed URLs itself. - -The pull starts with an empty cache, checks the hydrated bytes and TDR-provided checksums, and the runner asserts that the emulator served exactly one object download. - -Run locally with Java 17, Go, and Docker installed: - -```bash -tests/terra-tdr/run.sh /path/to/jade-data-repo -``` - -The CI workflow pins the TDR source revision. The test covers TDR's controller and `DrsService`, PostgreSQL DRS/snapshot/dataset lookup, Firestore file lookup, GCS client calls, the returned signed URL shape, and the CLI download path. It does not exercise Sam authorization or Google project resource lookup; the local token endpoint supplies the generated test identity's access token without validating Google-issued credentials. diff --git a/tests/terra-tdr/run.sh b/tests/terra-tdr/run.sh deleted file mode 100755 index 053ab555..00000000 --- a/tests/terra-tdr/run.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [[ $# -ne 1 ]]; then - echo "usage: $0 /path/to/jade-data-repo" >&2 - exit 2 -fi - -git_drs_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) -tdr_root=$(cd "$1" && pwd) -coord_dir=$(mktemp -d) -port_file="$coord_dir/port" -stop_file="$coord_dir/stop" -tdr_log="$coord_dir/tdr.log" -signing_public_key_file="$coord_dir/tdr-signing-public-key.pem" - -source "$git_drs_root/tests/terra-tdr/firestore-emulator.sh" -source "$git_drs_root/tests/terra-tdr/storage-emulator.sh" -FIRESTORE_CONTAINER= -STORAGE_CONTAINER= -start_firestore_emulator -start_storage_emulator "$coord_dir/gcs-seed" - -cp "$git_drs_root/tests/terra-tdr/TerraDrsHarnessTest.java" \ - "$tdr_root/src/test/java/bio/terra/service/filedata/TerraDrsHarnessTest.java" - -( - cd "$tdr_root" - GIT_DRS_TDR_PORT_FILE="$port_file" GIT_DRS_TDR_STOP_FILE="$stop_file" \ - GIT_DRS_TDR_SIGNING_PUBLIC_KEY_FILE="$signing_public_key_file" \ - FIRESTORE_EMULATOR_HOST="$FIRESTORE_EMULATOR_HOST" \ - GIT_DRS_TDR_GCS_ENDPOINT="$STORAGE_EMULATOR_HTTP_ENDPOINT" \ - ./gradlew testUnit --tests bio.terra.service.filedata.TerraDrsHarnessTest \ - --no-daemon --console=plain --rerun-tasks -) >"$tdr_log" 2>&1 & -tdr_pid=$! - -stop_services() { - touch "$stop_file" - if [[ -n "${tdr_pid:-}" ]]; then - wait "$tdr_pid" || true - fi - stop_firestore_emulator - stop_storage_emulator -} -trap stop_services EXIT - -for ((seconds=0; seconds<900; seconds++)); do - if [[ -s "$port_file" ]]; then - break - fi - if ! kill -0 "$tdr_pid" 2>/dev/null; then - cat "$tdr_log" >&2 - exit 1 - fi - sleep 1 -done - -if [[ ! -s "$port_file" ]]; then - echo "TDR controller did not start within 15 minutes" >&2 - cat "$tdr_log" >&2 - exit 1 -fi - -port=$(cat "$port_file") -cd "$git_drs_root" -if ! GIT_DRS_TDR_HTTP_ENDPOINT="http://127.0.0.1:$port" \ - GIT_DRS_TDR_GCS_ENDPOINT="$STORAGE_EMULATOR_HTTP_ENDPOINT" \ - GIT_DRS_TDR_SIGNING_PUBLIC_KEY_FILE="$signing_public_key_file" \ - go test -race -tags=integration -count=1 ./cmd/pull \ - -run '^TestIntegrationPullAgainstTDRController$' -v; then - cat "$tdr_log" >&2 - exit 1 -fi - -downloads=$(storage_object_download_count) -if [[ "$downloads" != 1 ]]; then - echo "GCS emulator served $downloads object downloads, want exactly one" >&2 - docker logs "$STORAGE_CONTAINER" >&2 || true - exit 1 -fi - -touch "$stop_file" -if ! wait "$tdr_pid"; then - cat "$tdr_log" >&2 - exit 1 -fi -tdr_pid= -stop_firestore_emulator -stop_storage_emulator -trap - EXIT - -tail -n 20 "$tdr_log" From 28a372aea9e1090a02d74d7bd1148239623e2d53 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 8 Oct 2026 12:21:28 -0700 Subject: [PATCH 17/22] Cover GREGoR-style Terra reference and hydration workflows --- cmd/pull/terra_hub_integration_test.go | 525 +++++++++++++++++++---- tests/terra-drs-hub/README.md | 6 +- tests/terra-drs-hub/run.sh | 5 +- tests/terra-tdr/TerraDrsHarnessTest.java | 111 +++-- tests/terra-tdr/storage-emulator.sh | 11 +- 5 files changed, 533 insertions(+), 125 deletions(-) diff --git a/cmd/pull/terra_hub_integration_test.go b/cmd/pull/terra_hub_integration_test.go index 4f972e0f..0b5fa0cb 100644 --- a/cmd/pull/terra_hub_integration_test.go +++ b/cmd/pull/terra_hub_integration_test.go @@ -27,6 +27,7 @@ import ( "os/exec" "path/filepath" "strings" + "sync" "sync/atomic" "testing" "time" @@ -35,13 +36,30 @@ import ( ) func TestIntegrationPullThroughTerraHubAndTDR(t *testing.T) { - payload := []byte("git-drs Terra DRS HTTP integration fixture\n") + fixtures := []terraCLIObject{ + { + filename: "1614321.merge_output.gvcf.gz", + oid: "drs://drs.anv0:v2_4f770147-e372-339b-b9fa-0a7a83cf30cf", + payload: []byte("git-drs Terra DRS HTTP integration fixture\n"), + }, + { + filename: "gregor-1614321.merge_output.gvcf.gz", + oid: "drs://drs.anv0:v2_c5ae75de-1f5c-3d40-bcd9-02f827fbf2d3", + payload: []byte("synthetic GREGoR 1614321 fixture; not genomic data\n"), + }, + { + filename: "1614322.merge_output.gvcf.gz", + oid: "drs://drs.anv0:v2_4872d195-f80a-3662-90a7-2aa90e0a7f53", + payload: []byte("synthetic GREGoR 1614322 fixture; not genomic data\n"), + }, + } binary := os.Getenv("GIT_DRS_BINARY") hubEndpoint := os.Getenv("GIT_DRS_HUB_HTTP_ENDPOINT") proxyLog := os.Getenv("GIT_DRS_TDR_PROXY_LOG") gcsEndpoint := os.Getenv("GIT_DRS_TDR_GCS_ENDPOINT") keyPath := os.Getenv("GIT_DRS_TDR_SIGNING_PUBLIC_KEY_FILE") - if binary == "" || hubEndpoint == "" || proxyLog == "" || gcsEndpoint == "" || keyPath == "" { + tdrHTTPEndpoint := os.Getenv("GIT_DRS_TDR_HTTP_ENDPOINT") + if binary == "" || hubEndpoint == "" || proxyLog == "" || gcsEndpoint == "" || keyPath == "" || tdrHTTPEndpoint == "" { t.Skip("GIT_DRS_BINARY and the local Hub/TDR/GCS fixture endpoints are required") } publicKeyPEM, err := os.ReadFile(keyPath) @@ -64,7 +82,7 @@ func TestIntegrationPullThroughTerraHubAndTDR(t *testing.T) { } tokenEndpoint := fmt.Sprintf("http://host.docker.internal:%d", tokenListener.Addr().(*net.TCPAddr).Port) tokenServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost || r.URL.Path != "/token" { + if r.Method != http.MethodPost || (r.URL.Path != "/token" && r.URL.Path != "/bad-token") { http.Error(w, "unexpected OAuth request", http.StatusNotFound) return } @@ -91,7 +109,7 @@ func TestIntegrationPullThroughTerraHubAndTDR(t *testing.T) { Audience string `json:"aud"` Scope string `json:"scope"` } - if err := json.Unmarshal(claimsBytes, &claims); err != nil || claims.Issuer != "git-drs-ci@example.invalid" || claims.Audience != tokenEndpoint+"/token" || !strings.Contains(claims.Scope, "cloud-platform") { + if err := json.Unmarshal(claimsBytes, &claims); err != nil || claims.Issuer != "git-drs-ci@example.invalid" || claims.Audience != tokenEndpoint+r.URL.Path || !strings.Contains(claims.Scope, "cloud-platform") { http.Error(w, "unexpected service-account claims", http.StatusBadRequest) return } @@ -107,13 +125,20 @@ func TestIntegrationPullThroughTerraHubAndTDR(t *testing.T) { } tokenRequests.Add(1) w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"access_token":"tdr-ci-fixture","token_type":"Bearer","expires_in":3600}`) + accessToken := "tdr-ci-fixture" + if r.URL.Path == "/bad-token" { + accessToken = "tdr-ci-invalid" + } + _, _ = fmt.Fprintf(w, `{"access_token":%q,"token_type":"Bearer","expires_in":3600}`, accessToken) })) tokenServer.Listener = tokenListener tokenServer.Start() defer tokenServer.Close() var resolveRequests atomic.Int32 + var serviceInfoRequests atomic.Int32 + var resolvedFilesMu sync.Mutex + resolvedFiles := make(map[string]int) var hubBearer atomic.Value frontListener, err := net.Listen("tcp", "0.0.0.0:0") if err != nil { @@ -124,13 +149,22 @@ func TestIntegrationPullThroughTerraHubAndTDR(t *testing.T) { t.Fatalf("make local Hub TLS certificate: %v", err) } frontProxy := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - forwarded, err := http.NewRequestWithContext(r.Context(), r.Method, "http://"+upstream.Host+r.URL.RequestURI(), r.Body) + destination := "http://" + upstream.Host + r.URL.RequestURI() + if r.URL.Path == "/ga4gh/drs/v1/service-info" { + destination = strings.TrimRight(tdrHTTPEndpoint, "/") + r.URL.RequestURI() + serviceInfoRequests.Add(1) + } + forwarded, err := http.NewRequestWithContext(r.Context(), r.Method, destination, r.Body) if err != nil { http.Error(w, err.Error(), http.StatusBadGateway) return } forwarded.Header = r.Header.Clone() - forwarded.Host = upstream.Host + if r.URL.Path == "/ga4gh/drs/v1/service-info" { + forwarded.Host = "127.0.0.1" + } else { + forwarded.Host = upstream.Host + } response, err := http.DefaultTransport.RoundTrip(forwarded) if err != nil { http.Error(w, err.Error(), http.StatusBadGateway) @@ -140,13 +174,19 @@ func TestIntegrationPullThroughTerraHubAndTDR(t *testing.T) { if r.URL.Path == "/api/v4/drs/resolve" { resolveRequests.Add(1) hubBearer.Store(r.Header.Get("Authorization")) - if err := verifyTerraHubMetadata(response, payload); err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return - } - if err := rewriteTDRSignedURLForEmulator(response, gcsEndpoint, publicKeyPEM); err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return + if response.StatusCode == http.StatusOK { + filename, err := verifyTerraHubMetadata(response, fixtures) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + resolvedFilesMu.Lock() + resolvedFiles[filename]++ + resolvedFilesMu.Unlock() + if err := rewriteTDRSignedURLForEmulator(response, gcsEndpoint, publicKeyPEM); err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } } } for name, values := range response.Header { @@ -165,6 +205,7 @@ func TestIntegrationPullThroughTerraHubAndTDR(t *testing.T) { frontProxy.StartTLS() defer frontProxy.Close() frontProxyURL := fmt.Sprintf("https://host.docker.internal:%d", frontListener.Addr().(*net.TCPAddr).Port) + terraServiceEndpoint := fmt.Sprintf("https://data.terra.bio:%d", frontListener.Addr().(*net.TCPAddr).Port) credentialsPath := filepath.Join(t.TempDir(), "service-account.json") privateKeyDER, err := x509.MarshalPKCS8PrivateKey(key) @@ -192,41 +233,51 @@ func TestIntegrationPullThroughTerraHubAndTDR(t *testing.T) { runGitCmdTest(t, repo, "init", "-q") runGitCmdTest(t, repo, "config", "user.name", "git-drs CI fixture") runGitCmdTest(t, repo, "config", "user.email", "git-drs-ci@example.invalid") - runGitCmdTest(t, repo, "config", "drs.default-remote", "anvil") - runGitCmdTest(t, repo, "config", "drs.remote.anvil.type", "terra") - runGitCmdTest(t, repo, "config", "drs.remote.anvil.endpoint", "https://data.terra.bio") - runGitCmdTest(t, repo, "config", "drs.remote.anvil.hub-endpoint", frontProxyURL) - runGitCmdTest(t, repo, "config", "drs.remote.anvil.mode", "read-only") - const filename = "1614321.merge_output.gvcf.gz" - const oid = "drs://drs.anv0:v2_4f770147-e372-339b-b9fa-0a7a83cf30cf" - pointer := fmt.Sprintf("version https://calypr.github.io/spec/v1\noid %s\nsize %d\n", oid, len(payload)) - if err := os.WriteFile(filepath.Join(repo, ".gitattributes"), []byte(filename+" filter=drs diff=drs merge=drs -text\n"), 0o644); err != nil { - t.Fatal(err) + var pointers = make(map[string]string, len(fixtures)) + attributes := make([]string, 0, len(fixtures)) + for _, fixture := range fixtures { + pointer := fmt.Sprintf("version https://calypr.github.io/spec/v1\noid %s\nsize %d\n", fixture.oid, len(fixture.payload)) + pointers[fixture.filename] = pointer + attributes = append(attributes, fixture.filename+" filter=drs diff=drs merge=drs -text") + if err := os.WriteFile(filepath.Join(repo, fixture.filename), []byte(pointer), 0o644); err != nil { + t.Fatal(err) + } } - if err := os.WriteFile(filepath.Join(repo, filename), []byte(pointer), 0o644); err != nil { + if err := os.WriteFile(filepath.Join(repo, ".gitattributes"), []byte(strings.Join(attributes, "\n")+"\n"), 0o644); err != nil { t.Fatal(err) } - runGitCmdTest(t, repo, "add", filename) + for _, fixture := range fixtures { + runGitCmdTest(t, repo, "add", fixture.filename) + } t.Chdir(repo) objectsRoot, err := lfs.ResolveObjectsRoot(context.Background()) if err != nil { t.Fatal(err) } - cachePath, err := lfs.ObjectPath(objectsRoot, oid) - if err != nil { - t.Fatal(err) - } - if _, err := os.Stat(cachePath); !os.IsNotExist(err) { - t.Fatalf("cache must be empty before pull: %s: %v", cachePath, err) + cachePaths := make(map[string]string, len(fixtures)) + for _, fixture := range fixtures { + cachePath, err := lfs.ObjectPath(objectsRoot, fixture.oid) + if err != nil { + t.Fatal(err) + } + cachePaths[fixture.filename] = cachePath + if _, err := os.Stat(cachePath); !os.IsNotExist(err) { + t.Fatalf("cache must be empty before pull: %s: %v", cachePath, err) + } } - runPull := func() []byte { + runContainer := func(credentialsFile, executable string, arguments ...string) ([]byte, error) { + commandLine := `mkdir -p /tmp/git-drs-ci-home && git config --global --add safe.directory /repo && git config --global filter.drs.clean "/usr/local/bin/git-drs clean -- %f" && git config --global filter.drs.smudge "/usr/local/bin/git-drs smudge -- %f" && git config --global filter.drs.process "/usr/local/bin/git-drs filter" && git config --global filter.drs.required true && exec ` + shellQuote(executable) + for _, argument := range arguments { + commandLine += " " + shellQuote(argument) + } command := exec.Command("docker", "run", "--rm", "--add-host", "host.docker.internal:host-gateway", + "--add-host", "data.terra.bio:host-gateway", "--user", fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()), "--volume", binary+":/usr/local/bin/git-drs:ro", "--volume", repo+":/repo", - "--volume", credentialsPath+":/ci/service-account.json:ro", + "--volume", credentialsFile+":/ci/service-account.json:ro", "--volume", certPath+":/ci/hub-proxy-ca.pem:ro", "--workdir", "/repo", "--env", "HOME=/tmp/git-drs-ci-home", @@ -234,39 +285,89 @@ func TestIntegrationPullThroughTerraHubAndTDR(t *testing.T) { "--env", "GOOGLE_APPLICATION_CREDENTIALS=/ci/service-account.json", "--env", "SSL_CERT_FILE=/ci/hub-proxy-ca.pem", "--entrypoint", "/bin/sh", "alpine/git@sha256:062a01ad7a0eb17cff382bc5e26086b4d710e56dfdfdf001109a49b6d9bd378c", "-c", - "mkdir -p /tmp/git-drs-ci-home && git config --global --add safe.directory /repo && git config --global filter.drs.clean \"/usr/local/bin/git-drs clean -- %f\" && git config --global filter.drs.smudge \"/usr/local/bin/git-drs smudge -- %f\" && git config --global filter.drs.process \"/usr/local/bin/git-drs filter\" && git config --global filter.drs.required true && exec /usr/local/bin/git-drs pull -I "+filename) - output, err := command.CombinedOutput() + commandLine) + return command.CombinedOutput() + } + runCLIResult := func(credentialsFile string, arguments ...string) ([]byte, error) { + return runContainer(credentialsFile, "/usr/local/bin/git-drs", arguments...) + } + runCLI := func(arguments ...string) []byte { + output, err := runCLIResult(credentialsPath, arguments...) if err != nil { - t.Fatalf("git-drs pull exited with %v:\n%s", err, output) + t.Fatalf("git-drs %s exited with %v:\n%s", strings.Join(arguments, " "), err, output) } return output } - output := runPull() - if got, err := os.ReadFile(filepath.Join(repo, filename)); err != nil || !bytes.Equal(got, payload) { - t.Fatalf("CLI worktree bytes = %q, want %q (error %v)\n%s", got, payload, err, output) + runGit := func(arguments ...string) []byte { + output, err := runContainer(credentialsPath, "git", arguments...) + if err != nil { + t.Fatalf("git %s exited with %v:\n%s", strings.Join(arguments, " "), err, output) + } + return output + } + remoteOutput := runCLI("remote", "add", "anvil", terraServiceEndpoint, "--provider", "terra", "--auth", "google-adc", "--hub-endpoint", frontProxyURL) + if !strings.Contains(string(remoteOutput), "Remote saved.") { + t.Fatalf("remote add output does not confirm configuration:\n%s", remoteOutput) + } + runCLI("remote", "set", "anvil") + remoteList := runCLI("remote", "list") + if !strings.Contains(string(remoteList), "* anvil terra") || !strings.Contains(string(remoteList), terraServiceEndpoint) { + t.Fatalf("remote list does not report the selected Terra remote:\n%s", remoteList) + } + pingOutput := runCLI("ping", "anvil") + if !strings.Contains(string(pingOutput), "health: ok") || !strings.Contains(string(pingOutput), "service-info:") { + t.Fatalf("Terra ping did not report service health and service-info:\n%s", pingOutput) + } + if got := serviceInfoRequests.Load(); got != 1 { + t.Fatalf("Terra ping service-info requests = %d, want one real TDR request", got) + } + selective := runCLI("pull", "-I", fixtures[1].filename) + assertFileBytes(t, repo, fixtures[1], selective) + if _, err := os.Stat(cachePaths[fixtures[0].filename]); !os.IsNotExist(err) { + t.Fatalf("selective pull populated an unselected cache object: %v", err) } if got := tokenRequests.Load(); got != 1 { - t.Fatalf("ADC token exchanges = %d, want one", got) + t.Fatalf("ADC token exchanges after selective pull = %d, want one", got) } + assertResolvedCounts(t, &resolvedFilesMu, resolvedFiles, map[string]int{fixtures[1].filename: 1}) if got := resolveRequests.Load(); got != 1 { - t.Fatalf("Hub resolve requests = %d, want one", got) + t.Fatalf("Hub resolve requests after selective pull = %d, want one", got) } if got := hubBearer.Load(); got != "Bearer tdr-ci-fixture" { t.Fatalf("Hub received Authorization %v, want ADC fixture token", got) } - if got := runGitOutputTest(t, repo, "show", ":"+filename); got != pointer { - t.Fatalf("staged index bytes = %q, want original pointer %q", got, pointer) - } - cached, err := os.ReadFile(cachePath) - if err != nil || !bytes.Equal(cached, payload) { - t.Fatalf("cache bytes do not match fixture: %d bytes, %v", len(cached), err) + assertStagedPointers(t, repo, fixtures, pointers) + assertCachedObject(t, cachePaths[fixtures[1].filename], fixtures[1]) + for _, unselected := range []terraCLIObject{fixtures[0], fixtures[2]} { + if _, err := os.Stat(cachePaths[unselected.filename]); !os.IsNotExist(err) { + t.Fatalf("selective pull unexpectedly cached %s: %v", unselected.filename, err) + } } + + allPull := runCLI("pull") + for _, fixture := range fixtures { + assertFileBytes(t, repo, fixture, allPull) + } + assertStagedPointers(t, repo, fixtures, pointers) + assertCachedObjects(t, cachePaths, fixtures) + if got := tokenRequests.Load(); got != 2 { + t.Fatalf("ADC token exchanges after all-files pull = %d, want two total", got) + } + if got := resolveRequests.Load(); got != 3 { + t.Fatalf("Hub resolve requests after all-files pull = %d, want three total", got) + } + assertResolvedCounts(t, &resolvedFilesMu, resolvedFiles, map[string]int{ + fixtures[1].filename: 1, + fixtures[0].filename: 1, + fixtures[2].filename: 1, + }) proxyLogBytes, err := os.ReadFile(proxyLog) if err != nil { t.Fatal(err) } - var metadataRequests, accessRequests int - var unauthenticatedProbe bool + metadataRequests := make(map[string]int) + accessRequests := make(map[string]int) + probes := make(map[string]int) for _, line := range bytes.Split(bytes.TrimSpace(proxyLogBytes), []byte("\n")) { var request struct { Method string `json:"method"` @@ -277,39 +378,194 @@ func TestIntegrationPullThroughTerraHubAndTDR(t *testing.T) { if err := json.Unmarshal(line, &request); err != nil { t.Fatalf("invalid TDR proxy log line %q: %v", line, err) } - if request.Method == "OPTIONS" && strings.HasSuffix(request.Path, "/objects/v2_4f770147-e372-339b-b9fa-0a7a83cf30cf") && request.Authorization == "" && request.Status == http.StatusOK { - unauthenticatedProbe = true + objectID := fixtureIDForProxyPath(request.Path, fixtures) + if objectID == "" { + t.Fatalf("unexpected TDR request path in proxy log: %s", line) + } + if request.Method == "OPTIONS" { + if request.Authorization != "" || request.Status != http.StatusOK { + t.Fatalf("invalid unauthenticated TDR OPTIONS request: %s", line) + } + probes[objectID]++ } - if request.Method == "GET" && strings.HasSuffix(request.Path, "/objects/v2_4f770147-e372-339b-b9fa-0a7a83cf30cf") { + if request.Method == "GET" && !strings.Contains(request.Path, "/access/") { if request.Authorization != "Bearer tdr-ci-fixture" || request.Status != http.StatusOK { t.Fatalf("invalid TDR metadata request: %s", line) } - metadataRequests++ + metadataRequests[objectID]++ } - if request.Method == "GET" && strings.Contains(request.Path, "/objects/v2_4f770147-e372-339b-b9fa-0a7a83cf30cf/access/") { + if request.Method == "GET" && strings.Contains(request.Path, "/access/") { if request.Authorization != "Bearer tdr-ci-fixture" || request.Status != http.StatusOK { t.Fatalf("invalid TDR access request: %s", line) } - accessRequests++ + accessRequests[objectID]++ } } - if !unauthenticatedProbe || metadataRequests != 1 || accessRequests != 1 { - t.Fatalf("Hub did not complete the unauthenticated probe and authenticated TDR requests: %s", proxyLogBytes) + for _, fixture := range fixtures { + objectID := fixtureObjectID(fixture) + if probes[objectID] != 1 || metadataRequests[objectID] != 1 || accessRequests[objectID] != 1 { + t.Fatalf("TDR request counts for %s = OPTIONS:%d metadata:%d access:%d, want one each; log:\n%s", objectID, probes[objectID], metadataRequests[objectID], accessRequests[objectID], proxyLogBytes) + } } - secondOutput := runPull() - if got, err := os.ReadFile(filepath.Join(repo, filename)); err != nil || !bytes.Equal(got, payload) { - t.Fatalf("cached pull changed worktree file: %d bytes, %v\n%s", len(got), err, secondOutput) + // Git checkout reads the staged pointer through the configured process filter. + if err := os.Remove(filepath.Join(repo, fixtures[1].filename)); err != nil { + t.Fatal(err) } - if got := tokenRequests.Load(); got != 1 { - t.Fatalf("ADC token exchanges after cached pull = %d, want one", got) + checkout := runGit("checkout", "--", fixtures[1].filename) + assertFileBytes(t, repo, fixtures[1], checkout) + secondOutput := runCLI("pull") + for _, fixture := range fixtures { + assertFileBytes(t, repo, fixture, secondOutput) } - if got := resolveRequests.Load(); got != 1 { - t.Fatalf("Hub resolve requests after cached pull = %d, want one", got) + if got := tokenRequests.Load(); got != 2 { + t.Fatalf("ADC token exchanges after cache-only pull/smudge = %d, want two", got) + } + if got := resolveRequests.Load(); got != 3 { + t.Fatalf("Hub resolve requests after cache-only pull/smudge = %d, want three", got) } secondProxyLog, err := os.ReadFile(proxyLog) if err != nil || !bytes.Equal(secondProxyLog, proxyLogBytes) { - t.Fatalf("cached pull contacted TDR: before %d bytes, after %d bytes (error %v)", len(proxyLogBytes), len(secondProxyLog), err) + t.Fatalf("cached pull or checkout smudge contacted TDR: before %d bytes, after %d bytes (error %v)", len(proxyLogBytes), len(secondProxyLog), err) + } + + manifest := "drs_uri\tpath\n" + fixtures[1].oid + "\trefs/gregor-1614321-copy.gz\n" + fixtures[2].oid + "\trefs/gregor-1614322-copy.gz\n" + if err := os.WriteFile(filepath.Join(repo, "terra-references.tsv"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + addRefs := runCLI("add-ref", "--manifest", "terra-references.tsv") + if !strings.Contains(string(addRefs), "2 reference(s) added") && !strings.Contains(string(addRefs), "2 reference(s) validated") { + t.Fatalf("add-ref manifest output does not report both refs:\n%s", addRefs) + } + for i, path := range []string{"refs/gregor-1614321-copy.gz", "refs/gregor-1614322-copy.gz"} { + wantPointer := pointers[fixtures[i+1].filename] + worktreePointer, err := os.ReadFile(filepath.Join(repo, path)) + if err != nil || string(worktreePointer) != wantPointer { + t.Fatalf("add-ref worktree pointer for %s = %q, want %q (error %v)", path, worktreePointer, wantPointer, err) + } + runGit("add", path) + got := runGitOutputTest(t, repo, "show", ":"+path) + if got != wantPointer { + t.Fatalf("staged add-ref pointer for %s = %q, want %q", path, got, wantPointer) + } + } + singleRefPath := "refs/single-1614321-copy.gz" + singleRef := runCLI("add-ref", fixtures[0].oid, singleRefPath) + if len(strings.TrimSpace(string(singleRef))) != 0 { + t.Fatalf("single add-ref returned unexpected output:\n%s", singleRef) + } + singleWorktreePointer, err := os.ReadFile(filepath.Join(repo, singleRefPath)) + if err != nil || string(singleWorktreePointer) != pointers[fixtures[0].filename] { + t.Fatalf("single add-ref worktree pointer = %q, want %q (error %v)", singleWorktreePointer, pointers[fixtures[0].filename], err) + } + runGit("add", singleRefPath) + if got := runGitOutputTest(t, repo, "show", ":"+singleRefPath); got != pointers[fixtures[0].filename] { + t.Fatalf("single add-ref staged pointer = %q, want %q", got, pointers[fixtures[0].filename]) + } + if got := tokenRequests.Load(); got != 5 { + t.Fatalf("ADC token exchanges after add-ref operations = %d, want five total (one per resolver)", got) + } + if got := resolveRequests.Load(); got != 6 { + t.Fatalf("Hub resolve requests after add-ref operations = %d, want six total", got) + } + assertResolvedCounts(t, &resolvedFilesMu, resolvedFiles, map[string]int{ + fixtures[1].filename: 2, + fixtures[0].filename: 2, + fixtures[2].filename: 2, + }) + finalProxyLog, err := os.ReadFile(proxyLog) + if err != nil { + t.Fatal(err) + } + finalMetadata, finalAccess, finalProbes := countTDRRequests(t, finalProxyLog, fixtures) + for _, fixture := range fixtures { + objectID := fixtureObjectID(fixture) + want := 2 + if finalMetadata[objectID] != want || finalAccess[objectID] != want || finalProbes[objectID] != want { + t.Fatalf("final TDR request counts for %s = OPTIONS:%d metadata:%d access:%d, want %d each", objectID, finalProbes[objectID], finalMetadata[objectID], finalAccess[objectID], want) + } + } + + var badCredentialDocument map[string]string + if err := json.Unmarshal(credentials, &badCredentialDocument); err != nil { + t.Fatal(err) + } + badCredentialDocument["token_uri"] = tokenEndpoint + "/bad-token" + badCredentials, err := json.Marshal(badCredentialDocument) + if err != nil { + t.Fatal(err) + } + badCredentialsPath := filepath.Join(t.TempDir(), "invalid-service-account.json") + if err := os.WriteFile(badCredentialsPath, badCredentials, 0o600); err != nil { + t.Fatal(err) + } + + // Remove the already verified object so this request must authenticate and resolve remotely. + if err := os.WriteFile(filepath.Join(repo, fixtures[0].filename), []byte(pointers[fixtures[0].filename]), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Remove(cachePaths[fixtures[0].filename]); err != nil { + t.Fatalf("remove verified cache object before unauthorized pull: %v", err) + } + badOutput, badErr := runCLIResult(badCredentialsPath, "pull", "-I", fixtures[0].filename) + if badErr == nil { + t.Fatalf("pull with an invalid fixture bearer token unexpectedly succeeded:\n%s", badOutput) + } + if lower := strings.ToLower(string(badOutput)); !strings.Contains(lower, "401") && !strings.Contains(lower, "unauthorized") && !strings.Contains(lower, "not authorized") && !strings.Contains(lower, "invalid fixture bearer token") { + t.Fatalf("unauthorized pull did not report the authentication failure clearly (exit %v):\n%s", badErr, badOutput) + } + if got, err := os.ReadFile(filepath.Join(repo, fixtures[0].filename)); err != nil || string(got) != pointers[fixtures[0].filename] { + t.Fatalf("unauthorized pull changed worktree bytes: got %q, want pointer %q (error %v)", got, pointers[fixtures[0].filename], err) + } + if _, err := os.Stat(cachePaths[fixtures[0].filename]); !os.IsNotExist(err) { + t.Fatalf("unauthorized pull wrote an object to cache: %v", err) + } + if got := runGitOutputTest(t, repo, "show", ":"+fixtures[0].filename); got != pointers[fixtures[0].filename] { + t.Fatalf("unauthorized pull changed staged pointer: got %q, want %q", got, pointers[fixtures[0].filename]) + } + if got := tokenRequests.Load(); got != 6 { + t.Fatalf("ADC exchanges after unauthorized pull = %d, want six total", got) + } + if got := resolveRequests.Load(); got != 7 { + t.Fatalf("Hub resolve requests after unauthorized pull = %d, want seven total", got) + } + if got := hubBearer.Load(); got != "Bearer tdr-ci-invalid" { + t.Fatalf("Hub received Authorization %v during unauthorized pull, want invalid fixture token", got) + } + unauthorizedProxyLog, err := os.ReadFile(proxyLog) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(unauthorizedProxyLog, finalProxyLog) { + t.Fatalf("TDR proxy log changed before unauthorized pull: baseline %d bytes, final %d", len(finalProxyLog), len(unauthorizedProxyLog)) + } + unauthorizedLines := bytes.Split(bytes.TrimSpace(unauthorizedProxyLog[len(finalProxyLog):]), []byte("\n")) + if len(unauthorizedLines) != 2 { + t.Fatalf("unauthorized pull generated %d new TDR requests, want OPTIONS and rejected metadata GET:\n%s", len(unauthorizedLines), unauthorizedProxyLog[len(finalProxyLog):]) + } + var sawUnauthorizedOptions, sawRejectedMetadata bool + for _, line := range unauthorizedLines { + var request struct { + Method string `json:"method"` + Path string `json:"path"` + Authorization string `json:"authorization"` + Status int `json:"status"` + } + if err := json.Unmarshal(line, &request); err != nil { + t.Fatalf("invalid unauthorized TDR proxy log line %q: %v", line, err) + } + if fixtureIDForProxyPath(request.Path, fixtures) != fixtureObjectID(fixtures[0]) { + t.Fatalf("unauthorized pull contacted the wrong TDR object: %s", line) + } + if request.Method == "OPTIONS" && request.Authorization == "" && request.Status == http.StatusOK { + sawUnauthorizedOptions = true + } + if request.Method == "GET" && !strings.Contains(request.Path, "/access/") && request.Authorization == "Bearer tdr-ci-invalid" && request.Status == http.StatusUnauthorized { + sawRejectedMetadata = true + } + } + if !sawUnauthorizedOptions || !sawRejectedMetadata { + t.Fatalf("TDR did not reject the bad bearer token as expected:\n%s", unauthorizedProxyLog[len(finalProxyLog):]) } } @@ -324,7 +580,7 @@ func makeDockerHostCertificate() (tls.Certificate, []byte, error) { NotBefore: now.Add(-time.Minute), NotAfter: now.Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, IsCA: true, DNSNames: []string{"host.docker.internal"}, + BasicConstraintsValid: true, IsCA: true, DNSNames: []string{"host.docker.internal", "data.terra.bio"}, } der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) if err != nil { @@ -340,13 +596,19 @@ func makeDockerHostCertificate() (tls.Certificate, []byte, error) { return certificate, certificatePEM, err } -func verifyTerraHubMetadata(response *http.Response, payload []byte) error { +type terraCLIObject struct { + filename string + oid string + payload []byte +} + +func verifyTerraHubMetadata(response *http.Response, fixtures []terraCLIObject) (string, error) { if response.StatusCode != http.StatusOK { - return fmt.Errorf("Hub returned HTTP %d", response.StatusCode) + return "", fmt.Errorf("Hub returned HTTP %d", response.StatusCode) } body, err := io.ReadAll(io.LimitReader(response.Body, 4<<20)) if err != nil { - return fmt.Errorf("read Hub response: %w", err) + return "", fmt.Errorf("read Hub response: %w", err) } _ = response.Body.Close() response.Body = io.NopCloser(bytes.NewReader(body)) @@ -356,13 +618,122 @@ func verifyTerraHubMetadata(response *http.Response, payload []byte) error { Hashes map[string]string `json:"hashes"` } if err := json.Unmarshal(body, &metadata); err != nil { - return fmt.Errorf("decode Hub metadata: %w", err) + return "", fmt.Errorf("decode Hub metadata: %w", err) } - crc := crc32.Checksum(payload, crc32.MakeTable(crc32.Castagnoli)) - if metadata.FileName != "1614321.merge_output.gvcf.gz" || metadata.Size != int64(len(payload)) || - metadata.Hashes["md5"] != fmt.Sprintf("%x", md5.Sum(payload)) || - metadata.Hashes["crc32c"] != fmt.Sprintf("%08x", crc) { - return fmt.Errorf("Hub returned inconsistent file metadata: %s", body) + for _, fixture := range fixtures { + if metadata.FileName != fixture.filename { + continue + } + crc := crc32.Checksum(fixture.payload, crc32.MakeTable(crc32.Castagnoli)) + if metadata.Size != int64(len(fixture.payload)) || + metadata.Hashes["md5"] != fmt.Sprintf("%x", md5.Sum(fixture.payload)) || + metadata.Hashes["crc32c"] != fmt.Sprintf("%08x", crc) { + return "", fmt.Errorf("Hub returned inconsistent metadata for %s: %s", fixture.filename, body) + } + return fixture.filename, nil } - return nil + return "", fmt.Errorf("Hub returned unexpected file metadata: %s", body) +} + +func assertFileBytes(t *testing.T, repo string, fixture terraCLIObject, output []byte) { + t.Helper() + got, err := os.ReadFile(filepath.Join(repo, fixture.filename)) + if err != nil || !bytes.Equal(got, fixture.payload) { + t.Fatalf("worktree bytes for %s = %q, want %q (error %v)\n%s", fixture.filename, got, fixture.payload, err, output) + } +} + +func assertStagedPointers(t *testing.T, repo string, fixtures []terraCLIObject, pointers map[string]string) { + t.Helper() + for _, fixture := range fixtures { + if got := runGitOutputTest(t, repo, "show", ":"+fixture.filename); got != pointers[fixture.filename] { + t.Fatalf("staged index bytes for %s = %q, want pointer %q", fixture.filename, got, pointers[fixture.filename]) + } + } +} + +func assertCachedObjects(t *testing.T, paths map[string]string, fixtures []terraCLIObject) { + t.Helper() + for _, fixture := range fixtures { + assertCachedObject(t, paths[fixture.filename], fixture) + } +} + +func assertCachedObject(t *testing.T, path string, fixture terraCLIObject) { + t.Helper() + cached, err := os.ReadFile(path) + if err != nil || !bytes.Equal(cached, fixture.payload) { + t.Fatalf("cache bytes for %s do not match fixture: got %d bytes, want %d (error %v)", fixture.filename, len(cached), len(fixture.payload), err) + } +} + +func assertResolvedCounts(t *testing.T, mu *sync.Mutex, counts map[string]int, expected map[string]int) { + t.Helper() + mu.Lock() + defer mu.Unlock() + for filename, want := range expected { + if got := counts[filename]; got != want { + t.Fatalf("Hub resolved %s %d times, want %d", filename, got, want) + } + } +} + +func fixtureIDForProxyPath(path string, fixtures []terraCLIObject) string { + for _, fixture := range fixtures { + id := fixtureObjectID(fixture) + if strings.Contains(path, "/objects/"+id) { + return id + } + } + return "" +} + +func fixtureObjectID(fixture terraCLIObject) string { + colon := strings.LastIndex(fixture.oid, ":") + if colon < 0 || colon == len(fixture.oid)-1 { + return "" + } + return fixture.oid[colon+1:] +} + +func countTDRRequests(t *testing.T, proxyLog []byte, fixtures []terraCLIObject) (map[string]int, map[string]int, map[string]int) { + t.Helper() + metadataRequests := make(map[string]int) + accessRequests := make(map[string]int) + probes := make(map[string]int) + for _, line := range bytes.Split(bytes.TrimSpace(proxyLog), []byte("\n")) { + var request struct { + Method string `json:"method"` + Path string `json:"path"` + Authorization string `json:"authorization"` + Status int `json:"status"` + } + if err := json.Unmarshal(line, &request); err != nil { + t.Fatalf("invalid TDR proxy log line %q: %v", line, err) + } + objectID := fixtureIDForProxyPath(request.Path, fixtures) + if objectID == "" { + t.Fatalf("unexpected TDR request path in proxy log: %s", line) + } + if request.Method == "OPTIONS" { + if request.Authorization != "" || request.Status != http.StatusOK { + t.Fatalf("invalid unauthenticated TDR OPTIONS request: %s", line) + } + probes[objectID]++ + continue + } + if request.Method != "GET" || request.Authorization != "Bearer tdr-ci-fixture" || request.Status != http.StatusOK { + t.Fatalf("invalid authenticated TDR request: %s", line) + } + if strings.Contains(request.Path, "/access/") { + accessRequests[objectID]++ + } else { + metadataRequests[objectID]++ + } + } + return metadataRequests, accessRequests, probes +} + +func shellQuote(argument string) string { + return "'" + strings.ReplaceAll(argument, "'", "'\\''") + "'" } diff --git a/tests/terra-drs-hub/README.md b/tests/terra-drs-hub/README.md index e831748f..bb26f430 100644 --- a/tests/terra-drs-hub/README.md +++ b/tests/terra-drs-hub/README.md @@ -1,12 +1,14 @@ # Terra pull through DRS Hub and TDR -The `Resolve through DRS Hub and TDR` CI job starts the pinned public TDR and DRS Hub source revisions and runs the real `git-drs` CLI binary against them. Hub expands the `drs.anv0` compact ID, selects its Terra provider, and calls TDR's DRS API. TDR's controller and `DrsService` resolve the PostgreSQL DRS-to-snapshot mapping, snapshot metadata, and Firestore file metadata, then return a Google Cloud Storage access URL. +The `Resolve through DRS Hub and TDR` CI job starts pinned public TDR and DRS Hub source revisions and runs the real `git-drs` CLI binary against them. The fixture has three DRS objects: the original test ID and two IDs used by the public [AnVIL GREGoR reference repository](https://github.com/EllrottLab/AnVIL_GREGoR_R05_GRU). All payloads are tiny synthetic text, with truthful fixture sizes and checksums; this test does not download controlled genomic data. Hub expands `drs.anv0`, selects its Terra provider, and calls TDR's DRS API. TDR's controller and `DrsService` resolve the PostgreSQL DRS-to-snapshot mapping, snapshot metadata, and Firestore file metadata, then return Google Cloud Storage access URLs. The test uses a generated service-account ADC JSON key and a local OAuth token endpoint. The token endpoint validates the signed JWT grant before returning a fixture bearer token. Hub's initial unauthenticated `OPTIONS` probe reaches TDR, followed by authenticated metadata and access requests. The test checks the request methods, statuses, and bearer token. TDR authorization and some snapshot/provider dependencies remain fixture-backed; Google credentials and external Sam are not needed. TDR's real Google Storage client generates a V4 RSA signed URL using a generated test key. Before the CLI receives the Hub response, a local HTTPS bridge verifies the returned metadata and validates the signed URL's canonical request, RSA signature, and expiry. The bridge then changes only the signed URL authority so the CLI can reach the local GCS-compatible emulator from its Linux container. The emulator serves the fixture bytes but does not validate signed URLs itself. -The runner builds a Linux `git-drs` binary for the host architecture and runs it inside a pinned multi-architecture Alpine Git image. It configures the actual clean, smudge, and process filters, checks that the worktree contains the downloaded bytes while the index still contains the pointer, verifies the cache, and runs a second pull to confirm it makes no additional Hub, OAuth, or TDR requests. It also asserts that the emulator served exactly one object download. +The runner builds a Linux `git-drs` binary for the host architecture and runs it inside a pinned multi-architecture Alpine Git image. It configures a Terra remote with the CLI, checks `remote list` and `ping` against TDR's real service-info endpoint, selectively pulls one GREGoR ID, pulls the remaining pointers, repeats the pull from cache, and checks out a pointer through Git's process filter. It creates pointers with both `add-ref --manifest` and single-URI `add-ref`. Finally, a fixture token rejected by TDR must produce a clear authorization error without hydrating the file or writing the cache. The test checks the worktree, index, cache, metadata and per-object HTTP requests throughout, and asserts exactly one emulator download for each of the three objects. + +This job covers reference authoring, successful hydration, and a TDR rejection of a fixture token. It does not exercise production Google/Sam authorization, a real GCS bucket, large transfers, or interrupted resume. Terra remotes are read-only; uploading and deleting Terra objects are not supported operations to test here. Run locally using the pinned TDR and DRS Hub revisions in `.github/workflows/pr-checks.yaml`: diff --git a/tests/terra-drs-hub/run.sh b/tests/terra-drs-hub/run.sh index 45a4099e..ab1ceafe 100755 --- a/tests/terra-drs-hub/run.sh +++ b/tests/terra-drs-hub/run.sh @@ -182,6 +182,7 @@ cli_binary="$coord_dir/git-drs-linux" GOOS=linux GOARCH="$(go env GOARCH)" CGO_ENABLED=0 go build -o "$cli_binary" . if ! GIT_DRS_BINARY="$cli_binary" \ GIT_DRS_HUB_HTTP_ENDPOINT="http://127.0.0.1:$hub_port" \ + GIT_DRS_TDR_HTTP_ENDPOINT="http://127.0.0.1:$tdr_port" \ GIT_DRS_TDR_PROXY_LOG="$proxy_log" \ GIT_DRS_TDR_GCS_ENDPOINT="${STORAGE_EMULATOR_HTTP_ENDPOINT/127.0.0.1/host.docker.internal}" \ GIT_DRS_TDR_SIGNING_PUBLIC_KEY_FILE="$signing_public_key_file" \ @@ -195,8 +196,8 @@ if ! GIT_DRS_BINARY="$cli_binary" \ fi downloads=$(storage_object_download_count) -if [[ "$downloads" != 1 ]]; then - echo "GCS emulator served $downloads object downloads, want exactly one" >&2 +if [[ "$downloads" != 3 ]]; then + echo "GCS emulator served $downloads object downloads, want exactly three" >&2 docker logs "$STORAGE_CONTAINER" >&2 || true exit 1 fi diff --git a/tests/terra-tdr/TerraDrsHarnessTest.java b/tests/terra-tdr/TerraDrsHarnessTest.java index 34c17c08..f6a16840 100644 --- a/tests/terra-tdr/TerraDrsHarnessTest.java +++ b/tests/terra-tdr/TerraDrsHarnessTest.java @@ -120,6 +120,10 @@ class TerraDrsHarnessTest { private static final String OBJECT_ID = "v2_4f770147-e372-339b-b9fa-0a7a83cf30cf"; private static final UUID SNAPSHOT_ID = UUID.fromString("11111111-2222-4333-8444-555555555555"); private static final UUID FILE_ID = UUID.fromString("4f770147-e372-339b-b9fa-0a7a83cf30cf"); + private static final UUID GREGOR_1614321_FILE_ID = + UUID.fromString("c5ae75de-1f5c-3d40-bcd9-02f827fbf2d3"); + private static final UUID GREGOR_1614322_FILE_ID = + UUID.fromString("4872d195-f80a-3662-90a7-2aa90e0a7f53"); private static final UUID BILLING_PROFILE_ID = UUID.fromString("aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"); private static final UUID DATASET_ID = UUID.fromString("bbbbbbbb-cccc-4ddd-8eee-ffffffffffff"); @@ -130,6 +134,28 @@ class TerraDrsHarnessTest { private static final byte[] FIXTURE_BYTES = "git-drs Terra DRS HTTP integration fixture\n" .getBytes(java.nio.charset.StandardCharsets.UTF_8); + // IDs and sample names are inspired by GREGoR records; these payloads are tiny synthetic text, + // and each seeded size/checksum reflects the synthetic bytes rather than the public file. + // Prefixing 1614321 avoids colliding with the original fixture path used by existing scenarios. + private static final List FIXTURE_OBJECTS = + List.of( + new FixtureObject( + OBJECT_ID, + FILE_ID, + "1614321.merge_output.gvcf.gz", + FIXTURE_BYTES), + new FixtureObject( + "v2_c5ae75de-1f5c-3d40-bcd9-02f827fbf2d3", + GREGOR_1614321_FILE_ID, + "gregor-1614321.merge_output.gvcf.gz", + "synthetic GREGoR 1614321 fixture; not genomic data\n" + .getBytes(java.nio.charset.StandardCharsets.UTF_8)), + new FixtureObject( + "v2_4872d195-f80a-3662-90a7-2aa90e0a7f53", + GREGOR_1614322_FILE_ID, + "1614322.merge_output.gvcf.gz", + "synthetic GREGoR 1614322 fixture; not genomic data\n" + .getBytes(java.nio.charset.StandardCharsets.UTF_8))); private static final Duration STOP_TIMEOUT = Duration.ofMinutes(20); @LocalServerPort private int port; private HttpServer tokenServer; @@ -273,30 +299,35 @@ void stopFixtureTokenServer() { private void seedFirestoreFile() throws Exception { Snapshot snapshot = fixtureSnapshot(); Dataset dataset = snapshot.getFirstSnapshotSource().getDataset(); - fireStoreDao.upsertFileMetadata( - dataset, - new FireStoreFile() - .fileId(FILE_ID.toString()) - .fileCreatedDate("2026-10-07T00:00:00Z") - .gspath("gs://fixture-bucket/1614321.merge_output.gvcf.gz") - .checksumCrc32c(fixtureFile().getChecksumCrc32c()) - .checksumMd5(fixtureFile().getChecksumMd5()) - .size((long) FIXTURE_BYTES.length) - .mimeType("application/gzip") - .description("CI file fixture") - .bucketResourceId("fixture-bucket-resource") - .loadTag("git-drs-ci")); - fireStoreDao.createDirectoryEntry( + for (FixtureObject fixture : FIXTURE_OBJECTS) { + fireStoreDao.upsertFileMetadata( + dataset, + new FireStoreFile() + .fileId(fixture.fileId().toString()) + .fileCreatedDate("2026-10-07T00:00:00Z") + .gspath("gs://fixture-bucket/" + fixture.filename()) + .checksumCrc32c(crc32cHex(fixture.bytes())) + .checksumMd5(md5Hex(fixture.bytes())) + .size((long) fixture.bytes().length) + .mimeType("application/gzip") + .description("Synthetic CI fixture; no real genomic payload") + .bucketResourceId("fixture-bucket-resource") + .loadTag("git-drs-ci")); + fireStoreDao.createDirectoryEntry( + dataset, + new FireStoreDirectoryEntry() + .fileId(fixture.fileId().toString()) + .isFileRef(true) + .path("/") + .name(fixture.filename()) + .datasetId(dataset.getId().toString()) + .fileCreatedDate("2026-10-07T00:00:00Z") + .loadTag("git-drs-ci")); + } + fireStoreDao.addFilesToSnapshot( dataset, - new FireStoreDirectoryEntry() - .fileId(FILE_ID.toString()) - .isFileRef(true) - .path("/") - .name("1614321.merge_output.gvcf.gz") - .datasetId(dataset.getId().toString()) - .fileCreatedDate("2026-10-07T00:00:00Z") - .loadTag("git-drs-ci")); - fireStoreDao.addFilesToSnapshot(dataset, snapshot, java.util.List.of(FILE_ID.toString())); + snapshot, + FIXTURE_OBJECTS.stream().map(fixture -> fixture.fileId().toString()).toList()); } private void migrateAndSeedDrsRecord() throws Exception { @@ -345,9 +376,10 @@ private void migrateAndSeedDrsRecord() throws Exception { "INSERT INTO snapshot_source (snapshot_id, dataset_id) VALUES (?, ?)", SNAPSHOT_ID, DATASET_ID); - DrsId objectId = drsIdService.fromObjectId(OBJECT_ID); - if (drsDao.recordDrsIdToSnapshot(SNAPSHOT_ID, java.util.List.of(objectId)) != 1) { - throw new IllegalStateException("TDR DrsDao did not insert the fixture DRS mapping"); + List objectIds = + FIXTURE_OBJECTS.stream().map(fixture -> drsIdService.fromObjectId(fixture.drsId())).toList(); + if (drsDao.recordDrsIdToSnapshot(SNAPSHOT_ID, objectIds) != FIXTURE_OBJECTS.size()) { + throw new IllegalStateException("TDR DrsDao did not insert all fixture DRS mappings"); } } @@ -388,30 +420,23 @@ private static Snapshot fixtureSnapshot() { .googleProjectId("fixture-dataset-project"))))); } - private static FSFile fixtureFile() { + private static String crc32cHex(byte[] bytes) { java.util.zip.CRC32C crc32c = new java.util.zip.CRC32C(); - crc32c.update(FIXTURE_BYTES, 0, FIXTURE_BYTES.length); - String crcHex = String.format("%08x", crc32c.getValue()); - String md5Hex; + crc32c.update(bytes, 0, bytes.length); + return String.format("%08x", crc32c.getValue()); + } + + private static String md5Hex(byte[] bytes) { try { - md5Hex = - java.util.HexFormat.of() - .formatHex(java.security.MessageDigest.getInstance("MD5").digest(FIXTURE_BYTES)); + return java.util.HexFormat.of() + .formatHex(java.security.MessageDigest.getInstance("MD5").digest(bytes)); } catch (java.security.NoSuchAlgorithmException e) { throw new IllegalStateException(e); } - return new FSFile() - .fileId(FILE_ID) - .path("1614321.merge_output.gvcf.gz") - .cloudPath("gs://fixture-bucket/1614321.merge_output.gvcf.gz") - .cloudPlatform(CloudPlatform.GCP) - .bucketResourceId("fixture-bucket-resource") - .createdDate(java.time.Instant.parse("2026-10-07T00:00:00Z")) - .size((long) FIXTURE_BYTES.length) - .checksumCrc32c(crcHex) - .checksumMd5(md5Hex); } + private record FixtureObject(String drsId, UUID fileId, String filename, byte[] bytes) {} + private static Path requiredPath(String envName) { String value = System.getenv(envName); if (value == null || value.isBlank()) { diff --git a/tests/terra-tdr/storage-emulator.sh b/tests/terra-tdr/storage-emulator.sh index 65238708..4ceddb21 100755 --- a/tests/terra-tdr/storage-emulator.sh +++ b/tests/terra-tdr/storage-emulator.sh @@ -14,6 +14,10 @@ start_storage_emulator() { mkdir -p "$seed_dir/fixture-bucket" printf 'git-drs Terra DRS HTTP integration fixture\n' \ >"$seed_dir/fixture-bucket/1614321.merge_output.gvcf.gz" + printf 'synthetic GREGoR 1614321 fixture; not genomic data\n' \ + >"$seed_dir/fixture-bucket/gregor-1614321.merge_output.gvcf.gz" + printf 'synthetic GREGoR 1614322 fixture; not genomic data\n' \ + >"$seed_dir/fixture-bucket/1614322.merge_output.gvcf.gz" docker run --detach --rm \ --name "$STORAGE_CONTAINER" \ @@ -54,6 +58,11 @@ stop_storage_emulator() { } storage_object_download_count() { + local filename=${1:-} + local marker='GET /fixture-bucket/' + if [[ -n "$filename" ]]; then + marker="GET /fixture-bucket/$filename?X-Goog-Algorithm=" + fi docker logs "$STORAGE_CONTAINER" 2>&1 | - python3 -c 'import sys; marker = "GET /fixture-bucket/1614321.merge_output.gvcf.gz?X-Goog-Algorithm="; print(sum(marker in line for line in sys.stdin))' + python3 -c 'import sys; marker = sys.argv[1]; print(sum(marker in line and "X-Goog-Algorithm=" in line for line in sys.stdin))' "$marker" } From eaee916ffeeac4cfdd8acef7b733621cb36acf05 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 8 Oct 2026 12:25:06 -0700 Subject: [PATCH 18/22] Format expanded TDR integration fixture --- tests/terra-tdr/TerraDrsHarnessTest.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/terra-tdr/TerraDrsHarnessTest.java b/tests/terra-tdr/TerraDrsHarnessTest.java index f6a16840..9096f122 100644 --- a/tests/terra-tdr/TerraDrsHarnessTest.java +++ b/tests/terra-tdr/TerraDrsHarnessTest.java @@ -139,11 +139,7 @@ class TerraDrsHarnessTest { // Prefixing 1614321 avoids colliding with the original fixture path used by existing scenarios. private static final List FIXTURE_OBJECTS = List.of( - new FixtureObject( - OBJECT_ID, - FILE_ID, - "1614321.merge_output.gvcf.gz", - FIXTURE_BYTES), + new FixtureObject(OBJECT_ID, FILE_ID, "1614321.merge_output.gvcf.gz", FIXTURE_BYTES), new FixtureObject( "v2_c5ae75de-1f5c-3d40-bcd9-02f827fbf2d3", GREGOR_1614321_FILE_ID, @@ -377,7 +373,9 @@ private void migrateAndSeedDrsRecord() throws Exception { SNAPSHOT_ID, DATASET_ID); List objectIds = - FIXTURE_OBJECTS.stream().map(fixture -> drsIdService.fromObjectId(fixture.drsId())).toList(); + FIXTURE_OBJECTS.stream() + .map(fixture -> drsIdService.fromObjectId(fixture.drsId())) + .toList(); if (drsDao.recordDrsIdToSnapshot(SNAPSHOT_ID, objectIds) != FIXTURE_OBJECTS.size()) { throw new IllegalStateException("TDR DrsDao did not insert all fixture DRS mappings"); } From b04284250b9b27f96d34be05558ec5c1f86796eb Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 8 Oct 2026 12:31:49 -0700 Subject: [PATCH 19/22] Replace hydrated file before unauthorized pull fixture --- cmd/pull/terra_hub_integration_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/pull/terra_hub_integration_test.go b/cmd/pull/terra_hub_integration_test.go index 0b5fa0cb..be1cbcaa 100644 --- a/cmd/pull/terra_hub_integration_test.go +++ b/cmd/pull/terra_hub_integration_test.go @@ -501,6 +501,9 @@ func TestIntegrationPullThroughTerraHubAndTDR(t *testing.T) { } // Remove the already verified object so this request must authenticate and resolve remotely. + if err := os.Remove(filepath.Join(repo, fixtures[0].filename)); err != nil { + t.Fatal(err) + } if err := os.WriteFile(filepath.Join(repo, fixtures[0].filename), []byte(pointers[fixtures[0].filename]), 0o644); err != nil { t.Fatal(err) } From 90615b4f03edf499c2c4789db98abbd7b138ad56 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 8 Oct 2026 12:43:28 -0700 Subject: [PATCH 20/22] Consolidate Terra integration fixtures --- tests/{terra-tdr => terra-drs-hub}/TerraDrsHarnessTest.java | 0 tests/{terra-tdr => terra-drs-hub}/firestore-emulator.sh | 0 tests/terra-drs-hub/run.sh | 6 +++--- tests/{terra-tdr => terra-drs-hub}/storage-emulator.sh | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename tests/{terra-tdr => terra-drs-hub}/TerraDrsHarnessTest.java (100%) rename tests/{terra-tdr => terra-drs-hub}/firestore-emulator.sh (100%) rename tests/{terra-tdr => terra-drs-hub}/storage-emulator.sh (100%) diff --git a/tests/terra-tdr/TerraDrsHarnessTest.java b/tests/terra-drs-hub/TerraDrsHarnessTest.java similarity index 100% rename from tests/terra-tdr/TerraDrsHarnessTest.java rename to tests/terra-drs-hub/TerraDrsHarnessTest.java diff --git a/tests/terra-tdr/firestore-emulator.sh b/tests/terra-drs-hub/firestore-emulator.sh similarity index 100% rename from tests/terra-tdr/firestore-emulator.sh rename to tests/terra-drs-hub/firestore-emulator.sh diff --git a/tests/terra-drs-hub/run.sh b/tests/terra-drs-hub/run.sh index ab1ceafe..212007b3 100755 --- a/tests/terra-drs-hub/run.sh +++ b/tests/terra-drs-hub/run.sh @@ -9,8 +9,8 @@ fi git_drs_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) tdr_root=$(cd "$1" && pwd) hub_root=$(cd "$2" && pwd) -source "$git_drs_root/tests/terra-tdr/firestore-emulator.sh" -source "$git_drs_root/tests/terra-tdr/storage-emulator.sh" +source "$git_drs_root/tests/terra-drs-hub/firestore-emulator.sh" +source "$git_drs_root/tests/terra-drs-hub/storage-emulator.sh" coord_dir=$(mktemp -d) port_file="$coord_dir/tdr-port" stop_file="$coord_dir/tdr-stop" @@ -25,7 +25,7 @@ private_key="$coord_dir/localhost.key" truststore="$coord_dir/truststore.p12" signing_public_key_file="$coord_dir/tdr-signing-public-key.pem" -cp "$git_drs_root/tests/terra-tdr/TerraDrsHarnessTest.java" \ +cp "$git_drs_root/tests/terra-drs-hub/TerraDrsHarnessTest.java" \ "$tdr_root/src/test/java/bio/terra/service/filedata/TerraDrsHarnessTest.java" cp "$git_drs_root/tests/terra-drs-hub/TerraDrsHubIntegrationTest.java" \ "$hub_root/service/src/test/java/bio/terra/drshub/controllers/TerraDrsHubIntegrationTest.java" diff --git a/tests/terra-tdr/storage-emulator.sh b/tests/terra-drs-hub/storage-emulator.sh similarity index 100% rename from tests/terra-tdr/storage-emulator.sh rename to tests/terra-drs-hub/storage-emulator.sh From 03acf55bfa0ae481323a3b6fcaea918a00821076 Mon Sep 17 00:00:00 2001 From: Brian Walsh Date: Wed, 14 Oct 2026 12:00:49 -0700 Subject: [PATCH 21/22] adds comments --- tests/terra-drs-hub/run.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/terra-drs-hub/run.sh b/tests/terra-drs-hub/run.sh index 212007b3..9b419bc4 100755 --- a/tests/terra-drs-hub/run.sh +++ b/tests/terra-drs-hub/run.sh @@ -25,6 +25,7 @@ private_key="$coord_dir/localhost.key" truststore="$coord_dir/truststore.p12" signing_public_key_file="$coord_dir/tdr-signing-public-key.pem" +# Inject the harness tests and Hub settings into the two external source trees. cp "$git_drs_root/tests/terra-drs-hub/TerraDrsHarnessTest.java" \ "$tdr_root/src/test/java/bio/terra/service/filedata/TerraDrsHarnessTest.java" cp "$git_drs_root/tests/terra-drs-hub/TerraDrsHubIntegrationTest.java" \ @@ -43,6 +44,8 @@ drshub: fetchAccessUrl: true YAML +# Hub calls the local TDR through HTTPS, so give it a localhost certificate +# trusted only by this run's copied Java truststore. openssl req -new -newkey rsa:2048 -nodes -keyout "$private_key" -x509 -days 1 \ -out "$certificate" -subj "/CN=localhost" \ -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" @@ -65,6 +68,7 @@ if ! JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:-} -Djavax.net.ssl.trustStore=$trust fi tls_port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') +# TDR reads file metadata from Firestore and signs URLs for the local GCS fixture. FIRESTORE_CONTAINER= start_firestore_emulator STORAGE_CONTAINER= @@ -83,6 +87,8 @@ tdr_pid=$! proxy_pid= hub_pid= +# Java harnesses create their port files when ready and stop when signaled by +# their stop files; keep cleanup installed while any service is running. stop_services() { touch "$hub_stop_file" touch "$stop_file" @@ -117,6 +123,8 @@ if [[ ! -s "$port_file" ]]; then fi tdr_port=$(cat "$port_file") +# The HTTPS bridge lets Hub use TDR's signed URL flow while the CLI reaches +# the local storage emulator through its own network namespace. python3 "$git_drs_root/tests/terra-drs-hub/tdr_https_proxy.py" \ "http://127.0.0.1:$tdr_port" "$tls_port" "$certificate" "$private_key" "$proxy_log" \ >"$coord_dir/proxy.out" 2>&1 & @@ -179,6 +187,7 @@ fi hub_port=$(cat "$hub_port_file") cd "$git_drs_root" cli_binary="$coord_dir/git-drs-linux" +# The integration test runs the CLI inside a Linux container on this machine's architecture. GOOS=linux GOARCH="$(go env GOARCH)" CGO_ENABLED=0 go build -o "$cli_binary" . if ! GIT_DRS_BINARY="$cli_binary" \ GIT_DRS_HUB_HTTP_ENDPOINT="http://127.0.0.1:$hub_port" \ @@ -196,6 +205,7 @@ if ! GIT_DRS_BINARY="$cli_binary" \ fi downloads=$(storage_object_download_count) +# One download per fixture object proves the repeated pull used the cache. if [[ "$downloads" != 3 ]]; then echo "GCS emulator served $downloads object downloads, want exactly three" >&2 docker logs "$STORAGE_CONTAINER" >&2 || true From 9e599e2c5f92da23fe7a4ed15ca0e7c8e66b82b9 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 30 Oct 2026 13:13:07 -0700 Subject: [PATCH 22/22] Add manifest-based object indexing --- README.md | 9 +- cmd/addurl/main.go | 31 +- cmd/addurl/manifest_import.go | 368 +++++++++++++++++++++ cmd/addurl/manifest_import_test.go | 229 +++++++++++++ cmd/copyrecords/import.go | 104 ++++++ cmd/pull/main.go | 2 +- docs/adding-s3-files.md | 2 + docs/commands.md | 112 ++++--- docs/e2e-modes-and-local-setup.md | 2 + docs/globus-add-url.md | 2 + docs/index.md | 3 +- docs/pointer-files.md | 2 + internal/gitrepo/gitattributes_tracking.go | 18 + internal/transfer/download.go | 16 +- internal/transfer/download_test.go | 15 + mkdocs.yml | 1 - 16 files changed, 850 insertions(+), 66 deletions(-) create mode 100644 cmd/addurl/manifest_import.go create mode 100644 cmd/addurl/manifest_import_test.go create mode 100644 cmd/copyrecords/import.go diff --git a/README.md b/README.md index 4e44155d..16c12d83 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ At a high level: ```bash git drs install git drs remote add production https://example-gen3.org --provider gen3 \ - --scope HTAN_INT/BForePC --auth provider-helper:gen3-profile \ + --scope example-org/example-project --auth provider-helper:gen3-profile \ --credential file:/path/to/credentials.json git drs track "*.bam" git add .gitattributes @@ -72,7 +72,7 @@ The cleaned CLI intentionally removed legacy commands: Example: ```bash -git drs remote add production calypr --scope HTAN_INT/BForePC \ +git drs remote add production calypr --scope example-org/example-project \ --credential file:/path/to/credentials.json ``` @@ -83,7 +83,7 @@ Current command split: - `git drs pull` hydrates tracked pointer files already present in the checkout - `git drs ls-files` lists visible files and directories; use `--all` for the tracked DRS/LFS pointer inventory -- `git drs add-url` prepares pointer plus local metadata for existing provider objects +- `git drs add-url` indexes existing S3 objects from a TSV manifest, with optional pointer commits ## Bucket Mapping Model @@ -108,7 +108,7 @@ Push and pull depend on server-side bucket mapping for the requested scope. That | `git drs ls-files` | List visible files and directories | | `git drs pull` | Hydrate pointer files in the current checkout | | `git drs push` | Register/upload objects and push refs | -| `git drs add-url` | Add an existing provider object by URL or scoped key | +| `git drs add-url` | Index existing S3 objects from a TSV manifest | | `git drs add-ref` | Add a local reference to an existing DRS object | | `git drs query` | Query a DRS object by ID | | `git drs copy-records` | Copy Syfon records between remotes for one scope | @@ -119,7 +119,6 @@ Push and pull depend on server-side bucket mapping for the requested scope. That - [Getting Started](docs/getting-started.md) - [Commands Reference](docs/commands.md) - [Access-Method Selection](docs/access-method-selection-and-authentication.md) -- [Adding Globus Objects](docs/globus-add-url.md) - [Troubleshooting](docs/troubleshooting.md) - [Developer Guide](docs/developer-guide.md) - [Publishing documentation with GitHub Pages](docs/github-pages.md) diff --git a/cmd/addurl/main.go b/cmd/addurl/main.go index 11767c61..a942722a 100644 --- a/cmd/addurl/main.go +++ b/cmd/addurl/main.go @@ -1,10 +1,6 @@ package addurl -import ( - "errors" - - "github.com/spf13/cobra" -) +import "github.com/spf13/cobra" var Cmd = NewCommand() @@ -12,19 +8,20 @@ var Cmd = NewCommand() // wiring usage, argument validation and the RunE handler. func NewCommand() *cobra.Command { cmd := &cobra.Command{ - Use: "add-url [path]", - Short: "Add a file from a provider URL or configured bucket object key", - Args: func(cmd *cobra.Command, args []string) error { - if len(args) < 1 || len(args) > 2 { - return errors.New("usage: add-url [path]") - } - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - return NewAddURLService().Run(cmd, args) - }, + Use: "add-url ", + Short: "Index existing S3 objects from a TSV manifest", + Args: cobra.ExactArgs(1), + RunE: runManifest, } addFlags(cmd) + cmd.Flags().String("repo", ".", "Repository containing the git-drs remote configuration") + cmd.Flags().Bool("commit", false, "Also create and commit Git pointers") + cmd.Flags().Bool("git-only", false, "Verify existing Syfon records, commit pointers, and push the Git branch") + cmd.Flags().String("branch", "", "Required current branch when using --commit") + cmd.Flags().String("strip-prefix", "", "Remove this source prefix and resolve the remainder in the configured project scope") + for _, name := range []string{"sha256", "scheme", "recursive", "manifest"} { + _ = cmd.Flags().MarkHidden(name) + } return cmd } @@ -42,7 +39,7 @@ func addFlags(cmd *cobra.Command) { ) cmd.Flags().Bool("recursive", false, "Deprecated: Globus directories and wildcards are detected automatically") _ = cmd.Flags().MarkDeprecated("recursive", "Globus directories and wildcards are detected automatically") - cmd.Flags().Bool("dry-run", false, "List and validate matching Globus objects without writing files") + cmd.Flags().Bool("dry-run", false, "Probe and validate the manifest without writing records or a commit") cmd.Flags().String("manifest", "", "Optional TSV manifest supplying path, size, and sha256 for Globus collections") cmd.Flags().StringP("remote", "r", "", "Target DRS remote") } diff --git a/cmd/addurl/manifest_import.go b/cmd/addurl/manifest_import.go new file mode 100644 index 00000000..0f05c787 --- /dev/null +++ b/cmd/addurl/manifest_import.go @@ -0,0 +1,368 @@ +package addurl + +import ( + "context" + "encoding/csv" + "fmt" + "io" + "net/url" + "os" + "os/exec" + "path" + "path/filepath" + "strconv" + "strings" + + "github.com/calypr/git-drs/cmd/copyrecords" + "github.com/calypr/git-drs/internal/config" + "github.com/calypr/git-drs/internal/drslog" + "github.com/calypr/git-drs/internal/drsobject" + "github.com/calypr/git-drs/internal/gitrepo" + "github.com/calypr/git-drs/internal/remoteruntime" + drsapi "github.com/calypr/syfon/apigen/drs" + internalapi "github.com/calypr/syfon/apigen/internalapi" + "github.com/calypr/syfon/client/apierror" + "github.com/google/uuid" + "github.com/spf13/cobra" +) + +type manifestEntry struct { + source, destination, sha string + size int64 + resolved string +} + +func readImportManifest(filename string) ([]manifestEntry, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + reader := csv.NewReader(f) + reader.Comma = '\t' + reader.FieldsPerRecord = -1 + header, err := reader.Read() + if err != nil { + return nil, fmt.Errorf("read manifest header: %w", err) + } + cols := map[string]int{} + for i, v := range header { + key := strings.ToLower(strings.TrimSpace(v)) + if _, ok := cols[key]; ok { + return nil, fmt.Errorf("duplicate column %q", key) + } + cols[key] = i + } + for _, key := range []string{"s3_path", "repo_path", "sha256", "size_bytes"} { + if _, ok := cols[key]; !ok { + return nil, fmt.Errorf("manifest missing %q column", key) + } + } + var entries []manifestEntry + paths := map[string]bool{} + hashes := map[string]bool{} + for line := 2; ; line++ { + row, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("manifest row %d: %w", line, err) + } + val := func(key string) string { + i := cols[key] + if i >= len(row) { + return "" + } + return strings.TrimSpace(row[i]) + } + dest, source := val("repo_path"), val("s3_path") + if dest == "" || !filepath.IsLocal(filepath.FromSlash(dest)) || path.Clean(dest) != dest || strings.ContainsAny(dest, "\\\x00\n\r") || strings.HasPrefix(dest, ".git/") || dest == ".git" || paths[dest] { + return nil, fmt.Errorf("manifest row %d: invalid or duplicate repo_path %q", line, dest) + } + paths[dest] = true + if source == "" { + return nil, fmt.Errorf("manifest row %d: missing s3_path", line) + } + sha, err := normalizeSHA256(val("sha256")) + if err != nil { + return nil, fmt.Errorf("manifest row %d: %w", line, err) + } + if hashes[sha] { + return nil, fmt.Errorf("manifest row %d: repeated SHA-256 %s", line, sha) + } + hashes[sha] = true + size, err := strconv.ParseInt(val("size_bytes"), 10, 64) + if err != nil || size < 0 { + return nil, fmt.Errorf("manifest row %d: invalid size_bytes", line) + } + entries = append(entries, manifestEntry{source: source, destination: dest, sha: sha, size: size}) + } + if len(entries) == 0 { + return nil, fmt.Errorf("manifest contains no entries") + } + for _, e := range entries { + for parent := path.Dir(e.destination); parent != "."; parent = path.Dir(parent) { + if paths[parent] { + return nil, fmt.Errorf("manifest path %q is nested under another manifest file %q", e.destination, parent) + } + } + } + return entries, nil +} + +func probeManifest(ctx context.Context, api *internalapi.ClientWithResponses, entries []manifestEntry, org, project, strip string) error { + for start := 0; start < len(entries); start += 100 { + end := min(start+100, len(entries)) + items := make([]internalapi.InternalInspectObjectRequest, 0, end-start) + for i := start; i < end; i++ { + e := entries[i] + request := internalapi.InternalInspectObjectRequest{Id: strconv.Itoa(i), ExpectedSizeBytes: &e.size} + source := e.source + if strip != "" { + if !strings.HasPrefix(source, strip) { + return fmt.Errorf("manifest row %d: source does not start with --strip-prefix", i+2) + } + source = strings.TrimPrefix(source, strip) + } + if strings.HasPrefix(strings.ToLower(source), "s3://") { + u, err := url.Parse(source) + if err != nil || u.Host == "" || u.Path == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return fmt.Errorf("manifest row %d: invalid S3 URL", i+2) + } + request.ObjectUrl = source + } else { + if strings.Contains(source, "://") || !filepath.IsLocal(filepath.FromSlash(source)) { + return fmt.Errorf("manifest row %d: invalid scoped key", i+2) + } + request.Organization = org + request.Project = project + request.Scheme = "s3" + request.Key = source + } + items = append(items, request) + } + resp, err := api.InternalInspectObjectBulkWithResponse(ctx, internalapi.InternalInspectObjectsBulkRequest{Items: items}) + if err != nil { + return err + } + if resp.JSON200 == nil { + return apierror.FromResponse(resp.HTTPResponse, resp.Body) + } + if len(resp.JSON200.Items) != len(items) { + return fmt.Errorf("bulk probe returned %d results for %d objects", len(resp.JSON200.Items), len(items)) + } + seen := make(map[string]bool, len(items)) + for _, result := range resp.JSON200.Items { + n, err := strconv.Atoi(result.Id) + if err != nil || n < start || n >= end || seen[result.Id] { + return fmt.Errorf("bulk probe returned invalid result id %q", result.Id) + } + seen[result.Id] = true + if result.Status != "present" || !result.Exists || result.SizeMatch == nil || !*result.SizeMatch || result.ObjectUrl == "" { + return fmt.Errorf("probe %s: status=%s error=%s size_match=%v", entries[n].source, result.Status, result.Error, result.SizeMatch) + } + entries[n].resolved = result.ObjectUrl + } + } + return nil +} + +func runManifest(cmd *cobra.Command, args []string) (retErr error) { + for _, flag := range []string{"sha256", "scheme", "recursive", "manifest"} { + if cmd.Flags().Changed(flag) { + return fmt.Errorf("--%s is no longer supported; use a TSV manifest", flag) + } + } + repo, _ := cmd.Flags().GetString("repo") + branch, _ := cmd.Flags().GetString("branch") + commit, _ := cmd.Flags().GetBool("commit") + gitOnly, _ := cmd.Flags().GetBool("git-only") + commit = commit || gitOnly + remote, _ := cmd.Flags().GetString("remote") + strip, _ := cmd.Flags().GetString("strip-prefix") + dry, _ := cmd.Flags().GetBool("dry-run") + if strings.TrimSpace(repo) == "" { + return fmt.Errorf("--repo cannot be empty") + } + if commit && strings.TrimSpace(branch) == "" { + return fmt.Errorf("--branch is required with --commit") + } + if !commit && branch != "" { + return fmt.Errorf("--branch requires --commit") + } + manifest, err := filepath.Abs(args[0]) + if err != nil { + return err + } + entries, err := readImportManifest(manifest) + if err != nil { + return err + } + repo, err = filepath.Abs(repo) + if err != nil { + return err + } + old, err := os.Getwd() + if err != nil { + return err + } + if err = os.Chdir(repo); err != nil { + return err + } + defer os.Chdir(old) + run := func(args ...string) (string, error) { + c := exec.CommandContext(cmd.Context(), "git", args...) + out, err := c.CombinedOutput() + if err != nil { + return "", fmt.Errorf("git %s: %s: %w", strings.Join(args, " "), strings.TrimSpace(string(out)), err) + } + return strings.TrimSpace(string(out)), nil + } + root, err := gitrepo.GitTopLevel() + if err != nil { + return err + } + if err = os.Chdir(root); err != nil { + return err + } + if commit { + current, err := run("branch", "--show-current") + if err != nil { + return err + } + if current != branch { + return fmt.Errorf("repository is on branch %q, expected %q", current, branch) + } + status, err := run("status", "--porcelain") + if err != nil { + return err + } + if status != "" { + return fmt.Errorf("repository must have a clean working tree and index") + } + validation := make([]globusEntry, 0, len(entries)) + for _, e := range entries { + validation = append(validation, globusEntry{destination: e.destination}) + } + if err = validateImportDestinations(validation); err != nil { + return err + } + } + cfg, err := config.LoadConfig() + if err != nil { + return err + } + name, err := cfg.GetRemoteOrDefault(remote) + if err != nil { + return err + } + logger := drslog.GetLogger() + runtime, err := remoteruntime.New(cfg, name, logger) + if err != nil { + return err + } + org, project := runtime.Organization, runtime.ProjectId + if org == "" || project == "" || runtime.BucketName == "" || runtime.Client == nil || !runtime.CanRegister() { + return fmt.Errorf("remote %q must support registration and specify organization/project", name) + } + fmt.Fprintf(cmd.OutOrStdout(), "Calypr endpoint: %s (remote %s, scope %s/%s)\n", runtime.Endpoint, name, org, project) + if err = probeManifest(cmd.Context(), runtime.Client.InternalAPI(), entries, org, project, strip); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Probed %d objects in %s/%s\n", len(entries), org, project) + if dry && !gitOnly { + return nil + } + objects := make([]*drsapi.DrsObject, 0, len(entries)) + builder := drsobjectBuilder(runtime.BucketName, org, project, runtime.StoragePrefix) + for _, e := range entries { + id := uuid.NewSHA1(drsobject.UUIDNamespace, []byte(fmt.Sprintf("%s:%s", project, e.sha))).String() + obj, err := builder.Build(e.destination, e.sha, e.size, id) + if err != nil { + return err + } + methods := []drsapi.AccessMethod{{Type: drsapi.AccessMethodTypeS3, AccessUrl: &drsapi.AccessURL{Url: e.resolved}}} + obj.AccessMethods = &methods + objects = append(objects, obj) + } + defer func() { + if retErr != nil { + if gitOnly { + retErr = fmt.Errorf("Git-only import incomplete; repository files or a commit may already exist: %w", retErr) + } else { + retErr = fmt.Errorf("import incomplete; Syfon records or repository files may already have been written: %w", retErr) + } + } + }() + if gitOnly { + if err = copyrecords.VerifyObjects(cmd.Context(), runtime.Client.InternalAPI(), objects); err != nil { + return fmt.Errorf("verify existing Syfon records: %w", err) + } + fmt.Fprintf(cmd.OutOrStdout(), "Verified %d existing Syfon records\n", len(objects)) + if dry { + return nil + } + } else { + if err = copyrecords.RegisterObjects(cmd.Context(), runtime.Client.InternalAPI(), objects, org, project, logger); err != nil { + return fmt.Errorf("register Syfon records: %w", err) + } + fmt.Fprintf(cmd.OutOrStdout(), "Registered %d objects with Syfon\n", len(objects)) + } + if !commit { + return nil + } + paths, err := gitrepo.ResolveRepositoryPaths(cmd.Context()) + if err != nil { + return err + } + tracked := make([]string, 0, len(entries)) + for i, e := range entries { + destination := filepath.Join(root, filepath.FromSlash(e.destination)) + if err = os.MkdirAll(filepath.Dir(destination), 0755); err != nil { + return err + } + if err = writePointerFile(destination, e.sha, e.size, false); err != nil { + return err + } + tracked = append(tracked, e.destination) + if err = drsobject.WriteObject(paths.DRSObjectsDir(), objects[i], e.sha); err != nil { + return err + } + } + if err = gitrepo.TrackReadOnlyFiles(cmd.Context(), tracked); err != nil { + return err + } + gitargs := []string{"-c", "filter.drs.process=", "-c", "filter.drs.clean=cat", "-c", "filter.drs.required=false", "add", "--pathspec-from-file=-", "--pathspec-file-nul"} + var pathspec strings.Builder + pathspec.WriteString(".gitattributes\x00") + for _, e := range entries { + pathspec.WriteString(e.destination) + pathspec.WriteByte(0) + } + add := exec.CommandContext(cmd.Context(), "git", gitargs...) + add.Stdin = strings.NewReader(pathspec.String()) + add.Env = append(os.Environ(), "GIT_LITERAL_PATHSPECS=1") + if out, err := add.CombinedOutput(); err != nil { + return fmt.Errorf("git add: %s: %w", strings.TrimSpace(string(out)), err) + } + if _, err = run("-c", "filter.drs.process=", "-c", "filter.drs.clean=cat", "-c", "filter.drs.required=false", "-c", "core.hooksPath=/dev/null", "commit", "-m", fmt.Sprintf("Index %d existing objects", len(entries))); err != nil { + return err + } + head, err := run("rev-parse", "HEAD") + if err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Created commit %s on %s\n", head, branch) + if gitOnly { + push := exec.CommandContext(cmd.Context(), "git", "push") + push.Env = append(os.Environ(), "GIT_LFS_SKIP_PUSH=1") + if out, err := push.CombinedOutput(); err != nil { + return fmt.Errorf("commit %s created but Git push failed: %s: %w", head, strings.TrimSpace(string(out)), err) + } + fmt.Fprintln(cmd.OutOrStdout(), "Git push completed") + } else { + fmt.Fprintln(cmd.OutOrStdout(), "Use git push to publish it") + } + return nil +} diff --git a/cmd/addurl/manifest_import_test.go b/cmd/addurl/manifest_import_test.go new file mode 100644 index 00000000..1a7a5128 --- /dev/null +++ b/cmd/addurl/manifest_import_test.go @@ -0,0 +1,229 @@ +package addurl + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + internalapi "github.com/calypr/syfon/apigen/internalapi" +) + +func TestReadImportManifest(t *testing.T) { + file := filepath.Join(t.TempDir(), "input.tsv") + contents := "s3_path\trepo_path\tsha256\tsize_bytes\n" + "s3://source/a\tdir/a\t" + strings.Repeat("a", 64) + "\t45\n" + if err := os.WriteFile(file, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + rows, err := readImportManifest(file) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 || rows[0].destination != "dir/a" || rows[0].size != 45 { + t.Fatalf("unexpected rows: %+v", rows) + } + if err = os.WriteFile(file, []byte(strings.Replace(contents, "dir/a", "../escape", 1)), 0600); err != nil { + t.Fatal(err) + } + if _, err = readImportManifest(file); err == nil { + t.Fatal("expected path traversal rejection") + } +} + +func TestProbeManifestUsesScopeAndResolvedLocation(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path != "/data/inspect/bulk" { + t.Errorf("unexpected path %s", r.URL.Path) + } + var body internalapi.InternalInspectObjectsBulkRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + return + } + got := body.Items[0] + if got.Organization != "ExampleOrg" || got.Project != "ExampleProject" || got.Key != "study/a" || got.ExpectedSizeBytes == nil || *got.ExpectedSizeBytes != 45 { + t.Errorf("request: %+v", got) + } + _, _ = w.Write([]byte(`{"items":[{"id":"0","status":"present","exists":true,"size_match":true,"object_url":"s3://mapped-bucket/source-bucket/study/a"}]}`)) + })) + defer server.Close() + client, err := internalapi.NewClientWithResponses(server.URL) + if err != nil { + t.Fatal(err) + } + entries := []manifestEntry{{source: "s3://source-bucket/study/a", size: 45}} + if err = probeManifest(context.Background(), client, entries, "ExampleOrg", "ExampleProject", "s3://source-bucket/"); err != nil { + t.Fatal(err) + } + if entries[0].resolved != "s3://mapped-bucket/source-bucket/study/a" { + t.Fatalf("resolved %q", entries[0].resolved) + } +} + +func TestManifestImportCreatesRecordsAndCommit(t *testing.T) { + var created internalapi.BulkCreateRequest + var writes int + var lfsRequests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/data/inspect/bulk": + var req internalapi.InternalInspectObjectsBulkRequest + _ = json.NewDecoder(r.Body).Decode(&req) + _ = json.NewEncoder(w).Encode(map[string]any{"items": []map[string]any{{"id": req.Items[0].Id, "status": "present", "exists": true, "size_match": true, "object_url": "s3://bucket/prefix/a"}}}) + case "/index/bulk/documents": + _, _ = w.Write([]byte(`[]`)) + case "/index/bulk/hashes": + var req internalapi.BulkHashesRequest + _ = json.NewDecoder(r.Body).Decode(&req) + results := make(map[string][]internalapi.InternalRecord) + for _, hash := range req.Hashes { + if len(created.Records) > 0 { + results[hash] = created.Records + } + } + _ = json.NewEncoder(w).Encode(map[string]any{"results": results}) + case "/index/bulk": + writes++ + if err := json.NewDecoder(r.Body).Decode(&created); err != nil { + t.Error(err) + } + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"records":[]}`)) + case "/info/lfs/objects/batch": + lfsRequests++ + var req struct { + Objects []struct { + Oid string `json:"oid"` + Size int64 `json:"size"` + } `json:"objects"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + } + w.Header().Set("Content-Type", "application/vnd.git-lfs+json") + _ = json.NewEncoder(w).Encode(map[string]any{"transfer": "basic", "objects": req.Objects}) + case "/info/lfs/locks/verify": + w.WriteHeader(http.StatusNotFound) + default: + t.Errorf("unexpected endpoint %s", r.URL.Path) + http.NotFound(w, r) + } + })) + defer server.Close() + repo := setupGitRepo(t) + gitCmd(t, repo, "checkout", "-b", "import-test") + gitCmd(t, repo, "commit", "--allow-empty", "-m", "initial") + bare := filepath.Join(t.TempDir(), "remote.git") + gitCmd(t, repo, "init", "--bare", bare) + gitCmd(t, repo, "remote", "add", "origin", bare) + gitCmd(t, repo, "config", "remote.origin.lfsurl", server.URL+"/info/lfs") + if _, err := exec.LookPath("git-lfs"); err == nil { + hook := filepath.Join(repo, ".git", "hooks", "pre-push") + if err := os.WriteFile(hook, []byte("#!/bin/sh\nexec git lfs pre-push \"$@\"\n"), 0755); err != nil { + t.Fatal(err) + } + } + gitCmd(t, repo, "push", "-u", "origin", "import-test") + for _, args := range [][]string{{"config", "drs.default-remote", "test"}, {"config", "drs.remote.test.type", "local"}, {"config", "drs.remote.test.endpoint", server.URL}, {"config", "drs.remote.test.organization", "Org"}, {"config", "drs.remote.test.project", "Project"}, {"config", "drs.remote.test.bucket", "bucket"}} { + gitCmd(t, repo, args...) + } + gitCmd(t, repo, "config", "filter.drs.process", "false") + manifest := filepath.Join(t.TempDir(), "input.tsv") + sha := strings.Repeat("a", 64) + if err := os.WriteFile(manifest, []byte("s3_path\trepo_path\tsha256\tsize_bytes\nsource/a\tdir/a\t"+sha+"\t45\n"), 0600); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(repo, "dir"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, "dir/a"), []byte("existing"), 0644); err != nil { + t.Fatal(err) + } + before, err := exec.Command("git", "-C", repo, "rev-parse", "HEAD").Output() + if err != nil { + t.Fatal(err) + } + indexOnly := NewCommand() + indexOnly.SetArgs([]string{manifest, "--repo", repo}) + if err := indexOnly.Execute(); err != nil { + t.Fatal(err) + } + if len(created.Records) != 1 { + t.Fatalf("Syfon-only import did not register record: %+v", created) + } + data, err := os.ReadFile(filepath.Join(repo, "dir/a")) + if err != nil || string(data) != "existing" { + t.Fatalf("Syfon-only import changed file: %q: %v", data, err) + } + after, err := exec.Command("git", "-C", repo, "rev-parse", "HEAD").Output() + if err != nil || string(before) != string(after) { + t.Fatalf("Syfon-only import changed commit: %v", err) + } + if err := os.Remove(filepath.Join(repo, "dir/a")); err != nil { + t.Fatal(err) + } + command := NewCommand() + command.SetArgs([]string{manifest, "--repo", repo, "--git-only", "--branch", "import-test"}) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + if writes != 1 { + t.Fatalf("Git-only import changed Syfon records: %d writes", writes) + } + if lfsRequests != 0 { + t.Fatalf("Git-only import invoked LFS upload API %d times", lfsRequests) + } + if len(created.Records) != 1 || created.Records[0].Hashes == nil || (*created.Records[0].Hashes)["sha256"] != sha { + t.Fatalf("records: %+v", created) + } + methods := created.Records[0].AccessMethods + if methods == nil || (*methods)[0].AccessUrl.Url != "s3://bucket/prefix/a" { + t.Fatalf("access methods: %+v", methods) + } + show := exec.Command("git", "-C", repo, "show", "HEAD:dir/a") + out, err := show.CombinedOutput() + if err != nil { + t.Fatalf("git show: %v: %s", err, out) + } + if !strings.Contains(string(out), "version https://git-lfs.github.com/spec/v1") || !strings.Contains(string(out), "oid sha256:"+sha) || !strings.Contains(string(out), "size 45") { + t.Fatalf("committed pointer: %s", out) + } + if _, err := exec.LookPath("git-lfs"); err == nil { + cmd := exec.Command("git", "-C", repo, "lfs", "ls-files", "-n") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git lfs ls-files: %v: %s", err, out) + } + if !strings.Contains(string(out), "dir/a") { + t.Fatalf("Git LFS does not recognize the imported pointer: %s", out) + } + } + verify := exec.Command("git", "--git-dir", bare, "show", "refs/heads/import-test:dir/a") + remotePointer, err := verify.CombinedOutput() + if err != nil || !strings.Contains(string(remotePointer), "oid sha256:"+sha) { + t.Fatalf("remote pointer: %s: %v", remotePointer, err) + } +} + +func TestProbeManifestRejectsSizeMismatch(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"items":[{"id":"0","status":"present","exists":true,"size_match":false,"object_url":"s3://bucket/a"}]}`)) + })) + defer server.Close() + client, err := internalapi.NewClientWithResponses(server.URL) + if err != nil { + t.Fatal(err) + } + entries := []manifestEntry{{source: "s3://bucket/a", size: 45}} + if err := probeManifest(context.Background(), client, entries, "org", "project", ""); err == nil { + t.Fatal("expected size mismatch to fail") + } +} diff --git a/cmd/copyrecords/import.go b/cmd/copyrecords/import.go new file mode 100644 index 00000000..7f50b494 --- /dev/null +++ b/cmd/copyrecords/import.go @@ -0,0 +1,104 @@ +package copyrecords + +import ( + "context" + "fmt" + "log/slog" + "strings" + + drsapi "github.com/calypr/syfon/apigen/drs" + internalapi "github.com/calypr/syfon/apigen/internalapi" +) + +// RegisterObjects merges existing records by DID or checksum and registers them in batches. +func RegisterObjects(ctx context.Context, client *internalapi.ClientWithResponses, objects []*drsapi.DrsObject, organization, project string, logger *slog.Logger) error { + records := make([]copyRecord, 0, len(objects)) + for _, obj := range objects { + record := copyRecordFromLocalObject(obj) + record.CreatedTime = nil + record.Organization = &organization + record.Project = &project + records = append(records, record) + } + api := newRawIndexAPI(client) + if _, err := copyProjectRecordsWithOptions(ctx, logger, records, api, organization, project, defaultCopyBatchSize, false, false); err != nil { + return err + } + for start := 0; start < len(objects); start += defaultCopyBatchSize { + end := min(start+defaultCopyBatchSize, len(objects)) + queries := make([]string, 0, end-start) + for _, record := range records[start:end] { + queries = append(queries, "sha256:"+copyRecordSHA256(record)) + } + result, err := api.BulkHashes(ctx, queries) + if err != nil { + return fmt.Errorf("verify registered records: %w", err) + } + for i, query := range queries { + matches := dedupeCopyRecordsByDID(result.Results[query]) + var id string + for _, match := range matches { + if match.Did == records[start+i].Did { + id = match.Did + break + } + } + if id == "" && len(matches) == 1 { + id = matches[0].Did + } + if strings.TrimSpace(id) == "" { + return fmt.Errorf("cannot resolve unique registered record for %s", query) + } + objects[start+i].Id = id + } + } + return nil +} + +// VerifyObjects confirms existing records match the requested locations and scope without writing them. +func VerifyObjects(ctx context.Context, client *internalapi.ClientWithResponses, objects []*drsapi.DrsObject) error { + api := newRawIndexAPI(client) + for start := 0; start < len(objects); start += defaultCopyBatchSize { + end := min(start+defaultCopyBatchSize, len(objects)) + queries := make([]string, 0, end-start) + for _, obj := range objects[start:end] { + queries = append(queries, "sha256:"+copyRecordSHA256(copyRecordFromLocalObject(obj))) + } + result, err := api.BulkHashes(ctx, queries) + if err != nil { + return err + } + for i, query := range queries { + wanted := objects[start+i] + found := false + for _, match := range result.Results[query] { + if match.Size == nil || *match.Size != wanted.Size || match.AccessMethods == nil || wanted.AccessMethods == nil || wanted.ControlledAccess == nil || match.ControlledAccess == nil { + continue + } + scopeFound, urlFound := false, false + for _, scope := range *wanted.ControlledAccess { + for _, existing := range *match.ControlledAccess { + if scope == existing { + scopeFound = true + } + } + } + for _, method := range *wanted.AccessMethods { + for _, existing := range *match.AccessMethods { + if method.AccessUrl != nil && existing.AccessUrl != nil && method.AccessUrl.Url == existing.AccessUrl.Url { + urlFound = true + } + } + } + if scopeFound && urlFound { + found = true + break + } + } + if !found { + return fmt.Errorf("no existing record with matching size, scope and storage location for %s", query) + } + } + } + return nil +} diff --git a/cmd/pull/main.go b/cmd/pull/main.go index 93e9e332..94f4466b 100644 --- a/cmd/pull/main.go +++ b/cmd/pull/main.go @@ -239,7 +239,7 @@ var Cmd = &cobra.Command{ for _, oid := range missingOIDs { if lfs.IsDRSURI(oid) { if anvil == nil { - obj, err := drsCtx.Client.DRS().GetObject(ctx, normalizeDRSPointerOID(oid)) + obj, err := internaltransfer.GetObjectForDRSURI(ctx, drsCtx, normalizeDRSPointerOID(oid)) if err != nil { return fmt.Errorf("look up DRS record %s: %w", oid, err) } diff --git a/docs/adding-s3-files.md b/docs/adding-s3-files.md index a99628de..b388b97a 100644 --- a/docs/adding-s3-files.md +++ b/docs/adding-s3-files.md @@ -1,5 +1,7 @@ # Adding Provider Objects with `git drs add-url` +> The single-object and Globus `add-url` examples on this page describe the previous command. See [the current manifest command](commands.md#git-drs-add-url-manifesttsv). + `git drs add-url` prepares a Git pointer plus local DRS metadata for an object that already exists in provider storage. Important behavior: diff --git a/docs/commands.md b/docs/commands.md index 11436e46..e16686d4 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -379,41 +379,75 @@ Notes: For details on pointer file formats and lifecycle state, see [Pointer Files and Reference State](pointer-files.md). -### `git drs add-url [path]` +### `git drs add-url ` -Create a pointer plus local DRS metadata for an object that already exists in provider storage. +Index existing S3 objects in Syfon. Run the command from a repository with a +configured git-drs remote, or select one with `--repo`. The remote must specify +an organization and project. ```bash -git drs add-url path/to/object.bin data/from-bucket.bin --scheme s3 -git drs add-url s3://my-bucket/path/to/object.bin data/from-bucket.bin -git drs add-url s3://my-bucket/path/to/object.bin data/from-bucket.bin --sha256 -git drs add-url globus:///project/sample.bam data/sample.bam -git drs add-url globus:///project/ data/project -git drs add-url 'globus:///project/**/*.bam' data/bam --dry-run +git drs add-url files.tsv --remote origin ``` -Notes: +The tab-separated manifest needs `s3_path`, `repo_path`, `sha256`, and +`size_bytes` columns. `s3_path` can be a physical `s3://bucket/key` URL or a +key relative to the configured project scope. Extra columns are ignored. +Paths are relative to the repository root. SHA-256 values must be unique. + +Syfon probes objects in batches and checks their sizes. The command trusts the +manifest's checksums; it does not download objects or verify their hashes. +To also create standard Git LFS pointers and a Git commit, add `--commit --branch main`. +The selected branch must already be checked out and the working tree must be +clean. Syfon records are registered first. Use plain `git push` to publish the +resulting commit. Your Git LFS pre-push hook must be configured to use a Syfon +endpoint that recognizes the indexed objects. + +```bash +git drs add-url files.tsv --repo /path/to/repository --remote origin \ + --commit --branch main +``` + +#### Resolve manifest URLs through a scope + +By default, an `s3_path` that starts with `s3://` is a physical storage URL. +Syfon probes that bucket and key directly. A value without a scheme is a key +relative to the remote's organization/project scope; Syfon applies that scope's +bucket and prefix before probing. + +Use `--strip-prefix` when every `s3_path` starts with the same prefix but the +remaining path should be resolved through the scope. The option removes the +exact, case-sensitive prefix from every row. If any row does not start with +it, the command fails. For example, with this fictional manifest row: + +```text +s3_path repo_path sha256 size_bytes +s3://source-bucket/study/sample.bam data/sample.bam aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 12345 +``` + +This command removes `s3://source-bucket/`, leaving the scoped key +`study/sample.bam`: + +```bash +git drs add-url files.tsv --remote origin --strip-prefix s3://source-bucket/ +``` + +If the server maps the scope to `s3://mapped-bucket/project-root/`, Syfon +resolves that key to `s3://mapped-bucket/project-root/study/sample.bam`. The +mapping belongs to the server; `--strip-prefix` does not create or change it. +The checksum and size above are placeholders, not values for a real file. + +Use `--git-only --branch main` when the Syfon records already exist. This +checks their checksum, size, scope, and storage location, creates the pointer +commit, and runs `git push` with the Git LFS upload hook skipped. It does not +write Syfon records or upload payloads. The branch needs a Git upstream. If +the push fails, the commit remains local; retry with +`GIT_LFS_SKIP_PUSH=1 git push`. + +The command prints the selected remote endpoint and scope before probing. -- object-key mode resolves against the configured bucket scope -- explicit provider URL mode remains supported -- `--scheme` is required for object-key mode -- when `--sha256` is omitted, the pointer uses a derived local/cache OID and source URL metadata remains the retrieval identity -- `globus://` URLs are detected automatically; a directory or wildcard import - does not require `--recursive` or `--manifest` -- `--recursive` is deprecated and accepted only for compatibility -- quote wildcard URLs; `*`, `?`, `[...]`, and recursive `**` are supported -- Globus files without SHA-256 use a marked placeholder derived from URL, size, - and `last_modified`; hydration validates size rather than treating it as a - content hash -- `--manifest` optionally supplies authoritative `path`, `size`, and `sha256` - values for a directory or wildcard selection -- `--dry-run` prints planned Globus members without changing the worktree; - `--remote` selects the DRS registration scope -- collection import writes one DRS pointer and local DRS object per member, plus - one `destination/**` read-only rule in `.gitattributes` -- registration happens later on `git drs push` - -See [Adding Globus Objects](globus-add-url.md) for wildcard and identity details. +Use `--dry-run` to probe without creating records or a commit. A failed import +can leave registered records or local files; inspect the output and repository +before retrying. ### `git drs add-ref ` @@ -464,11 +498,11 @@ Remote behavior on push: Copy Syfon metadata records from one configured remote to another for one scope. ```bash -git drs copy-records prod HTAN_INT/BForePC -git drs copy-records dev prod HTAN_INT/BForePC -git drs copy-records @local prod HTAN_INT/BForePC -git drs copy-records @local dev HTAN_INT/BForePC --include-path META --include-path CONFIG -git drs copy-records local dev HTAN_INT/BForePC --include-path META --include-path CONFIG +git drs copy-records prod example-org/example-project +git drs copy-records dev prod example-org/example-project +git drs copy-records @local prod example-org/example-project +git drs copy-records @local dev example-org/example-project --include-path META --include-path CONFIG +git drs copy-records local dev example-org/example-project --include-path META --include-path CONFIG ``` Behavior: @@ -511,8 +545,8 @@ Map an organization to a bucket path. ```bash git drs bucket add-organization production \ - --organization HTAN_INT \ - --path s3://cbds/htan-int + --organization example-org \ + --path s3://example-bucket/example-prefix ``` ### `git drs bucket add-project` @@ -521,9 +555,9 @@ Map a project to a bucket path. ```bash git drs bucket add-project production \ - --organization HTAN_INT \ - --project BForePC \ - --path s3://cbds/htan-int/bforepc + --organization example-org \ + --project example-project \ + --path s3://example-bucket/example-prefix ``` ## Version diff --git a/docs/e2e-modes-and-local-setup.md b/docs/e2e-modes-and-local-setup.md index 593f04e5..1b28bc03 100644 --- a/docs/e2e-modes-and-local-setup.md +++ b/docs/e2e-modes-and-local-setup.md @@ -1,5 +1,7 @@ # E2E Modes And Local Setup +> The single-object and Globus `add-url` examples on this page describe the previous command. See [the current manifest command](commands.md#git-drs-add-url-manifesttsv). + This guide documents how `git-drs` test suites run in: - `remote` mode (Gen3 token auth) diff --git a/docs/globus-add-url.md b/docs/globus-add-url.md index a1344fec..fd80da6a 100644 --- a/docs/globus-add-url.md +++ b/docs/globus-add-url.md @@ -1,5 +1,7 @@ # Adding Globus Objects with `git drs add-url` +> The single-object and Globus `add-url` examples on this page describe the previous command. See [the current manifest command](commands.md#git-drs-add-url-manifesttsv). + `git drs add-url` creates Git pointers and local DRS metadata for files that already exist in a Globus collection. It does not stage or download source files during import. diff --git a/docs/index.md b/docs/index.md index be55ecc3..d929f945 100644 --- a/docs/index.md +++ b/docs/index.md @@ -20,8 +20,7 @@ resolve common problems. commits in place of large payloads. - [Add Existing S3 Objects](adding-s3-files.md) — create references for data that is already in provider storage. -- [Add Existing Globus Objects](globus-add-url.md) — import one file, a - collection tree, or a wildcard selection without downloading it first. +- [Index Existing S3 Objects](commands.md#git-drs-add-url-manifesttsv) — import a TSV manifest. - [Globus Access Methods](globus.md) — log in with Globus Auth, configure a destination collection, and use Globus-backed hydration. - [Access-Method Selection](access-method-selection-and-authentication.md) — diff --git a/docs/pointer-files.md b/docs/pointer-files.md index 2aa4e1b9..06dc6023 100644 --- a/docs/pointer-files.md +++ b/docs/pointer-files.md @@ -1,5 +1,7 @@ # Pointer Files and Reference State +> The single-object and Globus `add-url` examples on this page describe the previous command. See [the current manifest command](commands.md#git-drs-add-url-manifesttsv). + `git-drs` keeps large payload bytes out of Git history by committing small pointer files. A pointer records enough identity to let `git-drs` find or rebuild the payload later, while the payload itself lives in local cache, provider storage, or behind a DRS server. This page documents the pointer shapes used by reference-first workflows and how state changes as commands create references, download content, and obtain temporary access URLs (the resolved "real" download URLs used for the transfer). diff --git a/internal/gitrepo/gitattributes_tracking.go b/internal/gitrepo/gitattributes_tracking.go index b5816a66..301dbfc3 100644 --- a/internal/gitrepo/gitattributes_tracking.go +++ b/internal/gitrepo/gitattributes_tracking.go @@ -157,6 +157,24 @@ func TrackReadOnly(ctx context.Context, path string) (bool, error) { return trackReadOnly(ctx, path, true) } +// TrackReadOnlyFiles tracks literal repository paths in a single attributes update. +func TrackReadOnlyFiles(ctx context.Context, paths []string) error { + root, err := GitTopLevel() + if err != nil { + return err + } + attributes := filepath.Join(root, ".gitattributes") + if _, err := trackPatternsAtPath(paths, false, false, attributes, true); err != nil { + return err + } + encoded := make([]string, 0, len(paths)) + for _, p := range paths { + encoded = append(encoded, escapeLiteralAttrPattern(p)) + } + _, err = UpsertDRSRouteLines(attributes, "ro", encoded) + return err +} + func TrackReadOnlyPattern(ctx context.Context, pattern string) (bool, error) { return trackReadOnly(ctx, pattern, false) } diff --git a/internal/transfer/download.go b/internal/transfer/download.go index 16e19fe5..fe50fbf1 100644 --- a/internal/transfer/download.go +++ b/internal/transfer/download.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "path/filepath" "strings" @@ -62,7 +63,7 @@ func ResolvedAccessURLForDRSURI(ctx context.Context, drsCtx *remoteruntime.GitCo if strings.HasPrefix(drsURI, "//") { drsURI = "drs:" + drsURI } - obj, err := drsCtx.Client.DRS().GetObject(ctx, drsURI) + obj, err := GetObjectForDRSURI(ctx, drsCtx, drsURI) if err != nil { return ResolvedAccess{}, nil, err } @@ -76,6 +77,19 @@ func ResolvedAccessURLForDRSURI(ctx context.Context, drsCtx *remoteruntime.GitCo return access, &obj, nil } +// GetObjectForDRSURI resolves local Syfon DRS URIs by their object ID. +func GetObjectForDRSURI(ctx context.Context, drsCtx *remoteruntime.GitContext, drsURI string) (drsapi.DrsObject, error) { + if strings.HasPrefix(drsURI, "//") { + drsURI = "drs:" + drsURI + } + uri, uriErr := url.Parse(drsURI) + endpoint, endpointErr := url.Parse(drsCtx.Endpoint) + if uriErr == nil && endpointErr == nil && strings.EqualFold(uri.Scheme, "drs") && endpoint.Host != "" && strings.EqualFold(uri.Host, endpoint.Host) && strings.Trim(uri.Path, "/") != "" { + drsURI = strings.Trim(uri.Path, "/") + } + return drsCtx.Client.DRS().GetObject(ctx, drsURI) +} + func DownloadDRSURIToCachePath(ctx context.Context, drsCtx *remoteruntime.GitContext, drsURI, cachePath string) error { if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil { return fmt.Errorf("mkdir for cache path: %w", err) diff --git a/internal/transfer/download_test.go b/internal/transfer/download_test.go index 526583f5..d9e290f6 100644 --- a/internal/transfer/download_test.go +++ b/internal/transfer/download_test.go @@ -672,3 +672,18 @@ func TestDownloadResolvedToPathRefreshesSelectedAccessWithHeaders(t *testing.T) t.Fatalf("retry Authorization = %q, want refreshed header", retryHeader) } } + +func TestSyfonDRSURIUsesObjectID(t *testing.T) { + var requestPath string + client, err := syclient.New("http://example.test", syclient.WithHTTPClient(&http.Client{Transport: downloadRoundTripFunc(func(r *http.Request) (*http.Response, error) { requestPath = r.URL.Path; return nil, io.EOF })})) + if err != nil { + t.Fatal(err) + } + _, err = GetObjectForDRSURI(context.Background(), &remoteruntime.GitContext{Client: client, Endpoint: "http://example.test"}, "drs://example.test/object-1") + if err == nil { + t.Fatal("expected fixture transport error") + } + if !strings.HasSuffix(requestPath, "/objects/object-1") { + t.Fatalf("request path = %q", requestPath) + } +} diff --git a/mkdocs.yml b/mkdocs.yml index e8287f64..7fc3ada9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -26,7 +26,6 @@ nav: - Work with data: - Pointer Files and Reference State: pointer-files.md - Add Existing S3 Objects: adding-s3-files.md - - Add Existing Globus Objects: globus-add-url.md - Globus Access Methods: globus.md - Access-Method Selection: access-method-selection-and-authentication.md - Remove Files: remove-files.md