Optimize verified DRS downloads and cache checkout

.github/workflows/pr-checks.yaml

@@ -75,3 +75,46 @@ jobs:
 
     - name: Run tests
       run: go test -v -race ./...
+
+  terra-drs-hub:
+    name: Resolve through DRS Hub and TDR
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    steps:
+    - name: Check out git-drs
+      uses: actions/checkout@v4
+
+    - name: Check out TDR
+      uses: actions/checkout@v4
+      with:
+        repository: DataBiosphere/jade-data-repo
+        ref: 6433cb4ea296edf873daa30f1da29e7caf848b8b
+        path: tdr
+        persist-credentials: false
+
+    - name: Check out DRS Hub
+      uses: actions/checkout@v4
+      with:
+        repository: DataBiosphere/terra-drs-hub
+        ref: 618348c2efeb61d3fb50fd75c447d6ebf3f4192d
+        path: drs-hub
+        persist-credentials: false
+
+    - name: Set up Java
+      uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: '17'
+        cache: gradle
+        cache-dependency-path: |
+          tdr/build.gradle
+          drs-hub/service/build.gradle
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: '1.26.6'
+
+    - name: Pull an AnVIL DRS pointer through DRS Hub and TDR
+      run: tests/terra-drs-hub/run.sh "$GITHUB_WORKSPACE/tdr" "$GITHUB_WORKSPACE/drs-hub"

README.md

@@ -43,7 +43,7 @@ At a high level:
 ```bash
 git drs install
 git drs remote add production https://example-gen3.org --provider gen3 \
-  --scope HTAN_INT/BForePC --auth provider-helper:gen3-profile \
+  --scope example-org/example-project --auth provider-helper:gen3-profile \
   --credential file:/path/to/credentials.json
 git drs track "*.bam"
 git add .gitattributes
@@ -72,7 +72,7 @@ The cleaned CLI intentionally removed legacy commands:
 Example:
 
 ```bash
-git drs remote add production calypr --scope HTAN_INT/BForePC \
+git drs remote add production calypr --scope example-org/example-project \
   --credential file:/path/to/credentials.json
 ```
 
@@ -83,7 +83,7 @@ Current command split:
 - `git drs pull` hydrates tracked pointer files already present in the checkout
 - `git drs ls-files` lists visible files and directories; use `--all` for
   the tracked DRS/LFS pointer inventory
-- `git drs add-url` prepares pointer plus local metadata for existing provider objects
+- `git drs add-url` indexes existing S3 objects from a TSV manifest, with optional pointer commits
 
 ## Bucket Mapping Model
 
@@ -108,7 +108,7 @@ Push and pull depend on server-side bucket mapping for the requested scope. That
 | `git drs ls-files` | List visible files and directories |
 | `git drs pull` | Hydrate pointer files in the current checkout |
 | `git drs push` | Register/upload objects and push refs |
-| `git drs add-url` | Add an existing provider object by URL or scoped key |
+| `git drs add-url` | Index existing S3 objects from a TSV manifest |
 | `git drs add-ref` | Add a local reference to an existing DRS object |
 | `git drs query` | Query a DRS object by ID |
 | `git drs copy-records` | Copy Syfon records between remotes for one scope |
@@ -119,7 +119,6 @@ Push and pull depend on server-side bucket mapping for the requested scope. That
 - [Getting Started](docs/getting-started.md)
 - [Commands Reference](docs/commands.md)
 - [Access-Method Selection](docs/access-method-selection-and-authentication.md)
-- [Adding Globus Objects](docs/globus-add-url.md)
 - [Troubleshooting](docs/troubleshooting.md)
 - [Developer Guide](docs/developer-guide.md)
 - [Publishing documentation with GitHub Pages](docs/github-pages.md)

cmd/addref/add-ref.go

@@ -333,7 +333,13 @@ type drsObjectGetter interface {
 
 func resolveAddRefObject(ctx context.Context, cfg *config.Config, primaryRemote config.Remote, primary *remoteruntime.GitContext, drsURI string) (drsapi.DrsObject, error) {
 	if primary != nil && primary.RemoteType == config.TerraServerType {
-		anvil, err := newAnVILResolver(ctx, primary.Endpoint)
+		var anvil resolver.Resolver
+		var err error
+		if primary.HubEndpoint != "" {
+			anvil, err = newHubResolver(ctx, primary.HubEndpoint)
+		} else {
+			anvil, err = newAnVILResolver(ctx, primary.Endpoint)
+		}
 		if err != nil {
 			return drsapi.DrsObject{}, err
 		}
@@ -376,6 +382,10 @@ var newAnVILResolver = func(ctx context.Context, endpoint string) (resolver.Reso
 	return resolver.NewAnVIL(ctx, endpoint)
 }
 
+var newHubResolver = func(ctx context.Context, endpoint string) (resolver.Resolver, error) {
+	return resolver.NewTerraHub(ctx, endpoint)
+}
+
 func parseDRSURIForSource(drsURI string) (objectID string, endpoint string, ok bool) {
 	u, err := url.Parse(strings.TrimSpace(drsURI))
 	if err != nil || !strings.EqualFold(u.Scheme, "drs") || strings.TrimSpace(u.Host) == "" {

cmd/addurl/main.go

@@ -1,30 +1,27 @@
 package addurl
 
-import (
-	"errors"
-
-	"github.com/spf13/cobra"
-)
+import "github.com/spf13/cobra"
 
 var Cmd = NewCommand()
 
 // NewCommand constructs the Cobra command for the `add-url` subcommand,
 // wiring usage, argument validation and the RunE handler.
 func NewCommand() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "add-url <object-url-or-key> [path]",
-		Short: "Add a file from a provider URL or configured bucket object key",
-		Args: func(cmd *cobra.Command, args []string) error {
-			if len(args) < 1 || len(args) > 2 {
-				return errors.New("usage: add-url <object-url-or-key> [path]")
-			}
-			return nil
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			return NewAddURLService().Run(cmd, args)
-		},
+		Use:   "add-url <manifest.tsv>",
+		Short: "Index existing S3 objects from a TSV manifest",
+		Args:  cobra.ExactArgs(1),
+		RunE:  runManifest,
 	}
 	addFlags(cmd)
+	cmd.Flags().String("repo", ".", "Repository containing the git-drs remote configuration")
+	cmd.Flags().Bool("commit", false, "Also create and commit Git pointers")
+	cmd.Flags().Bool("git-only", false, "Verify existing Syfon records, commit pointers, and push the Git branch")
+	cmd.Flags().String("branch", "", "Required current branch when using --commit")
+	cmd.Flags().String("strip-prefix", "", "Remove this source prefix and resolve the remainder in the configured project scope")
+	for _, name := range []string{"sha256", "scheme", "recursive", "manifest"} {
+		_ = cmd.Flags().MarkHidden(name)
+	}
 	return cmd
 }
 
@@ -42,7 +39,7 @@ func addFlags(cmd *cobra.Command) {
 	)
 	cmd.Flags().Bool("recursive", false, "Deprecated: Globus directories and wildcards are detected automatically")
 	_ = cmd.Flags().MarkDeprecated("recursive", "Globus directories and wildcards are detected automatically")
-	cmd.Flags().Bool("dry-run", false, "List and validate matching Globus objects without writing files")
+	cmd.Flags().Bool("dry-run", false, "Probe and validate the manifest without writing records or a commit")
 	cmd.Flags().String("manifest", "", "Optional TSV manifest supplying path, size, and sha256 for Globus collections")
 	cmd.Flags().StringP("remote", "r", "", "Target DRS remote")
 }