diff --git a/.github/ai/AGENTS.md b/.github/ai/AGENTS.md index 7f04760a..b5c76008 100644 --- a/.github/ai/AGENTS.md +++ b/.github/ai/AGENTS.md @@ -1,5 +1,13 @@ # Akoflow Agent Guide +## Required Contribution Workflow + +- Every repository change follows `ISSUE -> BRANCH -> PR`. +- Create or identify the GitHub issue before implementation. +- Create a dedicated branch linked to that issue; do not implement directly on `main`. +- Open a pull request from that branch and link it to the issue. +- Do not bypass this sequence unless the user explicitly instructs otherwise for a specific change. + ## Core Shape - Go project with CLI and server binaries. diff --git a/.github/workflows/docs-checks.yaml b/.github/workflows/docs-checks.yaml index 9f218224..0fc35c02 100644 --- a/.github/workflows/docs-checks.yaml +++ b/.github/workflows/docs-checks.yaml @@ -54,6 +54,8 @@ jobs: run: npm ci --prefix docs - name: Type-check documentation code run: npm run typecheck --prefix docs + - name: Test API handler contract extraction + run: npm run test:api-generator --prefix docs - name: Build the documentation site run: npm run build --prefix docs - name: Verify repository-owned documentation links diff --git a/.gitignore b/.gitignore index 49023daa..80d4dc03 100755 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ db.sqlite +/server storage/database.sqlite storage/* */.DS_Store diff --git a/AGENTS.md b/AGENTS.md index 48737a74..ee0460f2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,19 @@ # AkôFlow repository guidance +## Auxiliary tooling language + +- Use Go for repository tooling, scientific analysis, calibration, data exports, and their tests. +- Do not introduce new Ruby scripts or require Ruby to run these tools. Maintain or migrate existing functionality in Go while preserving scientific contracts, provenance, and safety checks. +- This is the user's persistent repository preference; do not replace Go tooling with Ruby in future work. + +## Contribution workflow + +- Every repository change follows `ISSUE -> BRANCH -> PR`. +- Create or identify the GitHub issue before implementation. +- Create a dedicated branch linked to that issue; do not implement directly on `main`. +- Open a pull request from that branch and link it to the issue. +- Do not bypass this sequence unless the user explicitly instructs otherwise for a specific change. + ## Documentation diagrams - Use Mermaid for architecture, lifecycle, hierarchy, relationship, state, and flow diagrams in the documentation. diff --git a/cmd/montage-hpc-budget/main.go b/cmd/montage-hpc-budget/main.go new file mode 100644 index 00000000..c112e642 --- /dev/null +++ b/cmd/montage-hpc-budget/main.go @@ -0,0 +1,84 @@ +// montage-hpc-budget versions simulation inputs after an explicit user decision. +// It never changes a live catalog, provisions resources, or submits executions. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "time" +) + +func removeCap(request map[string]any, at string) (int, error) { + run, ok := request["Run"].(map[string]any) + if !ok || run["mode"] != "simulation" { + return 0, fmt.Errorf("simulation-only request required") + } + resources, ok := request["Resources"].([]any) + if !ok { + return 0, fmt.Errorf("resources missing") + } + count := 0 + for _, raw := range resources { + r, ok := raw.(map[string]any) + if !ok || r["type"] != "hpc_machine" { + continue + } + m, ok := r["metadata"].(map[string]any) + if !ok { + m = map[string]any{} + r["metadata"] = m + } + previous := m["workspaceStorageBudget"] + delete(m, "workspaceStorageBudget") + m["workspaceStorageBudgetDecision"] = map[string]any{ + "status": "cap-removed", "source": "explicit-user-decision-20261011", + "recordedAt": at, "previousBudget": previous, + "meaning": "No enforced HPC storage cap in this forecast; not a measurement of unlimited physical capacity.", + } + count++ + } + if count == 0 { + return 0, fmt.Errorf("no HPC resources") + } + return count, nil +} + +func main() { + in := flag.String("request", "", "existing simulation input") + out := flag.String("output", "", "new input; refuses overwrite") + flag.Parse() + err := func() error { + data, err := os.ReadFile(*in) + if err != nil { + return err + } + var request map[string]any + if err := json.Unmarshal(data, &request); err != nil { + return err + } + n, err := removeCap(request, time.Now().UTC().Format(time.RFC3339)) + if err != nil { + return err + } + data, err = json.MarshalIndent(request, "", " ") + if err != nil { + return err + } + f, err := os.OpenFile(*out, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return err + } + defer f.Close() + if _, err := f.Write(data); err != nil { + return err + } + fmt.Printf("Versioned simulation input: %d HPC caps explicitly removed; other resources unchanged.\n", n) + return nil + }() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cmd/montage-hpc-budget/main_test.go b/cmd/montage-hpc-budget/main_test.go new file mode 100644 index 00000000..fe60f2c5 --- /dev/null +++ b/cmd/montage-hpc-budget/main_test.go @@ -0,0 +1,25 @@ +package main + +import ( + "github.com/stretchr/testify/require" + "testing" +) + +func TestRemoveOnlyHPCCapAndPreserveDecision(t *testing.T) { + hpc := map[string]any{"type": "hpc_machine", "metadata": map[string]any{"workspaceStorageBudget": 123, "runtime": "sif"}} + cloud := map[string]any{"type": "cloud_vm", "metadata": map[string]any{"workspaceStorageBudget": 456}} + r := map[string]any{"Run": map[string]any{"mode": "simulation"}, "Resources": []any{hpc, cloud}} + n, err := removeCap(r, "2026-10-11") + require.NoError(t, err) + require.Equal(t, 1, n) + m := hpc["metadata"].(map[string]any) + require.NotContains(t, m, "workspaceStorageBudget") + require.Equal(t, "sif", m["runtime"]) + require.Equal(t, 123, m["workspaceStorageBudgetDecision"].(map[string]any)["previousBudget"]) + require.Equal(t, 456, cloud["metadata"].(map[string]any)["workspaceStorageBudget"]) +} + +func TestRejectReal(t *testing.T) { + _, err := removeCap(map[string]any{"Run": map[string]any{"mode": "real"}}, "now") + require.ErrorContains(t, err, "simulation-only") +} diff --git a/cmd/montage-storage-check/main.go b/cmd/montage-storage-check/main.go new file mode 100644 index 00000000..a717c5d8 --- /dev/null +++ b/cmd/montage-storage-check/main.go @@ -0,0 +1,63 @@ +// Offline storage bound diagnostic. No API submission or filesystem deletion. +package main + +import ( + "encoding/json" + "fmt" + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain/storagebudget" + "github.com/UFFeScience/akoflow/internal/domain/workflow" + "os" +) + +func main() { + if len(os.Args) != 2 { + panic("request path required") + } + b, err := os.ReadFile(os.Args[1]) + if err != nil { + panic(err) + } + var r ports.ExecutionRequest + if err = json.Unmarshal(b, &r); err != nil { + panic(err) + } + for i := range r.Resources { + if string(r.Resources[i].Type) != "hpc_machine" { + continue + } + m := r.Resources[i].Metadata + decision, _ := m["workspaceStorageBudgetDecision"].(map[string]any) + if previous := decision["previousBudget"]; previous != nil { + m["workspaceStorageBudget"] = previous + } + m["workspaceStorageLifecycle"] = "consumer-barrier-v1" + } + t := []storagebudget.Timing{} + for _, a := range r.Plan.Assignments { + t = append(t, storagebudget.Timing{ActivityID: a.ActivityID, ResourceID: a.ResourceID, Start: a.PredictedStartAt, Finish: a.PredictedFinishAt}) + } + rows, missing, err := storagebudget.CheckScheduled(r.Workflow, r.Resources, t) + result := map[string]any{"reservations": rows, "missing": missing} + types := map[string]string{} + for _, a := range r.Plan.Assignments { + for _, resource := range r.Resources { + if resource.ID == a.ResourceID { + types[a.ActivityID] = string(resource.Type) + } + } + } + projected, projectionErr := workflow.ProjectWorkspaceDependenciesForResourceTypes(r.Workflow, types) + if projectionErr == nil { + volumes := []map[string]any{} + for _, a := range projected.Activities { + volumes = append(volumes, map[string]any{"activityId": a.ID, "volumes": a.Metadata["workspaceProjectionVolumes"]}) + } + result["projectedVolumes"] = volumes + } + if err != nil { + result["error"] = err.Error() + } + b, _ = json.MarshalIndent(result, "", " ") + fmt.Println(string(b)) +} diff --git a/cmd/montage-tools/README.md b/cmd/montage-tools/README.md new file mode 100644 index 00000000..7238ebb2 --- /dev/null +++ b/cmd/montage-tools/README.md @@ -0,0 +1,43 @@ +# Montage tooling in Go + +Run from the repository root: + +```sh +go run ./cmd/montage-tools +go test -race ./internal/montagetools ./cmd/montage-tools +``` + +Alternatively, build once with `go build -o /tmp/akoflow-montage-tools ./cmd/montage-tools`. + +The migrated commands preserve the positional arguments of the former scripts; +use the former basename without `.rb` as the command: + +- `audit_montage_results` +- `build_montage_execution_calibration` +- `build_montage_workspace_projection` +- `build_montage_work_features` +- `extract_montage_compute_work` +- `freeze_montage_compute_profiles` +- `freeze_montage_materialization` +- `prepare_montage_measured_requests` +- `supplement_montage_root_compute` +- `validate_montage_transfer_model` +- `export_cloud_workspace` + +Missing or invalid arguments produce a usage error and nonzero exit status. +Validation modes are `calibrate`, `supplement`, `request`, `compare`, +`shared-capacity`, and `report`. Request generation accepts simulation requests +only; it does not submit a real execution. + +Scientific JSON exports are private (0600). Historical evidence and predictions +are not rewritten. Newly serialized JSON may have a different SHA256 because Go +orders object keys deterministically; source-file digests remain provenance of +the original input bytes. Numerical equivalence tests allow only floating-point +rounding tolerance, not changes to routes, volumes, or model contracts. + +Only the explicitly requested cloud workspace export calls the daemon API, +using `AKOFLOW_API_TOKEN` without printing it. It verifies chunk lengths and the +archive checksum and refuses to overwrite an existing archive. Compute-input +extraction reads an existing container image via Docker; it does not execute a +workflow. None of these tools accesses the database directly or restarts the +daemon. Ruby is no longer required for these repository tools or their tests. diff --git a/cmd/montage-tools/main.go b/cmd/montage-tools/main.go new file mode 100644 index 00000000..0dcf4026 --- /dev/null +++ b/cmd/montage-tools/main.go @@ -0,0 +1,14 @@ +package main + +import ( + "fmt" + "github.com/UFFeScience/akoflow/internal/montagetools" + "os" +) + +func main() { + if err := montagetools.Run(os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cmd/server/api.go b/cmd/server/api.go index 7a613748..d0d2b01f 100644 --- a/cmd/server/api.go +++ b/cmd/server/api.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path/filepath" "strings" "time" @@ -48,7 +49,7 @@ func buildCloudProvisioner( sshKeys, terraformcloud.Runner{Root: settings.TerraformWorkspace, Binary: settings.TerraformBinary}, ansiblecloud.Runner{Root: settings.TerraformWorkspace, Binary: settings.AnsiblePlaybookBinary}, - ) + ).WithHostTrustVerifier(gcpcloud.HostVerifier{Catalog: gcpcloud.New(nil), Root: filepath.Join("storage", "credentials", "ssh", "cloud-generations")}) } func buildAPI( @@ -75,7 +76,7 @@ func buildAPI( } browsers[domain.StorageLocal], browsers[domain.StoragePVC], browsers[domain.StorageNFS], browsers[domain.StorageLustre] = fs, fs, fs, fs } - ssh := sshfilesystem.New(storage.environments, provider.OSCommandExecutor{}) + ssh := sshfilesystem.New(storage.environments, provider.OSCommandExecutor{}).WithCloudStore(storage.cloud) browsers[domain.StorageSSH] = ssh manager := appbuild.Manager{Root: settings.ArtifactStoreRoot, MaxBytes: settings.BuildContextMaxBytes, Catalog: storage.data} manager.Executor = appbuild.Executor{Catalog: storage.data, Contexts: manager, Runner: provider.OSCommandExecutor{}, Buildctl: settings.Buildctl, Apptainer: settings.Apptainer, ArtifactStoreRoot: settings.ArtifactStoreRoot} @@ -118,7 +119,7 @@ func buildAPI( Audit: storage.audit, Console: console, Terminal: terminal, - Storage: appstorage.NewBrowserCoordinator(storage.storage, browsers), + Storage: appstorage.NewBrowserCoordinator(appstorage.CloudWorkspaceCatalog{Catalog: storage.storage, Cloud: storage.cloud}, browsers), Build: manager, Planning: planning, PlanningStore: storage.plans, diff --git a/cmd/server/events.go b/cmd/server/events.go index f9af9c4f..7eac30b8 100644 --- a/cmd/server/events.go +++ b/cmd/server/events.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "strconv" "time" applicationexecution "github.com/UFFeScience/akoflow/internal/application/execution" @@ -84,7 +85,11 @@ func registerExecutionHandlers( } verifiedArtifacts := &applicationtransfer.VerifiedArtifactCache{} kubernetesTransfer := &infratransfer.KubernetesExec{BufferSize: bufferSize} - preparer := applicationtransfer.Coordinator{Catalog: data, WorkspaceSyncer: infratransfer.WorkspaceRsync{Resolver: resolver}, Materializer: applicationtransfer.Materializer{Resolver: resolver, Progress: data, VerifiedArtifacts: verifiedArtifacts, Connectors: []ports.TransferConnector{ + parallelism, err := strconv.Atoi(os.Getenv("AKOFLOW_WORKSPACE_TRANSFER_PARALLELISM")) + if err != nil || parallelism <= 0 { + parallelism = 8 + } + preparer := applicationtransfer.Coordinator{Catalog: data, WorkspaceSyncer: infratransfer.WorkspaceRsync{Resolver: resolver, Parallelism: parallelism}, Materializer: applicationtransfer.Materializer{Resolver: resolver, Progress: data, VerifiedArtifacts: verifiedArtifacts, Connectors: []ports.TransferConnector{ infratransfer.ArtifactStore{Root: artifactStoreRoot}, infratransfer.LocalFilesystem{BufferSize: bufferSize}, infratransfer.RsyncSSH{BufferSize: bufferSize}, kubernetesTransfer, infratransfer.HTTPDownload{}, infratransfer.S3Compatible{BufferSize: bufferSize}, infratransfer.GCS{}, }}} supervisor, err := controlexecution.New( diff --git a/cmd/simgrid-case/README.md b/cmd/simgrid-case/README.md new file mode 100644 index 00000000..1918523a --- /dev/null +++ b/cmd/simgrid-case/README.md @@ -0,0 +1,26 @@ +# Offline planning and simulation export + +Export an existing simulation request without submitting it: + +```sh +go run ./cmd/simgrid-case --request private-request.json --output new-case-dir +``` + +Generate experimental PRISM-time candidates first, requiring measured compute +profiles, then export the first time-ranked candidate for native SimGrid: + +```sh +go run ./cmd/simgrid-case --prism-time --request private-request.json --output new-case-dir +``` + +Input must be simulation-only. The caller supplies the complete resource scope, +topology, evidenced materialization profiles, compute profiles and runtimes. +Missing materialization route profiles do not acquire guessed rates. Both the +planner and simulation exporter consume the same request model. Timeout is ten +minutes; beam width and candidate count use PRISM defaults (120 and 25). +Private exports include candidates, selected request, platform and native input. +The output directory must not exist. This tool does not contact the daemon, +access the database, provision cloud resources or start a real execution. + +Model agreement is not independent real validation. This command does not +estimate missing physical interference or validate unobserved route classes. diff --git a/cmd/simgrid-case/main.go b/cmd/simgrid-case/main.go new file mode 100644 index 00000000..61cde1d3 --- /dev/null +++ b/cmd/simgrid-case/main.go @@ -0,0 +1,126 @@ +// simgrid-case exports a private simulation-only case using the production +// planner/SimGrid contract. It never submits an execution or provisions a VM. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/workflow" + "github.com/UFFeScience/akoflow/internal/planning/algorithms" + "github.com/UFFeScience/akoflow/internal/provider/simgrid" +) + +func main() { + requestPath := flag.String("request", "", "private simulation request JSON") + outputPath := flag.String("output", "", "new private case directory (must not exist)") + replan := flag.Bool("prism-time", false, "generate a new offline PRISM-time placement before exporting") + speedup := flag.Bool("speedup-fallback", false, "fill missing compute profiles with explicit speedup-derived estimates") + fixed := flag.Bool("evaluate-fixed", false, "reevaluate predicted metrics without changing placement or order") + flag.Parse() + if err := exportCaseWithOptions(*requestPath, *outputPath, *replan, *speedup, *fixed); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func export(requestPath, outputPath string) error { + return exportCase(requestPath, outputPath, false) +} + +type planSink struct{ plans []domain.SchedulePlan } + +func (s *planSink) Emit(_ context.Context, plan domain.SchedulePlan) error { + s.plans = append(s.plans, plan) + return nil +} + +func exportCase(requestPath, outputPath string, replan bool) error { + return exportCaseWithSpeedup(requestPath, outputPath, replan, false) +} + +func exportCaseWithSpeedup(requestPath, outputPath string, replan, speedup bool) error { + return exportCaseWithOptions(requestPath, outputPath, replan, speedup, false) +} + +func exportCaseWithOptions(requestPath, outputPath string, replan, speedup, fixed bool) error { + if replan && fixed { + return fmt.Errorf("replan and fixed evaluation are mutually exclusive") + } + if requestPath == "" || outputPath == "" { + return fmt.Errorf("--request and --output required") + } + payload, err := os.ReadFile(requestPath) + if err != nil { + return err + } + var request ports.ExecutionRequest + if err := json.Unmarshal(payload, &request); err != nil { + return err + } + if request.Run.Mode != domain.ExecutionModeSimulation { + return fmt.Errorf("offline replanning/export requires simulation mode") + } + if speedup { + request.ActivityProfiles, err = workflow.CompleteSpeedupProfiles(request.Workflow.Activities, request.Resources, request.ActivityProfiles) + if err != nil { + return err + } + } + var candidates []domain.SchedulePlan + if fixed { + planning := domain.PlanningRequest{Workflow: request.Workflow, ExecutionScope: request.ExecutionScope, Resources: request.Resources, NetworkTopology: request.NetworkTopology, ActivityProfiles: request.ActivityProfiles, DeadlineSeconds: request.Plan.DeadlineSeconds, Budget: request.Plan.Budget} + request.Plan, err = algorithms.EvaluateFixedPlan(planning, request.Plan) + if err != nil { + return err + } + } + if replan { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + sink := &planSink{} + planning := domain.PlanningRequest{Workflow: request.Workflow, ExecutionScope: request.ExecutionScope, + Resources: request.Resources, NetworkTopology: request.NetworkTopology, ActivityProfiles: request.ActivityProfiles, + DeadlineSeconds: request.Plan.DeadlineSeconds, Budget: request.Plan.Budget} + if err := algorithms.NewPRISMTime().Schedule(ctx, planning, map[string]any{"requireMeasuredProfiles": true}, nil, sink); err != nil { + return err + } + if len(sink.plans) == 0 { + return fmt.Errorf("no feasible evidenced PRISM-time candidate") + } + candidates = sink.plans + request.Plan = candidates[0] + request.Run.SchedulePlanID = request.Plan.ID + } + platform, input, err := simgrid.BuildOfflineCase(request, 1e9) + if err != nil { + return err + } + if err := os.Mkdir(outputPath, 0700); err != nil { + return err + } + files := map[string][]byte{"platform.xml": platform, "simulation.json": input} + if replan || speedup || fixed { + files["request.json"], err = json.MarshalIndent(request, "", " ") + if err != nil { + return err + } + files["candidates.json"], err = json.MarshalIndent(candidates, "", " ") + if err != nil { + return err + } + } + for name, content := range files { + if err := os.WriteFile(filepath.Join(outputPath, name), content, 0600); err != nil { + return err + } + } + return nil +} diff --git a/cmd/simgrid-case/main_test.go b/cmd/simgrid-case/main_test.go new file mode 100644 index 00000000..9807c05f --- /dev/null +++ b/cmd/simgrid-case/main_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestOfflineReplanRejectsRealWithoutCreatingOutput(t *testing.T) { + dir := t.TempDir() + input, output := filepath.Join(dir, "input.json"), filepath.Join(dir, "case") + request := ports.ExecutionRequest{Run: domain.ExecutionRun{Mode: domain.ExecutionModeReal}} + payload, _ := json.Marshal(request) + if err := os.WriteFile(input, payload, 0600); err != nil { + t.Fatal(err) + } + if err := exportCase(input, output, true); err == nil { + t.Fatal("real mode accepted") + } + if _, err := os.Stat(output); !os.IsNotExist(err) { + t.Fatal("real request created output") + } +} + +func TestOfflineReplanExportsNewPlanAndExactSimulationInput(t *testing.T) { + dir := t.TempDir() + input, output := filepath.Join(dir, "input.json"), filepath.Join(dir, "case") + request := ports.ExecutionRequest{ + Run: domain.ExecutionRun{ID: "test", Mode: domain.ExecutionModeSimulation}, + Workflow: domain.WorkflowVersion{ID: "workflow", Activities: []domain.Activity{{ID: "a", ActivityTypeID: "type", Kind: domain.ActivityKindTask, Resources: domain.ActivityResources{CPU: 1, MemoryBytes: 1}, Simulation: &domain.ActivitySimulation{DurationSeconds: 1}}}}, + ExecutionScope: domain.ExecutionScope{EnvironmentVersionIDs: []string{"environment"}}, + Resources: []domain.Resource{{ID: "r", EnvironmentVersionID: "environment", ExecutionTarget: domain.ExecutionTargetDirect, CPUCores: 1, CPUCapacity: 1, MemoryBytes: 1024, ComputeSpeedup: 1, Schedulable: true}}, + ActivityProfiles: []domain.ActivityResourceProfile{{ID: "profile", ActivityTypeID: "type", ResourceID: "r", RuntimeSeconds: 1, ModelVersion: "reference-work-v1"}}, + } + payload, _ := json.Marshal(request) + if err := os.WriteFile(input, payload, 0600); err != nil { + t.Fatal(err) + } + if err := exportCase(input, output, true); err != nil { + t.Fatal(err) + } + for _, name := range []string{"request.json", "candidates.json", "simulation.json", "platform.xml"} { + info, err := os.Stat(filepath.Join(output, name)) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0600 { + t.Fatal("private export has unsafe permissions") + } + } + encoded, _ := os.ReadFile(filepath.Join(output, "request.json")) + var selected ports.ExecutionRequest + if err := json.Unmarshal(encoded, &selected); err != nil { + t.Fatal(err) + } + if selected.Plan.Algorithm != "prism-time" || len(selected.Plan.Assignments) != 1 || selected.Run.SchedulePlanID != selected.Plan.ID { + t.Fatal("new plan not exported") + } +} diff --git a/docs/docs/showcase/ai/generative-ai/ai-video-studio.mdx b/docs/docs/showcase/ai/generative-ai/ai-video-studio.mdx new file mode 100644 index 00000000..63007432 --- /dev/null +++ b/docs/docs/showcase/ai/generative-ai/ai-video-studio.mdx @@ -0,0 +1,31 @@ +--- +title: AI Video Studio +description: Turn a creative brief into an offline storyboard, illustrated scenes, narration, and a validated video. +--- + +import {WorkflowDag, WorkflowPatternDetails} from '@site/src/components/WorkflowShowcase'; + +# AI Video Studio + +This runnable example turns a versioned creative brief into a complete short video without a hosted model or API. Python creates the storyboard, deterministic illustrations, and narration waveform; FFmpeg produces the MP4 inside the provided image. + + + + + +## Run locally + +```bash +git clone https://github.com/UFFeScience/akoflow-examples.git +cd akoflow-examples/generative-ai/video-studio +./run.sh +``` + +The default smoke profile is CPU-compatible. Set `AKOFLOW_PROFILE=demo` for a longer render. FFmpeg is optional for the host fixture; the OCI image includes it and creates an H.264/AAC MP4. + +[Open the source bundle](https://github.com/UFFeScience/akoflow-examples/tree/main/generative-ai/video-studio) · [Workflow YAML](https://raw.githubusercontent.com/UFFeScience/akoflow-examples/main/generative-ai/video-studio/workflow.yaml) + diff --git a/docs/docs/showcase/ai/generative-ai/index.mdx b/docs/docs/showcase/ai/generative-ai/index.mdx index 5c4a5681..04fd40ba 100644 --- a/docs/docs/showcase/ai/generative-ai/index.mdx +++ b/docs/docs/showcase/ai/generative-ai/index.mdx @@ -14,4 +14,5 @@ RAG, synthetic data, speech, and document-processing patterns. Each workflow bel | [Synthetic dataset generation](/docs/showcase/ai/generative-ai/synthetic-dataset) | Generate, filter, deduplicate, review, and publish synthetic records. | | [Audio transcription and summarization](/docs/showcase/ai/generative-ai/audio-transcription) | Transcribe segments concurrently, restore chronology, and produce a traceable summary. | | [OCR and document extraction](/docs/showcase/ai/generative-ai/document-extraction) | Combine native extraction and OCR before validating structured records. | - +| [AI Video Studio](/docs/showcase/ai/generative-ai/ai-video-studio) | Produce an offline storyboard, parallel visual and audio assets, and a validated video. | +| [Local RAG Factory](/docs/showcase/ai/generative-ai/local-rag-factory) | Build, query, cite, and evaluate a deterministic local retrieval index. | diff --git a/docs/docs/showcase/ai/generative-ai/local-rag-factory.mdx b/docs/docs/showcase/ai/generative-ai/local-rag-factory.mdx new file mode 100644 index 00000000..43326402 --- /dev/null +++ b/docs/docs/showcase/ai/generative-ai/local-rag-factory.mdx @@ -0,0 +1,31 @@ +--- +title: Local RAG Factory +description: Build and evaluate a network-free retrieval pipeline over a versioned local corpus. +--- + +import {WorkflowDag, WorkflowPatternDetails} from '@site/src/components/WorkflowShowcase'; + +# Local RAG Factory + +The workflow normalizes a local Markdown corpus, creates chunks, builds a hashed TF-IDF index, retrieves evidence, generates extractive answers with citations, and evaluates grounding. Its deterministic implementation makes retrieval behavior inspectable before an optional Ollama adapter is introduced. + + + + + +## Run locally + +```bash +git clone https://github.com/UFFeScience/akoflow-examples.git +cd akoflow-examples/generative-ai/local-rag +./run.sh +``` + +Execution is offline and uses only Python's standard library. The activity boundaries remain suitable for replacing the deterministic answer stage with a local Ollama model. + +[Open the source bundle](https://github.com/UFFeScience/akoflow-examples/tree/main/generative-ai/local-rag) · [Workflow YAML](https://raw.githubusercontent.com/UFFeScience/akoflow-examples/main/generative-ai/local-rag/workflow.yaml) + diff --git a/docs/docs/showcase/document-ai/document-intelligence.mdx b/docs/docs/showcase/document-ai/document-intelligence.mdx new file mode 100644 index 00000000..359b9eeb --- /dev/null +++ b/docs/docs/showcase/document-ai/document-intelligence.mdx @@ -0,0 +1,31 @@ +--- +title: Document Intelligence +description: Extract and reconcile auditable procurement records from an offline document corpus. +--- + +import {WorkflowDag, WorkflowPatternDetails} from '@site/src/components/WorkflowShowcase'; + +# Document Intelligence + +This example processes synthetic invoices and a purchase order through explicit ingestion, classification, extraction, reconciliation, and reporting activities. It verifies totals and duplicate invoice identifiers rather than stopping after text extraction. + + + + + +## Run locally + +```bash +git clone https://github.com/UFFeScience/akoflow-examples.git +cd akoflow-examples/document-ai/document-intelligence +./run.sh +``` + +The fixture is intentionally small and private-data-free. Replace `data/documents` with a larger local corpus while preserving the same artifact contract. + +[Open the source bundle](https://github.com/UFFeScience/akoflow-examples/tree/main/document-ai/document-intelligence) · [Workflow YAML](https://raw.githubusercontent.com/UFFeScience/akoflow-examples/main/document-ai/document-intelligence/workflow.yaml) + diff --git a/docs/docs/showcase/index.mdx b/docs/docs/showcase/index.mdx index 9ac2bc53..8f6a6ecf 100644 --- a/docs/docs/showcase/index.mdx +++ b/docs/docs/showcase/index.mdx @@ -1,21 +1,24 @@ --- title: Showcases sidebar_label: Overview -description: Executable AkôFlow examples for machine learning, generative AI, agentic workflows, and scientific AI. +description: Executable AkôFlow examples for AI, documents, media, rendering, and scientific workflows. --- import WorkflowCatalog from '@site/src/components/WorkflowCatalog'; # Showcases -Explore 22 executable workflows organized by scientific objective. Each page explains the workflow DAG, activity responsibilities, inputs, outputs, provenance, and execution considerations without embedding implementation code in the documentation. +Explore 27 executable workflows organized by technical domain. Each page explains the workflow DAG, activity responsibilities, inputs, outputs, provenance, and execution considerations without embedding implementation code in the documentation. The complete Dockerfiles, Python programs, generators, input fixtures, runners, and validators are maintained separately in [`UFFeScience/akoflow-examples`](https://github.com/UFFeScience/akoflow-examples). A single source-bundle link on each page connects the explanation to its versioned implementation. ## Choose a category - **Machine Learning** — training, tuning, inference, evaluation, conversion, vision, ensembles, and infrastructure selection. -- **Generative AI** — retrieval, synthetic data, audio transcription, and document extraction. +- **Generative AI** — retrieval, synthetic data, audio transcription, document extraction, and offline video creation. +- **Document AI** — classification, structured extraction, and reconciliation. +- **Media intelligence** — reproducible video inspection, segmentation, audio analysis, and timelines. +- **Rendering** — procedural scenes, parallel frame shards, and animation composition. - **Agentic workflows** — bounded agents, supervised tool use, and multi-agent review. - **Scientific AI** — AI-assisted analysis, surrogate models, and experiment feedback loops. diff --git a/docs/docs/showcase/media/video-intelligence.mdx b/docs/docs/showcase/media/video-intelligence.mdx new file mode 100644 index 00000000..df1d5257 --- /dev/null +++ b/docs/docs/showcase/media/video-intelligence.mdx @@ -0,0 +1,31 @@ +--- +title: Video Intelligence +description: Generate, inspect, segment, and summarize a reproducible offline test video. +--- + +import {WorkflowDag, WorkflowPatternDetails} from '@site/src/components/WorkflowShowcase'; + +# Video Intelligence + +The workflow creates a known three-scene video, inspects its streams, extracts scene thumbnails, measures audio, and assembles a searchable timeline. The known input makes scene and duration assertions deterministic while still exercising real FFmpeg processing in the OCI image. + + + + + +## Run locally + +```bash +git clone https://github.com/UFFeScience/akoflow-examples.git +cd akoflow-examples/media/video-intelligence +./run.sh +``` + +Minimal hosts use a deterministic portable fixture. The container installs FFmpeg and FFprobe and performs the complete media path. + +[Open the source bundle](https://github.com/UFFeScience/akoflow-examples/tree/main/media/video-intelligence) · [Workflow YAML](https://raw.githubusercontent.com/UFFeScience/akoflow-examples/main/media/video-intelligence/workflow.yaml) + diff --git a/docs/docs/showcase/rendering/blender-render-farm.mdx b/docs/docs/showcase/rendering/blender-render-farm.mdx new file mode 100644 index 00000000..2c2c3420 --- /dev/null +++ b/docs/docs/showcase/rendering/blender-render-farm.mdx @@ -0,0 +1,31 @@ +--- +title: Blender Render Farm +description: Render disjoint frame shards concurrently and compose a validated animation. +--- + +import {WorkflowDag, WorkflowPatternDetails} from '@site/src/components/WorkflowShowcase'; + +# Blender Render Farm + +This example creates a procedural Blender scene, divides frames into even and odd shards, renders both activities independently, and composes the complete sequence. The fan-out makes resource placement and parallel makespan visible in an AkôFlow plan. + + + + + +## Run locally + +```bash +git clone https://github.com/UFFeScience/akoflow-examples.git +cd akoflow-examples/rendering/blender-render-farm +./run.sh +``` + +The OCI image contains Blender and FFmpeg. A deterministic CPU software-render fixture keeps validation possible on minimal CI machines; set `AKOFLOW_REQUIRE_BLENDER=1` when Blender execution is mandatory. + +[Open the source bundle](https://github.com/UFFeScience/akoflow-examples/tree/main/rendering/blender-render-farm) · [Workflow YAML](https://raw.githubusercontent.com/UFFeScience/akoflow-examples/main/rendering/blender-render-farm/workflow.yaml) + diff --git a/docs/docs/tutorials/compare-plans.md b/docs/docs/tutorials/compare-plans.md new file mode 100644 index 00000000..4e15d7da --- /dev/null +++ b/docs/docs/tutorials/compare-plans.md @@ -0,0 +1,328 @@ +--- +title: Compare PRISM and HEFT plans +description: Generate alternative plans for the SimGrid example, compare their predictions, execute two of them, and read the difference between predicted and observed evidence. +sidebar_label: Compare PRISM and HEFT plans +slug: /tutorials/compare-plans +--- + +# Compare PRISM and HEFT plans + +The [SimGrid first-run tutorial](/docs/guides/workflows/first-run) ends with a single, fixed plan. This tutorial uses the same workflow and infrastructure but lets AkôFlow propose alternatives. You will run PRISM Time, PRISM Cost, and HEFT against the same inputs, compare their predictions, execute two candidates, and read the predicted-versus-observed evidence that AkôFlow records for each run. + +The goal is to learn how AkôFlow separates *predicting* from *executing*, how the same workflow looks different under different objectives, and what the recorded evidence actually allows you to conclude. + +## Before you begin + +You need the same prerequisites as the [SimGrid first-run tutorial](/docs/guides/workflows/first-run): + +- A clone of the AkôFlow repository. +- The AkôFlow daemon running with the SimGrid runner available. +- `curl` and `jq` on your command line. +- The daemon API token, when authentication is enabled. + +Set the API address and token. The development Compose stack listens on port 8080 by default: + +```bash +export AKOFLOW_API_URL="http://127.0.0.1:8080/akoflow-api" +export AKOFLOW_API_TOKEN="" +``` + +Confirm the daemon is healthy before continuing: + +```bash +curl --fail-with-body \ + -H "Authorization: Bearer $AKOFLOW_API_TOKEN" \ + "$AKOFLOW_API_URL/preflight/" | jq +``` + +Continue only when `server.available` is `true` and the SimGrid runner is reachable. + +## What you will compare + +The versioned six-file bundle in [`examples/simulation`](https://github.com/UFFeScience/akoflow/tree/main/examples/simulation) defines a `prepare → analyze → summarize` DAG with two data dependencies, a two-core edge resource, an eight-core cloud resource, and a 100 Mbit/s link. The fixed plan in `plan-request.yaml` places `analyze` on the cloud. This tutorial asks AkôFlow to propose plans and runs two of them. + +A session freezes one workflow version and one infrastructure scope. All algorithms in that session share the same inputs and produce distinct candidate plans. Only a selected candidate becomes an executable schedule plan. + +```mermaid +flowchart LR + WF[Workflow version] --> Session[Planning session] + Scope[Execution scope] --> Session + Net[Network topology] --> Session + Session -->|PRISM Time| C1[Candidate 1] + Session -->|PRISM Cost| C2[Candidate 2] + Session -->|HEFT| C3[Candidate 3] + C1 -->|Select| Plan1[Schedule plan] + C2 -->|Select| Plan2[Schedule plan] + C3 -->|Select| Plan3[Schedule plan] + Plan1 --> Run1[Execution run] + Plan2 --> Run2[Execution run] +``` + +## 1. Register the same example + +If you have already run the [first-run tutorial](/docs/guides/workflows/first-run), the catalog objects exist. Submit the bundle once more for a fresh start: + +```bash +for endpoint in environments execution-scopes network-topologies workflow-definitions; do + for file in examples/simulation/environment.yaml examples/simulation/scope.yaml \ + examples/simulation/topology.yaml examples/simulation/workflow.yaml; do + if [ "$endpoint" = "workflow-definitions" ] && [ "$file" != "examples/simulation/workflow.yaml" ]; then + continue + fi + if [ "$endpoint" = "execution-scopes" ] && [ "$file" != "examples/simulation/scope.yaml" ]; then + continue + fi + if [ "$endpoint" = "network-topologies" ] && [ "$file" != "examples/simulation/topology.yaml" ]; then + continue + fi + if [ "$endpoint" = "environments" ] && [ "$file" != "examples/simulation/environment.yaml" ]; then + continue + fi + curl --fail-with-body \ + -H "Authorization: Bearer $AKOFLOW_API_TOKEN" \ + -H 'Content-Type: application/yaml' \ + --data-binary @"$file" \ + "$AKOFLOW_API_URL/$endpoint/" > /dev/null + done +done +``` + +A shorter alternative is to call the bundled script, which already runs the same sequence: + +```bash +AKOFLOW_API_URL="$AKOFLOW_API_URL" \ +AKOFLOW_API_TOKEN="$AKOFLOW_API_TOKEN" \ +sh examples/simulation/run.sh +``` + +Stop on the first HTTP failure. Verify the workflow before planning: + +```bash +curl --fail-with-body \ + -H "Authorization: Bearer $AKOFLOW_API_TOKEN" \ + "$AKOFLOW_API_URL/workflow-definitions/simulation-example-workflow/" | + jq '{id: .workflow.id, + activities: [.version.activities[].name]}' +``` + +You should see three activities: `prepare`, `analyze`, and `summarize`. A workflow with different activity names means the rest of this tutorial will not work; restore the file or use a fresh instance. + +## 2. Discover the planning algorithms + +The available algorithms are reported by the server, not hard-coded in this guide: + +```bash +curl --fail-with-body -H "Authorization: Bearer $AKOFLOW_API_TOKEN" \ + "$AKOFLOW_API_URL/planning-algorithms/" | jq +``` + +The PRISM algorithms accept `optionCount` and `beamWidth`. The HEFT algorithm does not. A session that lists an algorithm the server does not return will be rejected. See [PRISM and HEFT](/docs/explanations/prism-and-heft) for what each algorithm optimises and why they can disagree. + +## 3. Open a planning session + +Create one session that runs all three algorithms against the same workflow version and scope: + +```bash +curl --fail-with-body \ + -H "Authorization: Bearer $AKOFLOW_API_TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{ + "id": "tutorial-compare-plans", + "workflowVersionId": "simulation-example-workflow-v1", + "executionScopeId": "simulation-example-v1-scope", + "networkTopologyId": "simulation-network-v1", + "algorithms": [ + {"id": "prism-time", "configuration": {"optionCount": 25, "beamWidth": 120}}, + {"id": "prism-cost", "configuration": {"optionCount": 25, "beamWidth": 120}}, + {"id": "heft", "configuration": {}} + ] + }' \ + "$AKOFLOW_API_URL/planning-sessions/" +``` + +Creation is asynchronous; the response is `202 Accepted` and a session ID. Poll until the status is `completed`: + +```bash +while :; do + session=$(curl --fail-with-body --silent --show-error \ + -H "Authorization: Bearer $AKOFLOW_API_TOKEN" \ + "$AKOFLOW_API_URL/planning-sessions/tutorial-compare-plans/") + state=$(printf '%s' "$session" | jq -r '.session.status') + printf '%s\n' "$session" | jq '{status: .session.status}' + case "$state" in completed|failed|cancelled) break ;; esac + sleep 1 +done +``` + +A `failed` status means the daemon could not finish planning; check the session events and the daemon log before continuing. + +## 4. Compare the candidates + +List the candidates the session produced: + +```bash +curl --fail-with-body -H "Authorization: Bearer $AKOFLOW_API_TOKEN" \ + "$AKOFLOW_API_URL/planning-sessions/tutorial-compare-plans/candidates/" | + jq '.candidates + | map({algorithm: .algorithm, + objective: .objective, + rank: .rank, + makespanSeconds: .predicted.makespanSeconds, + cost: .predicted.cost, + feasible: .feasible})' +``` + +Each candidate carries: + +- its **algorithm** and **objective** (one of `time`, `cost`, or HEFT's default); +- a **rank** that the server uses to order its response; +- a **feasibility** flag set by the current model; +- the **predicted makespan** and **cost**; +- a full activity-to-resource assignment list inside the schedule. + +Read the detailed candidate for the algorithm you want to inspect: + +```bash +curl --fail-with-body -H "Authorization: Bearer $AKOFLOW_API_TOKEN" \ + "$AKOFLOW_API_URL/planning-sessions/tutorial-compare-plans/candidates//" | + jq '.candidate.schedule.assignments + | map({activity: .activityName, + resource: .resourceId, + predictedStart: .predictedStart, + predictedFinish: .predictedFinish})' +``` + +Compare PRISM Time against HEFT. When the link bandwidth dominates the workflow, the algorithms often disagree on whether to keep `analyze` on the cloud or run it on the edge. Record the candidate IDs you want to keep; the next step selects one. + +## 5. Select two candidates and execute them + +Promote two candidates to canonical schedule plans. Pick one PRISM candidate and one HEFT candidate so that you can compare objective-driven placement against the comparison baseline: + +```bash +# Promote the PRISM Time candidate +curl --fail-with-body -X POST \ + -H "Authorization: Bearer $AKOFLOW_API_TOKEN" \ + "$AKOFLOW_API_URL/planning-sessions/tutorial-compare-plans/candidates//select/" + +# Promote the HEFT candidate +curl --fail-with-body -X POST \ + -H "Authorization: Bearer $AKOFLOW_API_TOKEN" \ + "$AKOFLOW_API_URL/planning-sessions/tutorial-compare-plans/candidates//select/" +``` + +Selection returns `201 Created` with the new plan IDs. Use a stable run ID per candidate so the comparison is reproducible: + +```bash +RUN_ID_PRISM='tutorial-compare-plans-prism-v1' +RUN_ID_HEFT='tutorial-compare-plans-heft-v1' +``` + +Submit the same execution request shape used by `examples/simulation/execution-request.yaml`, but reference the selected plan and the chosen run ID: + +```bash +curl --fail-with-body \ + -H "Authorization: Bearer $AKOFLOW_API_TOKEN" \ + -H 'Content-Type: application/yaml' \ + -d "{ + \"id\": \"$RUN_ID_PRISM\", + \"schedulePlanId\": \"\", + \"mode\": \"simulation\" + }" \ + "$AKOFLOW_API_URL/execution-runs/" + +curl --fail-with-body \ + -H "Authorization: Bearer $AKOFLOW_API_TOKEN" \ + -H 'Content-Type: application/yaml' \ + -d "{ + \"id\": \"$RUN_ID_HEFT\", + \"schedulePlanId\": \"\", + \"mode\": \"simulation\" + }" \ + "$AKOFLOW_API_URL/execution-runs/" +``` + +Both submissions return `202 Accepted`. Poll each run until the status is `completed`: + +```bash +for RUN_ID in "$RUN_ID_PRISM" "$RUN_ID_HEFT"; do + while :; do + run=$(curl --fail-with-body --silent --show-error \ + -H "Authorization: Bearer $AKOFLOW_API_TOKEN" \ + "$AKOFLOW_API_URL/execution-runs/$RUN_ID/") + state=$(printf '%s' "$run" | jq -r '.run.status') + printf '%s\n' "$run" | jq --arg id "$RUN_ID" \ + '{id: $id, status: .run.status, completed: .run.completedActivityCount}' + case "$state" in completed|failed|cancelled) break ;; esac + sleep 1 + done +done +``` + +## 6. Read the predicted-versus-observed evidence + +For each completed run, compare the predicted schedule against the observed one: + +```bash +for RUN_ID in "$RUN_ID_PRISM" "$RUN_ID_HEFT"; do + curl --fail-with-body \ + -H "Authorization: Bearer $AKOFLOW_API_TOKEN" \ + "$AKOFLOW_API_URL/execution-runs/$RUN_ID/" | + jq --arg id "$RUN_ID" '{ + id: $id, + status: .run.status, + completed: .run.completedActivityCount, + transferredBytes: .run.transferredBytes, + makespanSeconds: .run.makespanSeconds, + predictedMakespanSeconds: .plan.predictedMakespanSeconds, + breakdown: .run.breakdown + }' +done +``` + +The Desktop draws the same comparison in the **Plan vs execution** tab. A small predicted-versus-observed gap is normal; the SimGrid model does not include scheduler wake-up latency and the kernel command path adds a few milliseconds. Compare the activity placement and the bytes transferred rather than the seconds, because the example transfers 100 MB to the cloud and 20 MB back regardless of placement. + +You can also export the run evidence from the [Provenance and audit guide](/docs/guides/data/provenance-and-audit). The provenance SQL endpoint lets you query both runs in one statement: + +```bash +curl --fail-with-body -H "Authorization: Bearer $AKOFLOW_API_TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{ + "statement": "SELECT id, makespan_seconds, transferred_bytes FROM execution_runs WHERE id IN (?, ?) ORDER BY id", + "params": ["'"$RUN_ID_PRISM"'", "'"$RUN_ID_HEFT"'"] + }' \ + "$AKOFLOW_API_URL/provenance/sql/" +``` + +## What this evidence tells you + +- The **predicted makespan** is a model answer, not a guarantee. PRISM optimises against the supplied cost or time objective; HEFT does not search the same objective space. When they disagree on placement, the predicted makespan difference reflects the model, not the workload. +- The **transferred bytes** are observed. They are the same in both runs because the same workflow is producing the same data dependencies; only the placement changes. If the bytes differ between runs, the workflow changed. +- The **breakdown** shows where SimGrid spent its time. A larger `transferSeconds` value confirms that the chosen placement actually moved data across the link. +- The **activity count** is identical. If one run has fewer activities, the workflow or plan changed, not the scheduler. + +## Adapt the comparison + +The same pattern answers different questions: + +| Change to the example | Question you answer | +| --- | --- | +| Increase `analyze.simulation.durationSeconds` to 60 | How much does HEFT's makespan estimate grow compared with PRISM? | +| Increase `dataset.bin.sizeBytes` to 500000000 | Does PRISM Time decide to keep `analyze` on the edge instead of the cloud? | +| Add a third core to the cloud resource | Does PRISM Cost move `analyze` back to the cloud when the link dominates? | +| Set `deadlineSeconds` to 30 on the session | Which candidates are marked infeasible and what does Pareto dominance look like? | + +Apply one change at a time, regenerate the candidates, and re-run. The SimGrid runtime stays deterministic for a given workflow and plan, so the bytes transferred and the activity timeline stay reproducible. + +## Recover from common failures + +| Failure | Cause and recovery | +| --- | --- | +| `404` selecting a candidate | The session is from a different instance or has already been promoted; check `GET /planning-sessions/`. | +| `422` on session creation | An algorithm ID is not returned by `GET /planning-algorithms/`, or the workflow version ID is wrong. | +| Run stays `pending` | Inspect run events and the daemon log; the simulation runner may be unreachable. | +| Predicted makespan is `0` | The plan was authored without predicted timing; use an automatic-planning candidate, not an imported envelope. | +| Transferred bytes differ between runs | The workflow or topology changed between submissions. Re-register the workflow and topology before retrying. | + +## Next step + +Read [Observed timing](/docs/explanations/observed-timing) to understand how AkôFlow computes the breakdown fields, or return to the [Planning a workflow](/docs/guides/workflows/planning) reference to learn how to attach a deadline, budget, or interference matrix to a session. diff --git a/docs/package.json b/docs/package.json index d7936af7..dff00300 100644 --- a/docs/package.json +++ b/docs/package.json @@ -4,6 +4,7 @@ "private": true, "scripts": { "generate:api": "node scripts/generate-api-reference.mjs", + "test:api-generator": "node --test scripts/go-function-body.test.mjs", "capture:media": "node scripts/capture-interface-media.mjs", "prestart": "npm run generate:api", "prebuild": "npm run generate:api", diff --git a/docs/scripts/check-links.mjs b/docs/scripts/check-links.mjs index 03165a56..cc9dd63c 100644 --- a/docs/scripts/check-links.mjs +++ b/docs/scripts/check-links.mjs @@ -160,6 +160,11 @@ for (const sourceFile of documentationFiles) { for (const match of source.matchAll(/!?\[[^\]]*\]\(([^)]+)\)/g)) targets.add(match[1]); for (const match of source.matchAll(/<(?:a|img)\b[^>]+(?:href|src)=["']([^"']+)["']/gi)) targets.add(match[1]); for (const match of source.matchAll(/require\(['"](@site\/static\/[^'"]+)['"]\)/g)) targets.add(match[1]); + if (sourceFile.endsWith("interface-screens.mdx")) { + for (const match of source.matchAll(/\bpath:\s*'([^']+\.png)'/g)) { + targets.add(`/img/interface/${match[1]}`); + } + } for (const target of targets) checkTarget(sourceFile, target, sourceIsShowcase); } diff --git a/docs/scripts/generate-api-reference.mjs b/docs/scripts/generate-api-reference.mjs index c60c1bc2..deb39d5b 100644 --- a/docs/scripts/generate-api-reference.mjs +++ b/docs/scripts/generate-api-reference.mjs @@ -1,6 +1,7 @@ import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { extractFunctionBody } from "./go-function-body.mjs"; const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)); const docsDirectory = path.resolve(scriptDirectory, ".."); @@ -70,14 +71,8 @@ const groupMetadata = { "/docs/guides/workflows/executions", "workflow runs and results", ], - Artifacts: [ - "/docs/guides/data/artifacts", - "choose an artifact task", - ], - Provenance: [ - "/docs/guides/data/provenance", - "scientific provenance", - ], + Artifacts: ["/docs/guides/data/artifacts", "choose an artifact task"], + Provenance: ["/docs/guides/data/provenance", "scientific provenance"], Console: [ "/docs/guides/operations/interactive-console", "interactive console sessions and commands", @@ -87,8 +82,12 @@ const groupMetadata = { }; const queryDescriptions = { - async: "Set to `true` to start cloud catalog discovery in the background and return `202 Accepted` immediately. Repeat requests while it is running do not start another discovery.", - attempt: "Select the activity execution attempt whose metric samples or summary should be returned. Defaults to the first attempt.", + workflowVersionId: + "Filter execution runs by saved plan workflow version before pagination and total counting. Supplying this filter selects the paginated response, even without page/pageSize.", + async: + "Set to `true` to start cloud catalog discovery in the background and return `202 Accepted` immediately. Repeat requests while it is running do not start another discovery.", + attempt: + "Select the activity execution attempt whose metric samples or summary should be returned. Defaults to the first attempt.", connectionId: "Limit results to this environment connection.", cursor: "Opaque cursor returned by the preceding page.", depth: "Maximum lineage traversal depth.", @@ -111,7 +110,8 @@ const queryDescriptions = { q: "Free-text search query.", resourceId: "Limit results to this resource.", runId: "Limit results to this run.", - selectable: "Set to `true` to list only artifact versions with a verified, available executable representation for workflow selection.", + selectable: + "Set to `true` to list only artifact versions with a verified, available executable representation for workflow selection.", sessionId: "Limit results to this session.", sortField: "Entity field used to order results.", sortOrder: "Sort direction for the selected field.", @@ -142,28 +142,64 @@ const delegatedSuccessStatuses = { // Checked response shapes cover map values and ambiguous Go type names that // cannot be inferred safely from a handler's outer return type alone. const checkedResponseShapes = { + WorkspaceStorageBudget: { + example: { + storageId: "storage-id", + availableBytes: 1073741824, + observedAt: "2026-01-01T00:00:00Z", + source: "adapter-observed-quota", + }, + note: "The adapter returns an effective storage quota observation, not an assumption from nominal filesystem capacity. Inspect its storage identity, observation time, source and available bytes before using it for planning.", + }, + PreserveCancelledCloudWorkspaces: { + example: { + status: "preserved", + archive: `storage/preserved-workspaces/${"0".repeat(64)}.tar.gz`, + sha256: "0".repeat(64), + workspaceIds: ["workspace-id"], + instanceId: "cloud-instance-id", + }, + note: "This is an illustrative response shape, not evidence of a preserved archive. The real archive filename is its verified SHA-256, and workspace IDs identify only the matching retained generation. Source files and VMs are not deleted by this route.", + }, CreateExecution: { example: { - id: "queue-job-id", category: "execution", eventType: "execution.run.requested", - aggregateType: "execution_run", aggregateId: "run-id", status: "pending", - priority: 0, availableAt: "2026-01-01T00:00:00Z", attempts: 0, - maxAttempts: 5, idempotencyKey: "execution-run:run-id", + id: "queue-job-id", + category: "execution", + eventType: "execution.run.requested", + aggregateType: "execution_run", + aggregateId: "run-id", + status: "pending", + priority: 0, + availableAt: "2026-01-01T00:00:00Z", + attempts: 0, + maxAttempts: 5, + idempotencyKey: "execution-run:run-id", createdAt: "2026-01-01T00:00:00Z", }, note: "This is the queue job for a new request, not the execution-run record. `id` identifies the job; `aggregateId` matches the submitted `run.id`. Queue status may advance while the daemon works, and optional lease/completion fields appear later. A run is saved only after the worker validates the request.", }, ListAuditEvents: { - example: [{ - id: "audit-event-id", eventType: "connection.health.checked", - environmentId: "environment-id", connectionId: "connection-id", - outcome: "succeeded", summary: "Connection is healthy", - occurredAt: "2026-01-01T00:00:00Z", - }], + example: [ + { + id: "audit-event-id", + eventType: "connection.health.checked", + environmentId: "environment-id", + connectionId: "connection-id", + outcome: "succeeded", + summary: "Connection is healthy", + occurredAt: "2026-01-01T00:00:00Z", + }, + ], note: "This is one connection-health event. The list can be empty; current emitters record connection checks, resource discovery, and console actions, not every operation in AkôFlow.", }, RegisterDockerArtifact: { example: { - artifact: { id: "artifact-version-...", artifactId: "busybox", version: "1.36", scope: "system" }, + artifact: { + id: "artifact-version-...", + artifactId: "busybox", + version: "1.36", + scope: "system", + }, build: { id: "build-...", artifactVersionId: "artifact-version-...", @@ -180,7 +216,13 @@ const checkedResponseShapes = { }, }, ValidateCloudCredential: { - example: { valid: true, provider: "gcp", project: "project-id", region: "region", catalogStatus: "not_discovered" }, + example: { + valid: true, + provider: "gcp", + project: "project-id", + region: "region", + catalogStatus: "not_discovered", + }, note: "A `200 OK` response verifies authentication and access to the Compute Engine project. It does not discover machines, images, disks, or prices. Save the environment and refresh its cloud catalog separately.", }, TestEnvironmentConnection: { @@ -198,7 +240,13 @@ const checkedResponseShapes = { status: "completed", makespanSeconds: 12.4, cost: 0, - breakdown: { computeSeconds: 12.4, transferSeconds: 0, queueSeconds: 0, interferenceSeconds: 0, overheadSeconds: 0 }, + breakdown: { + computeSeconds: 12.4, + transferSeconds: 0, + queueSeconds: 0, + interferenceSeconds: 0, + overheadSeconds: 0, + }, transferredBytes: 0, kind: "workflow", }, @@ -212,34 +260,81 @@ const checkedResponseShapes = { ListExecutions: { type: "JSON array or paginated object", example: null, - note: "Without `page` or `pageSize`, the response is an array of `{ \"run\": ... }` items. Supplying either pagination parameter returns `{ \"items\": [...], \"page\": number, \"pageSize\": number, \"total\": number, \"hasNext\": boolean }`.", + note: 'Without `page`, `pageSize` or `workflowVersionId`, the response is an array of `{ "run": ... }` items. Supplying any of these parameters returns `{ "items": [...], "page": number, "pageSize": number, "total": number, "hasNext": boolean, "workflowVersionId": string }`. The workflow filter is applied before pagination and total counting; unrelated workflow runs and standalone commands are excluded.', }, Search: { example: { query: "string", results: [], total: 0 } }, ActivateArchiveInstance: { - example: { instance: { id: "default", name: "Default instance", source: "local", status: "active", readOnly: false, credentialsSet: true }, restarting: false }, + example: { + instance: { + id: "default", + name: "Default instance", + source: "local", + status: "active", + readOnly: false, + credentialsSet: true, + }, + restarting: false, + }, note: "`instance.status` is `active` after selection. `restarting` reports whether this server can trigger its own restart; when it is `false`, restart the daemon manually. An imported snapshot has `readOnly: true` instead of the default instance's `false`.", }, GetPlanningSession: { example: { session: { - id: "session-id", workflowVersionId: "workflow-version-id", executionScopeId: "scope-id", - networkTopologyId: "topology-id", status: "completed", algorithms: [{ id: "heft" }], - progress: 1, candidateCount: 1, createdAt: "2026-01-01T00:00:00Z", + id: "session-id", + workflowVersionId: "workflow-version-id", + executionScopeId: "scope-id", + networkTopologyId: "topology-id", + status: "completed", + algorithms: [{ id: "heft" }], + progress: 1, + candidateCount: 1, + createdAt: "2026-01-01T00:00:00Z", }, - algorithmRuns: [{ - id: "algorithm-run-id", planningSessionId: "session-id", algorithm: "heft", objective: "time", - status: "completed", progress: 1, candidateCount: 1, - estimate: { durationSeconds: 0.1, expandedStates: 1, activityCount: 1, dependencyCount: 0, compatibleResources: 1, confidence: "calibrated" }, - }], + algorithmRuns: [ + { + id: "algorithm-run-id", + planningSessionId: "session-id", + algorithm: "heft", + objective: "time", + status: "completed", + progress: 1, + candidateCount: 1, + estimate: { + durationSeconds: 0.1, + expandedStates: 1, + activityCount: 1, + dependencyCount: 0, + compatibleResources: 1, + confidence: "calibrated", + }, + }, + ], }, note: "The response pairs the saved session with its algorithm runs. `selectedCandidateId` and `selectedPlanId` appear only after selection; a completed session may have neither. The estimate describes predicted planning work, not execution time.", }, GetProvenanceSchema: { - example: { items: [{ name: "execution_runs", columns: [{ name: "id", type: "text" }, { name: "status", type: "text" }] }] }, + example: { + items: [ + { + name: "execution_runs", + columns: [ + { name: "id", type: "text" }, + { name: "status", type: "text" }, + ], + }, + ], + }, note: "This shows two columns from one safe table. The response lists all queryable tables and unblocked columns in the connected database; inspect it before writing a read-only SQL query.", }, QueryProvenanceSQL: { - example: { columns: [{ name: "id", type: "text" }], items: [], page: 1, pageSize: 50, truncated: false, elapsedMilliseconds: 0 }, + example: { + columns: [{ name: "id", type: "text" }], + items: [], + page: 1, + pageSize: 50, + truncated: false, + elapsedMilliseconds: 0, + }, note: "This shows an empty result for a query selecting `id`. `columns` follows the submitted SELECT list, and each item uses those column names. Pagination and elapsed time vary by query.", }, ExplainProvenanceSQL: { @@ -276,83 +371,159 @@ const runnableSimulationRequests = { // These notes come from handler calls and the credential/operation services, // not from JSON tags alone. Keep them scoped to fields the service validates. const verifiedRequestNotes = { - "POST /akoflow-api/execution-runs/": "`202 Accepted` publishes a queue job. The returned `id` is the job ID, while `aggregateId` is the submitted `run.id`. The worker validates the request before saving a run; `GET /execution-runs/{runId}/` can return `404` until then, or remain `404` if validation fails. Use the original `run.id` to read the run, and do not treat the queue job's status as the run's status. Repeating the same `run.id` returns the existing idempotent job rather than creating a second one.", - "POST /akoflow-api/execution-runs/{runId}/cancel/": "No JSON body is required. The run must exist and the instance must be writable. AkôFlow interrupts unfinished activities, cancels queued execution work, marks the run cancelled, and returns the updated run with `200 OK`. Cancelling an already-cancelled run is idempotent and also returns `200`; an unknown run returns `404`.", - "POST /akoflow-api/execution-runs/{runId}/recovery/": "The run must be failed or cancelled and its original queued execution request must still be available. Send `mode` as `continue` or `retry_failures`; omission defaults to `continue`. AkôFlow reuses completed activities whose workspaces remain available, queues the remaining activities, and returns the recovery job plus preview with `202 Accepted`. Inspect the run until it reaches a terminal status.", - "POST /akoflow-api/execution-runs/{runId}/activities/{activityId}/interrupt/": "No JSON body is required. The instance must be writable and the activity must belong to an interruptible active execution. Success returns the updated task with `200 OK`; an unavailable interrupter returns `503`, while an activity that cannot be interrupted returns `409`.", - "POST /akoflow-api/environments/": "The response echoes the submitted definition. The repository saves the environment, version, inventory, storage, and connections, but ignores `connectionChecks` and `connectorBindings` if they appear in the body. Read `GET /environments/{environmentId}/` for saved IDs and separately recorded connection checks. See the [environment YAML reference](/docs/reference/environment-yaml).", - "GET /akoflow-api/environments/": "Each item is a saved environment definition. `connectionChecks` can include recent health records for its connections; these were recorded separately from the environment YAML. The repository does not populate `connectorBindings` from a submitted definition.", - "GET /akoflow-api/environments/{environmentId}/": "Returns the saved definition, including IDs filled from the enclosing environment/version. `connectionChecks` comes from recent connection-health records, not from the submitted YAML. The repository does not populate `connectorBindings` from that document.", - "POST /akoflow-api/schedule-plans/": "Returns the saved `plan`. If `plan.networkTopologyId` is empty, the server fills it from `networkTopology.id`. Submitted predictions are saved without recalculation.", - "GET /akoflow-api/environments/{environmentId}/cloud-catalog/": "Returns the last synchronized catalog; this GET does not call the provider. Before the first successful refresh it returns `404`. Read catalog warnings as well as machine, image, and disk choices; missing price data does not make a machine free.", - "GET /akoflow-api/environments/{environmentId}/cloud-capacity-targets/": "Returns enabled targets for this environment, ordered by name. Deleting a target disables its record, so it disappears from this list without erasing historical operations or instances that used it.", - "GET /akoflow-api/environments/{environmentId}/cloud-instances/": "Returns this environment's saved instances newest first, including records marked destroyed. Inspect each instance's `status` and `destroyedAt` rather than treating the list length as the number of active VMs.", - "GET /akoflow-api/cloud-operations/": "Returns all saved cloud operations, newest first, as an array. This route has no environment filter; use the returned `environmentId` and `capacityTargetId` to find your operation, then open its detail and events.", - "GET /akoflow-api/cloud-operations/{operationId}/": "Returns the current operation or `404` for an unknown ID. `queued` and `running` are not final outcomes. A `failureReason` may appear while an operation is queued for a retry; rely on a terminal `completed`, `failed`, or `cancelled` status and inspect events for details.", - "GET /akoflow-api/cloud-operations/{operationId}/events/": "Returns stored events in sequence order, or `404` for an unknown operation. The server may add parsed provider-log events before returning this list. An `operation.failed` event can describe a retry; check the operation's current status before treating it as a final failure.", - "GET /akoflow-api/cloud-instances/{instanceId}/provisioning-log/": "Use a saved cloud instance ID, not a cloud operation ID. An unknown instance returns `404`. Before Terraform has written a log file, this route can return `200 OK` with a waiting message in `log`; follow the cloud operation status and events for the outcome.", - "GET /": "This health route is at the daemon root, outside `/akoflow-api/`. With the API URL from the [API access tutorial](/docs/tutorials/api-access), the cURL command removes that prefix and requests `/`. A healthy daemon returns plain text `ok`; this check does not verify Docker, BuildKit, or a workflow runtime. Use [preflight](/docs/api/endpoints/instance/get-preflight) for those local capability checks.", - "GET /akoflow-api/preflight/": "The response has `server`, `docker`, and `buildkit` objects, each with `available` and `message`. `server.available` is true when this route responds, but the Docker and BuildKit checks can be false while HTTP still returns `200 OK`. Inspect their individual values before running a local workflow. The checks run in the daemon process, not in your browser or shell.", - "GET /akoflow-api/console-sessions/{sessionId}/stream/": "Open this URL with a WebSocket client after creating a console session. A successful upgrade returns `101 Switching Protocols` and carries terminal input and output over the socket; it is not a JSON response. An unknown session returns `404`. See the [interactive console guide](/docs/guides/operations/interactive-console) for session lifecycle.", - "POST /akoflow-api/provenance/sql/": "Send a read-only `sql` query using `SELECT` or `WITH`; `parameters` supplies optional named values, and `page`/`pageSize` control results (at most 200 rows per page). Use `GET /provenance/sql/schema/` to see allowed tables and columns. The service enforces a 10-second timeout and rejects writes or restricted fields with `400`; an unavailable explorer returns `503`. See [provenance SQL guide](/docs/guides/data/provenance#query-with-read-only-sql).", - "POST /akoflow-api/provenance/sql/explain/": "Send the same `sql` and optional named `parameters` as the read-only SQL route. This runs `EXPLAIN QUERY PLAN` for a permitted `SELECT` or `WITH` statement and returns plan rows, not the query's data rows. It uses the same read-only table/column restrictions and 10-second timeout; invalid SQL returns `400` and an unavailable explorer returns `503`.", - "PUT /akoflow-api/instance/": "Send the complete current instance object with non-empty `id` and `name`; this route saves the supplied object, so preserve existing identity and metadata when changing one field. `transferBufferBytes` accepts 5–64 MiB; `0` selects the 8 MiB default. The [instance guide](/docs/guides/operations/instance-management#inspect-the-active-identity) reads the current object before updating it.", - "PUT /akoflow-api/environments/{environmentId}/": "Read `GET /environments/{environmentId}/` before editing and send a complete environment definition; `environment.id` must match the path ID. The response echoes the submitted definition, while the repository ignores `connectionChecks` and `connectorBindings` in it. Replacement returns `404` when the environment does not exist and can return `422` when references prevent replacing its inventory. Use new environment and version IDs for revised inventory already used by scopes or plans; see the [environment YAML reference](/docs/reference/environment-yaml#compatibility-and-common-failures).", - "POST /akoflow-api/instance-activations/{instanceId}/": "Use `default` to return to the writable instance or an ID returned by `POST /instances/import/` to open a read-only snapshot. See [instance management](/docs/guides/operations/instance-management#import-and-open-a-read-only-snapshot) for the switching procedure.", - "POST /akoflow-api/instances/import/": "Send a ZIP archive as the request body with `Content-Type: application/zip`, not JSON. The archive must be a compatible AkôFlow export with its redaction marker, valid database checksum and schema, no symbolic links, at most 100,000 entries, at most 8 GiB compressed and 64 GiB expanded. Success returns `201 Created` with the new read-only snapshot `id`; import does not replace the active instance. See [instance management](/docs/guides/operations/instance-management#import-and-open-a-read-only-snapshot).", - "PUT /akoflow-api/user-preferences/{clientId}/": "The path `clientId` must be 8–128 characters and overrides any body `clientId`. Send `theme` as `light` or `dark`; `animationsEnabled` is optional and defaults to false when omitted. The route saves preferences for this client profile and returns the stored record with `200 OK`; invalid ID or theme returns `422`. See [Personal preferences](/docs/guides/operations/personal-preferences).", - "POST /akoflow-api/factory-reset/": "No JSON body is required. This permanently clears user catalog/database records while retaining the schema and system instance identity; the server also removes managed Kubernetes token files. Success is `204 No Content`. The database reset runs before token-directory cleanup, so a `422` from that cleanup can arrive after the catalog has already been cleared. Export a snapshot first; see [instance management](/docs/guides/operations/instance-management#factory-reset).", - "PUT /akoflow-api/environment-connections/{connectionId}/": "Read the current connection in `GET /environments/{environmentId}/`, then send its complete object with the changed fields. Its `id` must match the path and `environmentId` must be non-empty. The route inserts a new connection or replaces saved fields, and returns the submitted object without running a health check. See the [HPC connection guide](/docs/guides/infrastructure/hpc-slurm#1-create-the-ssh-credential-and-proxy-aware-connection).", - "POST /akoflow-api/environment-connections/{connectionId}/health/": "Checks a saved connection and records a health result. Inspect the returned `status` and `message`: an unreachable target returns `200 OK` with `status: offline` when the probe itself completed. The environment status is updated to `connected` or `unreachable`.", - "POST /akoflow-api/environment-connections/{connectionId}/discover/": "Discovers through a saved connection and returns a `snapshots` array. The connection needs a configured discovery driver and bound resources; missing prerequisites return `422`. A successful request records observed inventory but does not reserve a scheduler allocation or guarantee that every site resource was found.", - "POST /akoflow-api/workflow-definitions/import/": "This compatibility route uses the same portable YAML/JSON importer as `POST /workflow-definitions/`. Supply `name`, `spec.namespace`, and activities with valid dependencies and execution or simulation fields. The [workflow specification](/docs/internal/workflow-spec) defines the authoring format; a duplicate normalized workflow ID returns `422`.", - "POST /akoflow-api/workflow-definition-actions/duplicate/{workflowId}/": "Send a non-empty new `name`; `namespace` is optional and otherwise inherited. The source workflow must exist. AkôFlow generates new workflow and activity IDs from the new name and returns the independent definition with `201 Created`; an ID collision or invalid name returns `422`.", - "POST /akoflow-api/planning-sessions/{sessionId}/candidates/{candidateId}/select/": "The candidate must belong to this session and be feasible. No JSON body is required. A successful selection returns the saved schedule plan with `201 Created` and records the selected IDs on the session. The API permits selecting before the session completes; wait for final ranking unless choosing an early candidate intentionally.", - "POST /akoflow-api/planning-sessions/{sessionId}/cancel/": "No JSON body is required. Cancelling a queued or running session marks its algorithm runs and session as cancelled and requests cancellation of active work; success returns `204 No Content`. Cancelling an already-cancelled session also returns `204`. A missing, completed, or failed session returns `409 Conflict`, not `404`.", - "POST /akoflow-api/environments/{environmentId}/cloud-catalog/refresh/": "Discovers the catalog using the saved cloud connection. Without `async`, returns the catalog with `200 OK` or an error with `422`. With `async=true`, returns `202 Accepted` immediately and runs discovery in the background. Repeated async requests for the same environment while discovery is active do not start another job. `GET /environments/{environmentId}/cloud-catalog/` returns `404` until a catalog has been synchronized. This is not a VM provisioning request.", + "POST /akoflow-api/cloud-operations/{operationId}/reconcile-orphan/": + "No JSON body is required. Only an unstarted queued workflow operation with no queue job can be reconciled; its owning workflow must already be completed, failed or cancelled. The server marks that orphan operation cancelled and returns it with `200 OK`. Unknown operations return `404`; an active, published or otherwise ineligible operation returns `409`. This route does not cancel an active infrastructure job.", + "POST /akoflow-api/execution-runs/{runId}/preserve-cloud-workspaces/": + "Supply `downloadId`, `instanceId` and the actual `sha256` of a ready archive for `runs/{runId}.tar.gz` in `cloud-workspace:{instanceId}`. The run must be cancelled and every handle terminal. The server requires matching run/generation identity, an unpinned retained workspace set, and both source and relay checksum validation before releasing preserved workspace leases. It copies the verified archive to daemon-local storage; it does not remove source files or VMs. No standalone request is supplied because these values must come from your verified recovery evidence.", + "POST /akoflow-api/execution-runs/": + "`202 Accepted` publishes a queue job. The returned `id` is the job ID, while `aggregateId` is the submitted `run.id`. The worker validates the request before saving a run; `GET /execution-runs/{runId}/` can return `404` until then, or remain `404` if validation fails. Use the original `run.id` to read the run, and do not treat the queue job's status as the run's status. Repeating the same `run.id` returns the existing idempotent job rather than creating a second one.", + "POST /akoflow-api/execution-runs/{runId}/cancel/": + "No JSON body is required. The run must exist and the instance must be writable. AkôFlow interrupts unfinished activities, cancels queued execution work, marks the run cancelled, and returns the updated run with `200 OK`. Cancelling an already-cancelled run is idempotent and also returns `200`; an unknown run returns `404`.", + "POST /akoflow-api/execution-runs/{runId}/recovery/": + "The run must be failed or cancelled and its original queued execution request must still be available. Send `mode` as `continue` or `retry_failures`; omission defaults to `continue`. AkôFlow reuses completed activities whose workspaces remain available, queues the remaining activities, and returns the recovery job plus preview with `202 Accepted`. Inspect the run until it reaches a terminal status.", + "POST /akoflow-api/execution-runs/{runId}/activities/{activityId}/interrupt/": + "No JSON body is required. The instance must be writable and the activity must belong to an interruptible active execution. Success returns the updated task with `200 OK`; an unavailable interrupter returns `503`, while an activity that cannot be interrupted returns `409`.", + "POST /akoflow-api/environments/": + "The response echoes the submitted definition. The repository saves the environment, version, inventory, storage, and connections, but ignores `connectionChecks` and `connectorBindings` if they appear in the body. Read `GET /environments/{environmentId}/` for saved IDs and separately recorded connection checks. See the [environment YAML reference](/docs/reference/environment-yaml).", + "GET /akoflow-api/environments/": + "Each item is a saved environment definition. `connectionChecks` can include recent health records for its connections; these were recorded separately from the environment YAML. The repository does not populate `connectorBindings` from a submitted definition.", + "GET /akoflow-api/environments/{environmentId}/": + "Returns the saved definition, including IDs filled from the enclosing environment/version. `connectionChecks` comes from recent connection-health records, not from the submitted YAML. The repository does not populate `connectorBindings` from that document.", + "POST /akoflow-api/schedule-plans/": + "Returns the saved `plan`. If `plan.networkTopologyId` is empty, the server fills it from `networkTopology.id`. Submitted predictions are saved without recalculation.", + "GET /akoflow-api/environments/{environmentId}/cloud-catalog/": + "Returns the last synchronized catalog; this GET does not call the provider. Before the first successful refresh it returns `404`. Read catalog warnings as well as machine, image, and disk choices; missing price data does not make a machine free.", + "GET /akoflow-api/environments/{environmentId}/cloud-capacity-targets/": + "Returns enabled targets for this environment, ordered by name. Deleting a target disables its record, so it disappears from this list without erasing historical operations or instances that used it.", + "GET /akoflow-api/environments/{environmentId}/cloud-instances/": + "Returns this environment's saved instances newest first, including records marked destroyed. Inspect each instance's `status` and `destroyedAt` rather than treating the list length as the number of active VMs.", + "GET /akoflow-api/cloud-operations/": + "Returns all saved cloud operations, newest first, as an array. This route has no environment filter; use the returned `environmentId` and `capacityTargetId` to find your operation, then open its detail and events.", + "GET /akoflow-api/cloud-operations/{operationId}/": + "Returns the current operation or `404` for an unknown ID. `queued` and `running` are not final outcomes. A `failureReason` may appear while an operation is queued for a retry; rely on a terminal `completed`, `failed`, or `cancelled` status and inspect events for details.", + "GET /akoflow-api/cloud-operations/{operationId}/events/": + "Returns stored events in sequence order, or `404` for an unknown operation. The server may add parsed provider-log events before returning this list. An `operation.failed` event can describe a retry; check the operation's current status before treating it as a final failure.", + "GET /akoflow-api/cloud-instances/{instanceId}/provisioning-log/": + "Use a saved cloud instance ID, not a cloud operation ID. An unknown instance returns `404`. Before Terraform has written a log file, this route can return `200 OK` with a waiting message in `log`; follow the cloud operation status and events for the outcome.", + "GET /": + "This health route is at the daemon root, outside `/akoflow-api/`. With the API URL from the [API access tutorial](/docs/tutorials/api-access), the cURL command removes that prefix and requests `/`. A healthy daemon returns plain text `ok`; this check does not verify Docker, BuildKit, or a workflow runtime. Use [preflight](/docs/api/endpoints/instance/get-preflight) for those local capability checks.", + "GET /akoflow-api/preflight/": + "The response has `server`, `docker`, and `buildkit` objects, each with `available` and `message`. `server.available` is true when this route responds, but the Docker and BuildKit checks can be false while HTTP still returns `200 OK`. Inspect their individual values before running a local workflow. The checks run in the daemon process, not in your browser or shell.", + "GET /akoflow-api/console-sessions/{sessionId}/stream/": + "Open this URL with a WebSocket client after creating a console session. A successful upgrade returns `101 Switching Protocols` and carries terminal input and output over the socket; it is not a JSON response. An unknown session returns `404`. See the [interactive console guide](/docs/guides/operations/interactive-console) for session lifecycle.", + "POST /akoflow-api/provenance/sql/": + "Send a read-only `sql` query using `SELECT` or `WITH`; `parameters` supplies optional named values, and `page`/`pageSize` control results (at most 200 rows per page). Use `GET /provenance/sql/schema/` to see allowed tables and columns. The service enforces a 10-second timeout and rejects writes or restricted fields with `400`; an unavailable explorer returns `503`. See [provenance SQL guide](/docs/guides/data/provenance#query-with-read-only-sql).", + "POST /akoflow-api/provenance/sql/explain/": + "Send the same `sql` and optional named `parameters` as the read-only SQL route. This runs `EXPLAIN QUERY PLAN` for a permitted `SELECT` or `WITH` statement and returns plan rows, not the query's data rows. It uses the same read-only table/column restrictions and 10-second timeout; invalid SQL returns `400` and an unavailable explorer returns `503`.", + "PUT /akoflow-api/instance/": + "Send the complete current instance object with non-empty `id` and `name`; this route saves the supplied object, so preserve existing identity and metadata when changing one field. `transferBufferBytes` accepts 5–64 MiB; `0` selects the 8 MiB default. The [instance guide](/docs/guides/operations/instance-management#inspect-the-active-identity) reads the current object before updating it.", + "PUT /akoflow-api/environments/{environmentId}/": + "Read `GET /environments/{environmentId}/` before editing and send a complete environment definition; `environment.id` must match the path ID. The response echoes the submitted definition, while the repository ignores `connectionChecks` and `connectorBindings` in it. Replacement returns `404` when the environment does not exist and can return `422` when references prevent replacing its inventory. Use new environment and version IDs for revised inventory already used by scopes or plans; see the [environment YAML reference](/docs/reference/environment-yaml#compatibility-and-common-failures).", + "POST /akoflow-api/instance-activations/{instanceId}/": + "Use `default` to return to the writable instance or an ID returned by `POST /instances/import/` to open a read-only snapshot. See [instance management](/docs/guides/operations/instance-management#import-and-open-a-read-only-snapshot) for the switching procedure.", + "POST /akoflow-api/instances/import/": + "Send a ZIP archive as the request body with `Content-Type: application/zip`, not JSON. The archive must be a compatible AkôFlow export with its redaction marker, valid database checksum and schema, no symbolic links, at most 100,000 entries, at most 8 GiB compressed and 64 GiB expanded. Success returns `201 Created` with the new read-only snapshot `id`; import does not replace the active instance. See [instance management](/docs/guides/operations/instance-management#import-and-open-a-read-only-snapshot).", + "PUT /akoflow-api/user-preferences/{clientId}/": + "The path `clientId` must be 8–128 characters and overrides any body `clientId`. Send `theme` as `light` or `dark`; `animationsEnabled` is optional and defaults to false when omitted. The route saves preferences for this client profile and returns the stored record with `200 OK`; invalid ID or theme returns `422`. See [Personal preferences](/docs/guides/operations/personal-preferences).", + "POST /akoflow-api/factory-reset/": + "No JSON body is required. This permanently clears user catalog/database records while retaining the schema and system instance identity; the server also removes managed Kubernetes token files. Success is `204 No Content`. The database reset runs before token-directory cleanup, so a `422` from that cleanup can arrive after the catalog has already been cleared. Export a snapshot first; see [instance management](/docs/guides/operations/instance-management#factory-reset).", + "PUT /akoflow-api/environment-connections/{connectionId}/": + "Read the current connection in `GET /environments/{environmentId}/`, then send its complete object with the changed fields. Its `id` must match the path and `environmentId` must be non-empty. The route inserts a new connection or replaces saved fields, and returns the submitted object without running a health check. See the [HPC connection guide](/docs/guides/infrastructure/hpc-slurm#1-create-the-ssh-credential-and-proxy-aware-connection).", + "POST /akoflow-api/environment-connections/{connectionId}/health/": + "Checks a saved connection and records a health result. Inspect the returned `status` and `message`: an unreachable target returns `200 OK` with `status: offline` when the probe itself completed. The environment status is updated to `connected` or `unreachable`.", + "POST /akoflow-api/environment-connections/{connectionId}/discover/": + "Discovers through a saved connection and returns a `snapshots` array. The connection needs a configured discovery driver and bound resources; missing prerequisites return `422`. A successful request records observed inventory but does not reserve a scheduler allocation or guarantee that every site resource was found.", + "POST /akoflow-api/workflow-definitions/import/": + "This compatibility route uses the same portable YAML/JSON importer as `POST /workflow-definitions/`. Supply `name`, `spec.namespace`, and activities with valid dependencies and execution or simulation fields. The [workflow specification](/docs/internal/workflow-spec) defines the authoring format; a duplicate normalized workflow ID returns `422`.", + "POST /akoflow-api/workflow-definition-actions/duplicate/{workflowId}/": + "Send a non-empty new `name`; `namespace` is optional and otherwise inherited. The source workflow must exist. AkôFlow generates new workflow and activity IDs from the new name and returns the independent definition with `201 Created`; an ID collision or invalid name returns `422`.", + "POST /akoflow-api/planning-sessions/{sessionId}/candidates/{candidateId}/select/": + "The candidate must belong to this session and be feasible. No JSON body is required. A successful selection returns the saved schedule plan with `201 Created` and records the selected IDs on the session. The API permits selecting before the session completes; wait for final ranking unless choosing an early candidate intentionally.", + "POST /akoflow-api/planning-sessions/{sessionId}/cancel/": + "No JSON body is required. Cancelling a queued or running session marks its algorithm runs and session as cancelled and requests cancellation of active work; success returns `204 No Content`. Cancelling an already-cancelled session also returns `204`. A missing, completed, or failed session returns `409 Conflict`, not `404`.", + "POST /akoflow-api/environments/{environmentId}/cloud-catalog/refresh/": + "Discovers the catalog using the saved cloud connection. Without `async`, returns the catalog with `200 OK` or an error with `422`. With `async=true`, returns `202 Accepted` immediately and runs discovery in the background. Repeated async requests for the same environment while discovery is active do not start another job. `GET /environments/{environmentId}/cloud-catalog/` returns `404` until a catalog has been synchronized. This is not a VM provisioning request.", "POST /akoflow-api/environments/{environmentId}/cloud-capacity-targets/": [ "Use an existing cloud environment with a cloud runtime. Send `name`, `providerMachineType`, and `imageReference`. The server supplies an omitted `id`, sets `environmentId` from the path, and enables the target.", "Choose the provider-specific region, image, machine type, and network policy using the [GCP guide](/docs/guides/infrastructure/gcp). Set `configuration.projectId` to the validated project; Terraform reads it from the target, not the connection. The built-in worker requires `amd64`. Set `fixedZone` if needed; otherwise Terraform uses the first active zone in the region. The stored `zonePolicy` does not affect that choice. Set approved `configuration.sshSourceRanges` before provisioning; otherwise the Terraform target defaults SSH ingress to `0.0.0.0/0`.", "This route creates a schedulable capacity resource, not a VM. If it returns `422`, inspect the target list before retrying: the target can be saved before resource binding fails.", ].join("\n\n"), - "DELETE /akoflow-api/cloud-capacity-targets/{targetId}/": "No body is required. An active provisioned instance using this target blocks removal with `409 Conflict`; destroy it first. Success returns `204 No Content`, disables and renames the target record, and marks its capacity resource unschedulable. A missing or already-disabled target returns `404`. This does not itself destroy a VM.", - "POST /akoflow-api/cloud-instances/{instanceId}/configure/": "No JSON body is required. The instance ID must exist (`404` otherwise). This returns a queued cloud operation with `202 Accepted`; inspect its status and events before treating configuration as complete. The worker needs a public address and the target's machine-configuration versions; missing prerequisites can fail asynchronously. Repeating the same active action returns its existing operation; a different active action on the instance returns `409`.", - "POST /akoflow-api/cloud-instances/{instanceId}/destroy/": "No JSON body is required. The instance ID must exist (`404` otherwise). `202 Accepted` queues destruction; inspect the operation until it completes or fails before assuming the VM is gone. A lifecycle safety check can block destruction after acceptance. The same active action returns its existing operation; another active action on this instance returns `409`.", - "POST /akoflow-api/cloud-instances/{instanceId}/start/": "No JSON body is required. The instance ID must exist (`404` otherwise). `202 Accepted` queues a start operation; the worker accepts a stopped instance and treats an already-ready instance as a no-op. Other states can fail asynchronously. A repeated active start returns the existing operation; a different active action returns `409`.", - "POST /akoflow-api/cloud-instances/{instanceId}/stop/": "No JSON body is required. The instance ID must exist (`404` otherwise). `202 Accepted` queues a stop operation; the worker accepts a ready instance and treats an already-stopped instance as a no-op. Other states or lifecycle safety checks can fail after acceptance. A repeated active stop returns the existing operation; a different active action returns `409`.", - "POST /akoflow-api/cloud-instances/{instanceId}/validate/": "No JSON body is required. The instance ID must exist (`404` otherwise). `202 Accepted` queues validation against the target's configured checks; it does not by itself provision, configure, start, or stop the instance. Inspect the operation status and events for the result. A repeated active validation returns the existing operation; a different active action returns `409`.", - "DELETE /akoflow-api/environments/{environmentId}/": "No body is required. The environment must exist (`404` otherwise) and must not be referenced by an execution scope, network topology, schedule plan, execution data or transfers, console activity, or simulation scenario (`422` if referenced). Success returns `204 No Content` and removes its saved versions, inventory, and connections. Inspect dependencies before deleting; see [environments](/docs/guides/infrastructure/environments).", - "DELETE /akoflow-api/storages/{storageId}/entries/": "Pass the entry path as a URL query parameter, for example `?path=/approved/root/file.txt`; there is no JSON body. The storage must be registered, writable, and usable, and the path must be allowed by its adapter. Success returns `204 No Content`; adapter or policy errors return `422`. This removes the entry rather than merely deleting its catalog record.", - "POST /akoflow-api/resources/": "The example adds an inventory-only resource to the SimGrid environment from the [first-run tutorial](/docs/guides/workflows/first-run); that environment version must already exist. Use a unique `id`: reusing one updates the resource and still returns `201 Created`. This route does not create a runtime binding, so the example is not a runnable planning resource. Add an enabled binding through a complete environment definition before selecting it in a plan.", - "DELETE /akoflow-api/execution-scopes/{scopeId}/": "No body is required. A missing scope returns `404`; a scope referenced by a saved schedule plan cannot be deleted and returns `422`. Success returns `204 No Content`. Inspect plans before deleting a scope used by planning or execution.", - "DELETE /akoflow-api/console-sessions/{sessionId}/": "No body is required. Close a session when its terminal work is finished; the route returns `204 No Content` on success and `404` for an unknown session. Closing the Desktop detail page alone does not close a daemon-owned session. See the [interactive console guide](/docs/guides/operations/interactive-console).", - "POST /akoflow-api/ssh-keys/": "The tested example creates a new key. Choose an unused `id`: 1–64 ASCII letters, digits, `_`, or `-`, starting with a letter or digit. `comment` is optional. The daemon needs `ssh-keygen`.", - "POST /akoflow-api/ssh-keys/import/": "Send a unique `id` and your own `privateKey` as a JSON string. The ID follows the same SSH key ID rule; the key must be a non-empty OpenSSH private key that `ssh-keygen -y` can read. The response returns public metadata, not the private key. No example key is supplied because its bytes must come from your credential store.", - "POST /akoflow-api/kubernetes-tokens/": "Send an `id` and your own non-empty `token` as JSON strings. The ID must be 1–63 lowercase letters, digits, or hyphens, starting with a letter or digit. The response returns a `credentialRef`; it does not echo the token. Supply the token from your cluster's credential process.", - "POST /akoflow-api/cloud-credentials/": "Send `id`, `provider`, and `credential` with your actual provider credential. The ID follows the Kubernetes credential ID rule; `provider` must be `gcp`, `aws`, or `azure`, and `credential` must be valid JSON. Saving a credential does not validate provider access or make every provider operation available. The response returns a `credentialRef`.", - "POST /akoflow-api/environments/{environmentId}/cloud-instances/": "Send a `capacityTargetId` for a target in this environment. `name` and `sshUsername` are optional; the server generates an `instanceId` when omitted. `202 Accepted` means the operation was queued. Target ownership, connection, and credential checks run later, as does VM provisioning. Inspect the returned operation's status, failure reason, and events; an active provision for the same target returns the existing operation. Follow the [GCP guide](/docs/guides/infrastructure/gcp) for account-specific setup.", - "POST /akoflow-api/environments/{environmentId}/cloud-provisioning/": "This compatibility route accepts the same `capacityTargetId` request as [Provision Cloud Instance](/docs/api/endpoints/cloud/post-environments-environmentid-cloud-instances). It queues the same operation and returns `202 Accepted`; target, environment, credential, and provider failures can appear later in its status and events. Wait for completion before treating the VM as ready.", - "POST /akoflow-api/planning-sessions/": "Required: `id`, an existing `workflowVersionId`, `executionScopeId`, and `networkTopologyId`, plus at least one `algorithms` entry. Each algorithm ID must appear in `GET /planning-algorithms/`; duplicate IDs are rejected. The server sets status and timestamps. The example IDs require the SimGrid environment, scope, topology, and workflow to be registered first.", - "POST /akoflow-api/schedule-plans/import/": "Send a JSON object with a complete `plan`. Use a unique plan `id`, unique assignment IDs with matching `planId` values, and unique lifecycle action IDs with matching `schedulePlanId` and `dependsOn` references. Its workflow version, execution scope, topology, and resources must already exist. The server sets `source` to `imported`, checks the schedule against those saved records, and returns `422` if it is invalid. It saves the supplied predicted metrics without recalculating them. See the [saved-plan import example](/docs/guides/workflows/planning#import-a-saved-plan); a body filled with placeholder IDs would not pass validation.", - "POST /akoflow-api/storages/{storageId}/promote-data/": "Replace the example `path` with an existing file within the selected storage's approved root. `id` is optional; the server generates one when omitted. `workflowVersionId`, `runId`, and `activityId` are optional associations and should identify real records when supplied. The [storage guide](/docs/guides/infrastructure/storage) shows the browsing step.", - "POST /akoflow-api/storages/{storageId}/promote-artifact/": "Replace the example `path` with an existing `.sif` file within the selected storage's approved root. `id`, `name`, `version`, and `scope` have server defaults. The [storage guide](/docs/guides/infrastructure/storage) shows the browsing step; no file is uploaded or moved.", - "POST /akoflow-api/storages/{storageId}/downloads/": "Replace the example `path` with an existing file within the selected storage's approved browse root; directories require the archive route. `id` is optional. The response is a ready download record, not a frozen copy: `GET /storage-downloads/{downloadId}/content/` opens the path when you fetch it. See [Browse and manage storage](/docs/guides/infrastructure/storage) for download and checksum steps.", - "POST /akoflow-api/storages/{storageId}/checksum/": "Replace the example `path` with an existing file within the selected storage's approved browse root. The server reads the file and returns a `sha256:`-prefixed checksum; it does not queue a background job.", - "POST /akoflow-api/storages/{storageId}/copies/": "Replace the example `path` with a file in the source storage and `destinationStorageId` with a registered, writable storage ID. `id` is optional. The copy runs in the background at the same path in the destination. Inspect `GET /storage-downloads/{downloadId}/` until its status is `completed` or `failed`; `202 Accepted` does not mean the bytes have arrived.", - "POST /akoflow-api/storages/{storageId}/archives/": "Replace the example `path` with an existing directory in a writable storage. `id` is optional. AkôFlow writes a `.tar.gz` beside that directory and returns a queued record. Inspect `GET /storage-downloads/{downloadId}/` until its status is `ready` or `failed`; only a ready archive can be streamed through the download content route.", - "POST /akoflow-api/storages/{storageId}/index-runs/": "Indexing must be enabled for this registered storage. Use a unique `id` in place of the example or omit it for a generated ID. The current service completes the bounded scan before responding; the returned record is `completed` or the request fails. The `202 Accepted` status does not mean this scan continues in the background.", - "POST /akoflow-api/build-contexts/": "To upload bytes, send multipart form data with file field `context`. The JSON form only records metadata for bytes already in the artifact store; it requires `digest`, `storageUri`, and positive `sizeBytes`. A browser-local path is not a server build context.", - "POST /akoflow-api/artifact-builds/": "Upload the build context first and use its returned digest as `contextDigest`; `artifactVersionId` must identify a saved artifact version. Required fields are `id`, `artifactVersionId`, `contextDigest`, `recipeDigest`, and `cacheKey`. An existing cache key returns that build with `200 OK`; a new specification returns `201 Created`. Creating a specification does not start a build run. See [Build an executable](/docs/guides/data/build-executable) for the simpler Docker-image path.", - "POST /akoflow-api/artifacts/docker/": "Required: `artifactId`, `version`, and a Docker image reference without whitespace. `architecture` defaults to `amd64`. This registers a version and build specification; it does not pull the image or create a SIF yet. Use `build.id` from the response in `POST /artifact-builds/{buildId}/runs/` to start conversion. The build run returns a separate ID.", - "POST /akoflow-api/artifact-builds/{buildId}/runs/": "`buildId` must identify an existing build specification. This request starts a build run and returns its record with `202 Accepted`; inspect `GET /build-runs/{runId}/` for its outcome. The request has no JSON body.", - "POST /akoflow-api/artifact-materializations/": "This endpoint stores a caller-supplied record; it does not copy or verify artifact bytes. Supply a distinct `id`, existing `variantId` and `resourceId`, the variant's `sha256:` digest, `destinationPath`, and a truthful `status`. The database checks references and digest format but does not prove the bytes exist at the destination. If supplied, `environmentId` must be the environment **version** ID, despite the field name. Use [Inspect artifact locations](/docs/guides/data/artifact-locations) to inspect evidence recorded by execution.", - "POST /akoflow-api/console-sessions/": "Replace the example `resourceId` with a saved resource that has a connected interactive runtime; `actorId` is optional. A successful request starts the terminal and returns a `connected` session. Resolution or startup failure returns `422`, not a successful session record. Use the [console guide](/docs/guides/operations/interactive-console) for streaming and closure.", - "POST /akoflow-api/console-commands/": "Replace the example `resourceId` with a saved resource that supports one-shot commands. `resourceId` and `command` are required; `timeoutSeconds` defaults to 30 and cannot exceed 3600. The request waits for the runner and returns a command record. A runner failure can return `201 Created` with `status: failed`; inspect `status`, `exitCode`, and `failure` instead of treating HTTP status as command success.", - "POST /akoflow-api/connection-tests/": "The example tests the local server. For SSH, agent, or Kubernetes, send the connection fields and credentials required by that type instead. This route tests the supplied connection without saving it. It returns `200 OK` with `healthy` and `message`; `healthy: false` means the probe failed even though the HTTP request succeeded. Cloud connections are not handled here.", - "POST /akoflow-api/machine-configuration-validations/": "Send `playbookYaml` containing the Ansible playbook text. Valid input returns `200 OK` with `valid: true` and a content hash; invalid input returns `422` with `valid: false` and errors. This validates structure only and does not run a playbook or provision a machine.", - "POST /akoflow-api/machine-configurations/": "`name` is required. `id` is optional and generated when omitted. The server sets `ownership: user` and `enabled: true`. Create a version separately; this request does not validate or execute a playbook.", - "POST /akoflow-api/machine-configurations/{configurationId}/versions/": "Create the configuration first, then use its returned ID as `configurationId` (the preceding example uses `example-machine-setup`). The path ID overrides any body `machineConfigurationId`. Send a positive `version` and valid `playbookYaml`; `id` is optional and `status` defaults to `draft`. The response is the stored configuration with its versions, not just the new version.", - "POST /akoflow-api/cloud-credentials/validate/": "Send `provider`, your actual provider `credential` JSON, and the project/region fields required by that provider. This route performs a live catalog discovery and does not save the credential. The current server registers only a GCP catalog adapter; AWS or Azure validation returns `422` as unsupported even though those credential records can be stored. No generic credential body can establish access; use the [GCP guide](/docs/guides/infrastructure/gcp) for the current provider path.", + "DELETE /akoflow-api/cloud-capacity-targets/{targetId}/": + "No body is required. An active provisioned instance using this target blocks removal with `409 Conflict`; destroy it first. Success returns `204 No Content`, disables and renames the target record, and marks its capacity resource unschedulable. A missing or already-disabled target returns `404`. This does not itself destroy a VM.", + "POST /akoflow-api/cloud-instances/{instanceId}/configure/": + "No JSON body is required. The instance ID must exist (`404` otherwise). This returns a queued cloud operation with `202 Accepted`; inspect its status and events before treating configuration as complete. The worker needs a public address and the target's machine-configuration versions; missing prerequisites can fail asynchronously. Repeating the same active action returns its existing operation; a different active action on the instance returns `409`.", + "POST /akoflow-api/cloud-instances/{instanceId}/destroy/": + "No JSON body is required. The instance ID must exist (`404` otherwise). `202 Accepted` queues destruction; inspect the operation until it completes or fails before assuming the VM is gone. A lifecycle safety check can block destruction after acceptance. The same active action returns its existing operation; another active action on this instance returns `409`.", + "POST /akoflow-api/cloud-instances/{instanceId}/start/": + "No JSON body is required. The instance ID must exist (`404` otherwise). `202 Accepted` queues a start operation; the worker accepts a stopped instance and treats an already-ready instance as a no-op. Other states can fail asynchronously. A repeated active start returns the existing operation; a different active action returns `409`.", + "POST /akoflow-api/cloud-instances/{instanceId}/stop/": + "No JSON body is required. The instance ID must exist (`404` otherwise). `202 Accepted` queues a stop operation; the worker accepts a ready instance and treats an already-stopped instance as a no-op. Other states or lifecycle safety checks can fail after acceptance. A repeated active stop returns the existing operation; a different active action returns `409`.", + "POST /akoflow-api/cloud-instances/{instanceId}/validate/": + "No JSON body is required. The instance ID must exist (`404` otherwise). `202 Accepted` queues validation against the target's configured checks; it does not by itself provision, configure, start, or stop the instance. Inspect the operation status and events for the result. A repeated active validation returns the existing operation; a different active action returns `409`.", + "DELETE /akoflow-api/environments/{environmentId}/": + "No body is required. The environment must exist (`404` otherwise) and must not be referenced by an execution scope, network topology, schedule plan, execution data or transfers, console activity, or simulation scenario (`422` if referenced). Success returns `204 No Content` and removes its saved versions, inventory, and connections. Inspect dependencies before deleting; see [environments](/docs/guides/infrastructure/environments).", + "DELETE /akoflow-api/storages/{storageId}/entries/": + "Pass the entry path as a URL query parameter, for example `?path=/approved/root/file.txt`; there is no JSON body. The storage must be registered, writable, and usable, and the path must be allowed by its adapter. Success returns `204 No Content`; adapter or policy errors return `422`. This removes the entry rather than merely deleting its catalog record.", + "POST /akoflow-api/resources/": + "The example adds an inventory-only resource to the SimGrid environment from the [first-run tutorial](/docs/guides/workflows/first-run); that environment version must already exist. Use a unique `id`: reusing one updates the resource and still returns `201 Created`. This route does not create a runtime binding, so the example is not a runnable planning resource. Add an enabled binding through a complete environment definition before selecting it in a plan.", + "DELETE /akoflow-api/execution-scopes/{scopeId}/": + "No body is required. A missing scope returns `404`; a scope referenced by a saved schedule plan cannot be deleted and returns `422`. Success returns `204 No Content`. Inspect plans before deleting a scope used by planning or execution.", + "DELETE /akoflow-api/console-sessions/{sessionId}/": + "No body is required. Close a session when its terminal work is finished; the route returns `204 No Content` on success and `404` for an unknown session. Closing the Desktop detail page alone does not close a daemon-owned session. See the [interactive console guide](/docs/guides/operations/interactive-console).", + "POST /akoflow-api/ssh-keys/": + "The tested example creates a new key. Choose an unused `id`: 1–64 ASCII letters, digits, `_`, or `-`, starting with a letter or digit. `comment` is optional. The daemon needs `ssh-keygen`.", + "POST /akoflow-api/ssh-keys/import/": + "Send a unique `id` and your own `privateKey` as a JSON string. The ID follows the same SSH key ID rule; the key must be a non-empty OpenSSH private key that `ssh-keygen -y` can read. The response returns public metadata, not the private key. No example key is supplied because its bytes must come from your credential store.", + "POST /akoflow-api/kubernetes-tokens/": + "Send an `id` and your own non-empty `token` as JSON strings. The ID must be 1–63 lowercase letters, digits, or hyphens, starting with a letter or digit. The response returns a `credentialRef`; it does not echo the token. Supply the token from your cluster's credential process.", + "POST /akoflow-api/cloud-credentials/": + "Send `id`, `provider`, and `credential` with your actual provider credential. The ID follows the Kubernetes credential ID rule; `provider` must be `gcp`, `aws`, or `azure`, and `credential` must be valid JSON. Saving a credential does not validate provider access or make every provider operation available. The response returns a `credentialRef`.", + "POST /akoflow-api/environments/{environmentId}/cloud-instances/": + "Send a `capacityTargetId` for a target in this environment. `name` and `sshUsername` are optional; the server generates an `instanceId` when omitted. `202 Accepted` means the operation was queued. Target ownership, connection, and credential checks run later, as does VM provisioning. Inspect the returned operation's status, failure reason, and events; an active provision for the same target returns the existing operation. Follow the [GCP guide](/docs/guides/infrastructure/gcp) for account-specific setup.", + "POST /akoflow-api/environments/{environmentId}/cloud-provisioning/": + "This compatibility route accepts the same `capacityTargetId` request as [Provision Cloud Instance](/docs/api/endpoints/cloud/post-environments-environmentid-cloud-instances). It queues the same operation and returns `202 Accepted`; target, environment, credential, and provider failures can appear later in its status and events. Wait for completion before treating the VM as ready.", + "POST /akoflow-api/planning-sessions/": + "Required: `id`, an existing `workflowVersionId`, `executionScopeId`, and `networkTopologyId`, plus at least one `algorithms` entry. Each algorithm ID must appear in `GET /planning-algorithms/`; duplicate IDs are rejected. The server sets status and timestamps. The example IDs require the SimGrid environment, scope, topology, and workflow to be registered first.", + "POST /akoflow-api/schedule-plans/import/": + "Send a JSON object with a complete `plan`. Use a unique plan `id`, unique assignment IDs with matching `planId` values, and unique lifecycle action IDs with matching `schedulePlanId` and `dependsOn` references. Its workflow version, execution scope, topology, and resources must already exist. The server sets `source` to `imported`, checks the schedule against those saved records, and returns `422` if it is invalid. It saves the supplied predicted metrics without recalculating them. See the [saved-plan import example](/docs/guides/workflows/planning#import-a-saved-plan); a body filled with placeholder IDs would not pass validation.", + "POST /akoflow-api/storages/{storageId}/promote-data/": + "Replace the example `path` with an existing file within the selected storage's approved root. `id` is optional; the server generates one when omitted. `workflowVersionId`, `runId`, and `activityId` are optional associations and should identify real records when supplied. The [storage guide](/docs/guides/infrastructure/storage) shows the browsing step.", + "POST /akoflow-api/storages/{storageId}/promote-artifact/": + "Replace the example `path` with an existing `.sif` file within the selected storage's approved root. `id`, `name`, `version`, and `scope` have server defaults. The [storage guide](/docs/guides/infrastructure/storage) shows the browsing step; no file is uploaded or moved.", + "POST /akoflow-api/storages/{storageId}/downloads/": + "Replace the example `path` with an existing file within the selected storage's approved browse root; directories require the archive route. `id` is optional. The response is a ready download record, not a frozen copy: `GET /storage-downloads/{downloadId}/content/` opens the path when you fetch it. See [Browse and manage storage](/docs/guides/infrastructure/storage) for download and checksum steps.", + "POST /akoflow-api/storages/{storageId}/checksum/": + "Replace the example `path` with an existing file within the selected storage's approved browse root. The server reads the file and returns a `sha256:`-prefixed checksum; it does not queue a background job.", + "POST /akoflow-api/storages/{storageId}/copies/": + "Replace the example `path` with a file in the source storage and `destinationStorageId` with a registered, writable storage ID. `id` is optional. The copy runs in the background at the same path in the destination. Inspect `GET /storage-downloads/{downloadId}/` until its status is `completed` or `failed`; `202 Accepted` does not mean the bytes have arrived.", + "POST /akoflow-api/storages/{storageId}/archives/": + "Replace the example `path` with an existing directory in a writable storage. `id` is optional. AkôFlow writes a `.tar.gz` beside that directory and returns a queued record. Inspect `GET /storage-downloads/{downloadId}/` until its status is `ready` or `failed`; only a ready archive can be streamed through the download content route.", + "POST /akoflow-api/storages/{storageId}/index-runs/": + "Indexing must be enabled for this registered storage. Use a unique `id` in place of the example or omit it for a generated ID. The current service completes the bounded scan before responding; the returned record is `completed` or the request fails. The `202 Accepted` status does not mean this scan continues in the background.", + "POST /akoflow-api/build-contexts/": + "To upload bytes, send multipart form data with file field `context`. The JSON form only records metadata for bytes already in the artifact store; it requires `digest`, `storageUri`, and positive `sizeBytes`. A browser-local path is not a server build context.", + "POST /akoflow-api/artifact-builds/": + "Upload the build context first and use its returned digest as `contextDigest`; `artifactVersionId` must identify a saved artifact version. Required fields are `id`, `artifactVersionId`, `contextDigest`, `recipeDigest`, and `cacheKey`. An existing cache key returns that build with `200 OK`; a new specification returns `201 Created`. Creating a specification does not start a build run. See [Build an executable](/docs/guides/data/build-executable) for the simpler Docker-image path.", + "POST /akoflow-api/artifacts/docker/": + "Required: `artifactId`, `version`, and a Docker image reference without whitespace. `architecture` defaults to `amd64`. This registers a version and build specification; it does not pull the image or create a SIF yet. Use `build.id` from the response in `POST /artifact-builds/{buildId}/runs/` to start conversion. The build run returns a separate ID.", + "POST /akoflow-api/artifact-builds/{buildId}/runs/": + "`buildId` must identify an existing build specification. This request starts a build run and returns its record with `202 Accepted`; inspect `GET /build-runs/{runId}/` for its outcome. The request has no JSON body.", + "POST /akoflow-api/artifact-materializations/": + "This endpoint stores a caller-supplied record; it does not copy or verify artifact bytes. Supply a distinct `id`, existing `variantId` and `resourceId`, the variant's `sha256:` digest, `destinationPath`, and a truthful `status`. The database checks references and digest format but does not prove the bytes exist at the destination. If supplied, `environmentId` must be the environment **version** ID, despite the field name. Use [Inspect artifact locations](/docs/guides/data/artifact-locations) to inspect evidence recorded by execution.", + "POST /akoflow-api/console-sessions/": + "Replace the example `resourceId` with a saved resource that has a connected interactive runtime; `actorId` is optional. A successful request starts the terminal and returns a `connected` session. Resolution or startup failure returns `422`, not a successful session record. Use the [console guide](/docs/guides/operations/interactive-console) for streaming and closure.", + "POST /akoflow-api/console-commands/": + "Replace the example `resourceId` with a saved resource that supports one-shot commands. `resourceId` and `command` are required; `timeoutSeconds` defaults to 30 and cannot exceed 3600. The request waits for the runner and returns a command record. A runner failure can return `201 Created` with `status: failed`; inspect `status`, `exitCode`, and `failure` instead of treating HTTP status as command success.", + "POST /akoflow-api/connection-tests/": + "The example tests the local server. For SSH, agent, or Kubernetes, send the connection fields and credentials required by that type instead. This route tests the supplied connection without saving it. It returns `200 OK` with `healthy` and `message`; `healthy: false` means the probe failed even though the HTTP request succeeded. Cloud connections are not handled here.", + "POST /akoflow-api/machine-configuration-validations/": + "Send `playbookYaml` containing the Ansible playbook text. Valid input returns `200 OK` with `valid: true` and a content hash; invalid input returns `422` with `valid: false` and errors. This validates structure only and does not run a playbook or provision a machine.", + "POST /akoflow-api/machine-configurations/": + "`name` is required. `id` is optional and generated when omitted. The server sets `ownership: user` and `enabled: true`. Create a version separately; this request does not validate or execute a playbook.", + "POST /akoflow-api/machine-configurations/{configurationId}/versions/": + "Create the configuration first, then use its returned ID as `configurationId` (the preceding example uses `example-machine-setup`). The path ID overrides any body `machineConfigurationId`. Send a positive `version` and valid `playbookYaml`; `id` is optional and `status` defaults to `draft`. The response is the stored configuration with its versions, not just the new version.", + "POST /akoflow-api/cloud-credentials/validate/": + "Send `provider`, your actual provider `credential` JSON, and the project/region fields required by that provider. This route performs a live catalog discovery and does not save the credential. The current server registers only a GCP catalog adapter; AWS or Azure validation returns `422` as unsupported even though those credential records can be stored. No generic credential body can establish access; use the [GCP guide](/docs/guides/infrastructure/gcp) for the current provider path.", }; const verifiedRequestExamples = { @@ -421,7 +592,8 @@ const verifiedRequestExamples = { architecture: "amd64", }, "POST /akoflow-api/machine-configuration-validations/": { - playbookYaml: "- hosts: all\n tasks:\n - ansible.builtin.debug:\n msg: ready\n", + playbookYaml: + "- hosts: all\n tasks:\n - ansible.builtin.debug:\n msg: ready\n", }, "POST /akoflow-api/machine-configurations/": { id: "example-machine-setup", @@ -429,7 +601,8 @@ const verifiedRequestExamples = { }, "POST /akoflow-api/machine-configurations/{configurationId}/versions/": { version: 1, - playbookYaml: "- hosts: all\n tasks:\n - ansible.builtin.debug:\n msg: ready\n", + playbookYaml: + "- hosts: all\n tasks:\n - ansible.builtin.debug:\n msg: ready\n", }, "POST /akoflow-api/provenance/sql/": { sql: "SELECT id, status FROM execution_runs WHERE status = :status", @@ -451,6 +624,7 @@ const verifiedRequestExamples = { }; const requestWithoutStandaloneExample = new Set([ + "POST /akoflow-api/execution-runs/{runId}/preserve-cloud-workspaces/", "POST /akoflow-api/schedule-plans/import/", "POST /akoflow-api/environments/{environmentId}/cloud-capacity-targets/", "POST /akoflow-api/environments/{environmentId}/cloud-instances/", @@ -503,21 +677,6 @@ function replaceParameters(routePath) { return routePath.replace(/\{([^}]+)\}/g, (_, name) => `<${name}>`); } -function extractFunctionBody(source, handler) { - const match = new RegExp( - `func (?:\\(h \\*Handler\\) )?${handler}\\([^\\n]*\\) \\{`, - ).exec(source); - if (!match) return ""; - const start = match.index + match[0].length; - let depth = 1; - for (let index = start; index < source.length; index += 1) { - if (source[index] === "{") depth += 1; - if (source[index] === "}" && --depth === 0) - return source.slice(start, index); - } - return ""; -} - function extractQueryParameters(body) { return [ ...new Set( @@ -605,7 +764,10 @@ function buildStructIndex(sources) { const existing = index.get(match[1]); // Distinct packages can use the same short type name. Never borrow // one package's fields for an unrelated request. - if (existing === null || (existing && JSON.stringify(existing) !== JSON.stringify(fields))) { + if ( + existing === null || + (existing && JSON.stringify(existing) !== JSON.stringify(fields)) + ) { index.set(match[1], null); } else if (!existing) { index.set(match[1], fields); @@ -613,7 +775,12 @@ function buildStructIndex(sources) { if (packageName) { const key = `${packageName}.${match[1]}`; const known = qualified.get(key); - qualified.set(key, known && JSON.stringify(known) !== JSON.stringify(fields) ? null : fields); + qualified.set( + key, + known && JSON.stringify(known) !== JSON.stringify(fields) + ? null + : fields, + ); } } } @@ -627,10 +794,13 @@ function buildStructIndex(sources) { // A domain alias must not replace a concrete type with the same name in // another package (for example console.Request vs simgrid.Request). if (index.has(alias)) continue; - const fields = target.includes(".") ? qualified.get(target) : index.get(target); + const fields = target.includes(".") + ? qualified.get(target) + : index.get(target); if (fields !== undefined) index.set(alias, fields); } - for (const [qualifiedName, fields] of qualified) index.set(qualifiedName, fields); + for (const [qualifiedName, fields] of qualified) + index.set(qualifiedName, fields); return index; } @@ -691,10 +861,18 @@ function extractHandlerFieldTypes(handlerSource) { function sampleValue(typeName, structIndex, depth = 0) { const clean = typeName.replace(/^\*+/, "").trim(); if (/^\[\]/.test(clean)) { - const item = sampleValue(clean.replace(/^\[\]/, ""), structIndex, depth + 1); + const item = sampleValue( + clean.replace(/^\[\]/, ""), + structIndex, + depth + 1, + ); // An unknown or depth-limited item would render as [{}], which looks // like a real row without documenting any of its fields. - return item === null || (item && typeof item === "object" && !Array.isArray(item) && Object.keys(item).length === 0) + return item === null || + (item && + typeof item === "object" && + !Array.isArray(item) && + Object.keys(item).length === 0) ? [] : [item]; } @@ -898,7 +1076,8 @@ function endpointDocument(endpoint, position) { ? `## Runnable SimGrid request\n\nUse [\`akoflow-examples/simulation/workflow.yaml\`](https://github.com/UFFeScience/akoflow-examples/blob/main/simulation/workflow.yaml) **instead of** the create-workflow request in the [first-run tutorial](/docs/guides/workflows/first-run). Keep the other five requests in order so their referenced IDs exist. Do not submit the same workflow through both routes.\n\n` : `## Runnable SimGrid request\n\nThe [first-run tutorial](/docs/guides/workflows/first-run) submits [\`akoflow-examples/simulation/${runnableFile}\`](https://github.com/UFFeScience/akoflow-examples/blob/main/simulation/${runnableFile}) in a six-request sequence. Follow that order so referenced IDs exist.\n\n` : ""; - const verifiedNote = verifiedRequestNotes[`${endpoint.method} ${endpoint.path}`]; + const verifiedNote = + verifiedRequestNotes[`${endpoint.method} ${endpoint.path}`]; const verifiedSection = verifiedNote ? `## Handler-checked behavior\n\n${verifiedNote}\n\n` : ""; @@ -907,10 +1086,11 @@ function endpointDocument(endpoint, position) { ? `## Response shape\n\n${checkedResponseNote}\n\n` : ""; const requestExample = - runnableFile || requestWithoutStandaloneExample.has(`${endpoint.method} ${endpoint.path}`) + runnableFile || + requestWithoutStandaloneExample.has(`${endpoint.method} ${endpoint.path}`) ? null - : verifiedRequestExamples[`${endpoint.method} ${endpoint.path}`] ?? - endpoint.request?.example; + : (verifiedRequestExamples[`${endpoint.method} ${endpoint.path}`] ?? + endpoint.request?.example); return `--- title: ${JSON.stringify(title)} sidebar_label: ${JSON.stringify(`${endpoint.method} ${relativePath}`)} @@ -953,6 +1133,9 @@ See the [${endpoint.group} guide](${groupMetadata[endpoint.group][0]}) for relat const source = await readFile(routerFile, "utf8"); const handlerSource = await readFile(handlerFile, "utf8"); +const completeHandlerSource = ( + await collectGoSources(path.dirname(handlerFile)) +).join("\n"); const goSources = await collectGoSources(sourceRoot); const structIndex = buildStructIndex(goSources); const returnTypeIndex = buildReturnTypeIndex(goSources); @@ -965,7 +1148,7 @@ for (const match of source.matchAll(routePattern)) { const [, method, routePath, handler] = match; const group = groupFor(routePath); const handlerBody = extractFunctionBody( - `${handlerSource}\n${source}`, + `${completeHandlerSource}\n${source}`, handler, ); const title = @@ -976,15 +1159,31 @@ for (const match of source.matchAll(routePattern)) { description: endpointDescription(endpoint), queryParameters: extractQueryParameters(handlerBody), successStatuses: - delegatedSuccessStatuses[handler] ?? extractSuccessStatuses(handlerBody, handler), + delegatedSuccessStatuses[handler] ?? + extractSuccessStatuses(handlerBody, handler), guide: groupMetadata[group][0], }; - const request = handler === "SaveBuildContext" - ? { type: "uploaded build context", example: null, mediaType: "multipart/form-data", fileName: "context.tar.gz", multipartField: "context" } - : handler === "ImportArchiveInstance" - ? { type: "ZIP instance archive (maximum 8 GiB)", example: null, mediaType: "application/zip", fileName: "instance.zip" } - : extractRequestContract(handlerBody, structIndex); - if ((handler === "CreateEnvironment" || handler === "ReplaceEnvironment") && request?.example) { + const request = + handler === "SaveBuildContext" + ? { + type: "uploaded build context", + example: null, + mediaType: "multipart/form-data", + fileName: "context.tar.gz", + multipartField: "context", + } + : handler === "ImportArchiveInstance" + ? { + type: "ZIP instance archive (maximum 8 GiB)", + example: null, + mediaType: "application/zip", + fileName: "instance.zip", + } + : extractRequestContract(handlerBody, structIndex); + if ( + (handler === "CreateEnvironment" || handler === "ReplaceEnvironment") && + request?.example + ) { delete request.example.connectionChecks; delete request.example.connectorBindings; } @@ -999,8 +1198,17 @@ for (const match of source.matchAll(routePattern)) { ), ...(checkedResponseShapes[handler] || {}), }; - if (["ListEnvironments", "CreateEnvironment", "GetEnvironment", "ReplaceEnvironment"].includes(handler)) { - const definitions = Array.isArray(response.example) ? response.example : [response.example]; + if ( + [ + "ListEnvironments", + "CreateEnvironment", + "GetEnvironment", + "ReplaceEnvironment", + ].includes(handler) + ) { + const definitions = Array.isArray(response.example) + ? response.example + : [response.example]; for (const definition of definitions) { if (!definition || typeof definition !== "object") continue; delete definition.connectorBindings; @@ -1021,22 +1229,29 @@ if (endpoints.length === 0) { throw new Error(`No API routes found in ${routerFile}`); } -const undocumentedQueryParameters = [...new Set(endpoints.flatMap((endpoint) => endpoint.queryParameters))] - .filter((name) => !queryDescriptions[name]); +const undocumentedQueryParameters = [ + ...new Set(endpoints.flatMap((endpoint) => endpoint.queryParameters)), +].filter((name) => !queryDescriptions[name]); if (undocumentedQueryParameters.length > 0) { - throw new Error(`Query parameters need descriptions: ${undocumentedQueryParameters.join(", ")}`); + throw new Error( + `Query parameters need descriptions: ${undocumentedQueryParameters.join(", ")}`, + ); } const missingCheckedResponses = Object.keys(checkedResponseShapes).filter( (handler) => !endpoints.some((endpoint) => endpoint.handler === handler), ); if (missingCheckedResponses.length > 0) { - throw new Error(`Checked response handlers are missing from the router: ${missingCheckedResponses.join(", ")}`); + throw new Error( + `Checked response handlers are missing from the router: ${missingCheckedResponses.join(", ")}`, + ); } for (const endpoint of endpoints) { if (/(?:^|\.)[A-Za-z0-9_]*Request$/.test(endpoint.response.type)) { - throw new Error(`Request type used as a response for ${endpoint.method} ${endpoint.path}: ${endpoint.response.type}`); + throw new Error( + `Request type used as a response for ${endpoint.method} ${endpoint.path}: ${endpoint.response.type}`, + ); } } @@ -1044,7 +1259,12 @@ function syntheticPlaceholderPath(value, path = "$") { if (value === null) return path; if (Array.isArray(value)) { for (const [index, item] of value.entries()) { - if (item && typeof item === "object" && !Array.isArray(item) && Object.keys(item).length === 0) + if ( + item && + typeof item === "object" && + !Array.isArray(item) && + Object.keys(item).length === 0 + ) return `${path}[${index}]`; const found = syntheticPlaceholderPath(item, `${path}[${index}]`); if (found) return found; @@ -1061,23 +1281,37 @@ function syntheticPlaceholderPath(value, path = "$") { for (const endpoint of endpoints) { const invalidPath = syntheticPlaceholderPath(endpoint.response.example ?? {}); if (invalidPath) - throw new Error(`Ambiguous placeholder in ${endpoint.method} ${endpoint.path} response example at ${invalidPath}`); - const invalidRequestPath = syntheticPlaceholderPath(endpoint.request?.example ?? {}); + throw new Error( + `Ambiguous placeholder in ${endpoint.method} ${endpoint.path} response example at ${invalidPath}`, + ); + const invalidRequestPath = syntheticPlaceholderPath( + endpoint.request?.example ?? {}, + ); if (invalidRequestPath) - throw new Error(`Ambiguous placeholder in ${endpoint.method} ${endpoint.path} request example at ${invalidRequestPath}`); + throw new Error( + `Ambiguous placeholder in ${endpoint.method} ${endpoint.path} request example at ${invalidRequestPath}`, + ); } const staleCheckedNotes = Object.keys(verifiedRequestNotes).filter( - (route) => !endpoints.some((endpoint) => route === `${endpoint.method} ${endpoint.path}`), + (route) => + !endpoints.some( + (endpoint) => route === `${endpoint.method} ${endpoint.path}`, + ), ); if (staleCheckedNotes.length > 0) { - throw new Error(`Handler-checked notes refer to missing routes: ${staleCheckedNotes.join(", ")}`); + throw new Error( + `Handler-checked notes refer to missing routes: ${staleCheckedNotes.join(", ")}`, + ); } const undocumentedMutations = endpoints .filter((endpoint) => endpoint.method !== "GET") .map((endpoint) => `${endpoint.method} ${endpoint.path}`) - .filter((route) => !verifiedRequestNotes[route] && !runnableSimulationRequests[route]); + .filter( + (route) => + !verifiedRequestNotes[route] && !runnableSimulationRequests[route], + ); if (undocumentedMutations.length > 0) { throw new Error( `Mutating API routes need handler-checked notes or a runnable request: ${undocumentedMutations.join(", ")}`, diff --git a/docs/scripts/go-function-body.mjs b/docs/scripts/go-function-body.mjs new file mode 100644 index 00000000..43dbce64 --- /dev/null +++ b/docs/scripts/go-function-body.mjs @@ -0,0 +1,47 @@ +// Extract handler bodies without treating braces in Go strings/comments as code. +export function extractFunctionBody(source, handler) { + const escaped = handler.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const match = new RegExp( + `func\\s+(?:\\(h \\*Handler\\)\\s+)?${escaped}\\([^)]*\\)\\s*\\{`, + ).exec(source); + if (!match) return ""; + const start = match.index + match[0].length; + let depth = 1; + let state = "code"; + for (let index = start; index < source.length; index += 1) { + const character = source[index]; + const next = source[index + 1]; + if (state === "line-comment") { + if (character === "\n") state = "code"; + continue; + } + if (state === "block-comment") { + if (character === "*" && next === "/") { + state = "code"; + index += 1; + } + continue; + } + if (state === "raw-string") { + if (character === "`") state = "code"; + continue; + } + if (state === '"' || state === "'") { + if (character === "\\") index += 1; + else if (character === state) state = "code"; + continue; + } + if (character === "/" && next === "/") { + state = "line-comment"; + index += 1; + } else if (character === "/" && next === "*") { + state = "block-comment"; + index += 1; + } else if (character === "`") state = "raw-string"; + else if (character === '"' || character === "'") state = character; + else if (character === "{") depth += 1; + else if (character === "}" && --depth === 0) + return source.slice(start, index); + } + return ""; +} diff --git a/docs/scripts/go-function-body.test.mjs b/docs/scripts/go-function-body.test.mjs new file mode 100644 index 00000000..efb19b7a --- /dev/null +++ b/docs/scripts/go-function-body.test.mjs @@ -0,0 +1,43 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { extractFunctionBody } from "./go-function-body.mjs"; + +test("finds a handler in any package source file with multiline arguments", () => { + const source = `package handler +func (h *Handler) WorkspaceStorageBudget( + w http.ResponseWriter, + r *http.Request, +) { + observer := h.storage.(interface { Budget() string }) + writeJSON(w, http.StatusOK, observer) +}`; + assert.match( + extractFunctionBody(source, "WorkspaceStorageBudget"), + /StatusOK/, + ); + assert.equal(extractFunctionBody(source, "MissingHandler"), ""); +}); + +test("ignores Go comments, escaped strings, raw strings and rune braces", () => { + const source = [ + "func (h *Handler) Inspect(w http.ResponseWriter, r *http.Request) {", + ' message := "} escaped \\" {"', + " raw := `}} raw { braces`", + " rune := '}'", + " // } ignored line comment", + " /* }} ignored block comment */", + " if ready { writeJSON(w, http.StatusOK, message) }", + "}", + "func unrelated() { panic() }", + ].join("\n"); + const body = extractFunctionBody(source, "Inspect"); + assert.match(body, /StatusOK/); + assert.doesNotMatch(body, /unrelated/); +}); + +test("fails closed on an unterminated handler", () => { + assert.equal( + extractFunctionBody("func Inspect() { if ready {", "Inspect"), + "", + ); +}); diff --git a/docs/sidebars.ts b/docs/sidebars.ts index b5fae0f8..69cd21fc 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -51,8 +51,28 @@ const sidebars: SidebarsConfig = { "showcase/ai/generative-ai/synthetic-dataset", "showcase/ai/generative-ai/audio-transcription", "showcase/ai/generative-ai/document-extraction", + "showcase/ai/generative-ai/ai-video-studio", + "showcase/ai/generative-ai/local-rag-factory", ], }, + { + type: "category", + label: "Document AI", + collapsed: true, + items: ["showcase/document-ai/document-intelligence"], + }, + { + type: "category", + label: "Media intelligence", + collapsed: true, + items: ["showcase/media/video-intelligence"], + }, + { + type: "category", + label: "Rendering", + collapsed: true, + items: ["showcase/rendering/blender-render-farm"], + }, { type: "category", label: "Agentic workflows", diff --git a/docs/src/components/WorkflowCatalog.tsx b/docs/src/components/WorkflowCatalog.tsx index 82f2edcf..9e2bb56b 100644 --- a/docs/src/components/WorkflowCatalog.tsx +++ b/docs/src/components/WorkflowCatalog.tsx @@ -33,6 +33,29 @@ const sections: Section[] = [ {title: "Synthetic dataset generation", href: "/docs/showcase/ai/generative-ai/synthetic-dataset", stages: ["define schema", ["batch A", "batch B", "batch N"], "filter", "deduplicate", "quality review", "publish"]}, {title: "Audio transcription and summarization", href: "/docs/showcase/ai/generative-ai/audio-transcription", stages: ["ingest audio", "segment", ["segment 1", "segment N"], "merge transcript", "summarize", "publish"]}, {title: "OCR and document extraction", href: "/docs/showcase/ai/generative-ai/document-extraction", stages: ["ingest documents", ["OCR", "native text"], "classify", "extract fields", "validate schema", "publish"]}, + {title: "AI Video Studio", href: "/docs/showcase/ai/generative-ai/ai-video-studio", stages: ["storyboard", ["scenes", "narration"], "compose", "quality control"]}, + {title: "Local RAG Factory", href: "/docs/showcase/ai/generative-ai/local-rag-factory", stages: ["ingest", "chunk", "index", "retrieve", "answer", "evaluate"]}, + ], + }, + { + title: "Document AI", + description: "Executable extraction and reconciliation workflows over auditable document corpora.", + examples: [ + {title: "Document Intelligence", href: "/docs/showcase/document-ai/document-intelligence", stages: ["ingest", "classify", "extract", "reconcile", "report"]}, + ], + }, + { + title: "Media intelligence", + description: "Offline audiovisual inspection, segmentation, measurement, and timeline workflows.", + examples: [ + {title: "Video Intelligence", href: "/docs/showcase/media/video-intelligence", stages: ["prepare", ["probe", "scenes", "audio"], "timeline", "report"]}, + ], + }, + { + title: "Rendering", + description: "Procedural scene construction and parallel frame rendering workflows.", + examples: [ + {title: "Blender Render Farm", href: "/docs/showcase/rendering/blender-render-farm", stages: ["setup", ["even frames", "odd frames"], "compose", "verify"]}, ], }, { diff --git a/docs/workspace-archive-relay.md b/docs/workspace-archive-relay.md new file mode 100644 index 00000000..c19c9887 --- /dev/null +++ b/docs/workspace-archive-relay.md @@ -0,0 +1,48 @@ +# Workspace archive relay + +Independent dependencies with an SSH destination are prepared concurrently, +with eight workers by default and a maximum of eight. Set +`AKOFLOW_WORKSPACE_TRANSFER_PARALLELISM` (1–8) on the daemon to select a smaller +limit; `WorkspaceRsync.Parallelism` controls the same bound internally. This is +a per-consumer preparation limit, not a global process or CPU-count limit. +Eight workers can increase contention on shared disks, links and small VMs; +it is a configured experiment limit, not a measured optimal value. +Proven shared NFS/Lustre/BeeGFS routes remain internal +and are not archived or counted as relay network traffic. + +Cloud-to-HPC and HPC-to-cloud dependencies always use the relay. Proven +HPC-to-HPC shared storage uses internal copy/merge. The same concrete cloud +instance uses runtime-local copy; distinct eligible cloud instances may use +authenticated private peer transfer, with an explicit relay fallback when the +required generation trust or connectivity cannot be proven. Route selection is +per dependency, including mixed-origin fan-in. + +Relay routes pack a complete gzip-compressed tar snapshot, receive it through a +daemon-local temporary file, and upload it into a unique remote preparation +directory. The daemon rejects traversal, duplicate entries, links and special +files before uploading. The receiver verifies the archive SHA-256 before +extraction. No archive is extracted over the consumer workspace. + +Every preparation must complete before aggregate path/content conflict checking, +merge and final comparison. Workers are joined on cancellation before their +temporary directories are cleaned. Producer workspaces are never modified. +Recovery repeats isolated preparation and reuses identical published consumer +files through the existing merge contract. Eligible private cloud peers may +reuse a sealed source snapshot only with its source manifest and generation +revalidated. Transfer preparation itself does not launch activity commands. + +Transfer route metadata exposes `packaging: tar-gzip`, +`preparationStartedAt`, `preparationFinishedAt` and `preparationSeconds`. +`durationSeconds` includes the final aggregate barrier and is not additive across +parallel dependencies. Network bytes count archive payload received by the relay +plus rsync bytes uploaded; they do not represent unique logical output bytes. + +This is not streaming forwarding: receipt of an individual archive precedes its +upload, while different dependencies overlap. Full snapshots can transmit more +bytes than incremental rsync when much of the destination already exists. No +live-network speedup is claimed by local concurrency/integrity tests. + +Deployment requires a safe execution boundary. Changing this checkout does not +change an already running daemon; do not restart it during valid active jobs. + +Tracked by issue #101. diff --git a/experiments/montage-micro-hpc-e2/clean-real-20261011-v1-comparison.json b/experiments/montage-micro-hpc-e2/clean-real-20261011-v1-comparison.json new file mode 100644 index 00000000..7381b3a9 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/clean-real-20261011-v1-comparison.json @@ -0,0 +1,34 @@ +{ + "schemaVersion": 1, + "runId": "montage-clean-real-20261011-v1", + "frozenForecastUnchanged": true, + "activities": 58, + "attemptsPerActivity": 1, + "currentFailures": 0, + "barrierViolations": 0, + "microLaneOverlaps": 0, + "planningMakespanSeconds": 1193.7378314588173, + "simulationMakespanSeconds": 1203.8392300266385, + "realReportedMakespanSeconds": 3150.6341450214386, + "realRunTimestampWallSeconds": 3207, + "makespanRelativeErrorAgainstSimulation": 1.6171552367102389, + "simulationComputeWorkSeconds": 5039.550763845444, + "realComputeWorkSeconds": 4960.082308292389, + "computeRelativeErrorAgainstSimulation": -0.015768956257604216, + "simulationDependencyNetworkBytes": 4843325695, + "realDependencyNetworkBytes": 4399426793, + "networkRelativeErrorAgainstSimulation": -0.09165167282849851, + "realLegacyAggregateBytes": 22623890261, + "networkComparisonBasis": "Physical dependency-route payload; each relay leg counted once. The legacy real aggregate includes filesystem copy work and is not comparable to native network traffic.", + "realCloudLifecycleEstimateUSDAtCompletionSnapshot": 0.017691911417972675, + "priceSnapshotAsOf": "2026-10-11T15:08:07.580761292Z", + "priceLimitations": ["Network egress, public IPv4, image storage, discounts and tax excluded; HPC prices unavailable.", "Post-run VM starts for verified product export and subsequent destruction are separate recovery/cleanup operations, not forecast performance."], + "product": {"name": "mosaic-color.png", "width": 1800, "height": 1800, "bytes": 4920451, "sha256": "63fa4d8447274bdb8a44f6be8cf5a23c0a2f0baa479bd2ae1ec3b541df4e19f2"}, + "allFourFinalPNGsExportedAndDigestVerified": true, + "cloudInstancesDestroyed": true, + "activeCloudOperations": 0, + "slurmSqueueEmpty": true, + "fidelityAccepted": false, + "nextActions": ["Apply bounded terminal inspection and measured SSH/preflight/merge timing at this safe point.", "Measure queue-free materialization service and fixed command overhead separately from network bytes and merge work.", "Freeze the next forecast before its independent real validation; retain the full resource scope and relay policy."], + "limitations": ["Transfer and queue work totals overlap and must not be added as wall makespan.", "The e2-medium interference matrix remains extrapolated to HPC and micro resources.", "One successful run is not proof of transfer-rate stability or forecasting fidelity."] +} diff --git a/experiments/montage-micro-hpc-e2/clean-real-20261011-v1-results.json b/experiments/montage-micro-hpc-e2/clean-real-20261011-v1-results.json new file mode 100644 index 00000000..fade5354 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/clean-real-20261011-v1-results.json @@ -0,0 +1,5856 @@ +{ + "schema_version": 1, + "run_id": "montage-clean-real-20261011-v1", + "attempt_history": "single_observed_attempt_per_activity", + "audit": { + "expected_count": 58, + "missing": [ + + ], + "unexpected": [ + + ], + "not_completed": [ + + ], + "incomplete_transfer_count": 0, + "barrier_violation_count": 0, + "unreleased_lease_count": 0, + "single_lane_micro_overlap_count": 0, + "missing_dependency_route_count": 0, + "cross_cloud_hpc_non_relay_count": 0 + }, + "plan": { + "makespanSeconds": 1193.7378314588173, + "cost": 0.004471322214927297 + }, + "plan_assignment_compute_work_seconds": 5039.550763845444, + "plan_assignment_transfer_work_seconds": 1109.5268686489007, + "simulation": { + "makespanSeconds": 1203.8392300266385, + "cost": 0.004511523136751095, + "breakdown": { + "computeSeconds": 5039.550763845444, + "transferSeconds": 1162.3209191687322, + "queueSeconds": 437.95766065454455, + "interferenceSeconds": 0, + "overheadSeconds": 0 + }, + "transferredBytes": 4843325695 + }, + "real_api_reported": { + "makespanSeconds": 3150.6341450214386, + "cost": 0.011403055533727306, + "breakdown": { + "computeSeconds": 4960.082308292389, + "transferSeconds": 10788.998977661133, + "queueSeconds": 6609.299883365631, + "interferenceSeconds": 0, + "overheadSeconds": 90.98878240585327 + }, + "transferredBytes": 22623890261 + }, + "real_cloud_lifecycle_estimate": { + "costBasis": "VM compute and boot-disk lifecycle at on-demand marginal rates; not a provider invoice", + "excludedCosts": [ + "network-egress", + "public-ipv4", + "image-storage", + "discounts-and-tax" + ], + "status": "estimated", + "currency": "USD", + "estimatedCost": 0.017691911417972675, + "knownCost": 0.017691911417972675, + "asOf": "2026-10-11T15:08:07.580761292Z" + }, + "real_cloud_generations": [ + { + "instance_id": "cloud-instance-586a4f3a-3b56-4425-b9a7-c98399e7defc", + "status": "ready", + "billing": { + "pricingStatus": "estimated", + "priceCurrency": "USD", + "priceSource": "gcp-public-catalog", + "priceObservedAt": "2026-10-10T21:57:24.147279919Z", + "priceRegion": "us-central1", + "priceMachineType": "e2-micro", + "startedAt": "2026-10-11T07:20:11.807-07:00", + "computeStartedAt": "2026-10-11T07:20:11.807-07:00", + "accumulatedComputeSeconds": 0, + "computePricePerSecond": 2.3267854166666665e-06, + "diskPricePerSecond": 1.1415525114155251e-06, + "asOf": "2026-10-11T15:08:07.580761292Z", + "currentCost": 0.009974155208872626 + } + }, + { + "instance_id": "cloud-instance-7c90f192-19f3-40de-9393-9e08b4fa8bed", + "status": "stopped", + "billing": { + "pricingStatus": "estimated", + "priceCurrency": "USD", + "priceSource": "gcp-public-catalog", + "priceObservedAt": "2026-10-10T21:57:24.147279919Z", + "priceRegion": "us-central1", + "priceMachineType": "e2-micro", + "startedAt": "2026-10-11T07:30:34.104-07:00", + "accumulatedComputeSeconds": 2211.331614843, + "computePricePerSecond": 2.3267854166666665e-06, + "diskPricePerSecond": 1.1415525114155251e-06, + "asOf": "2026-10-11T15:08:07.580761292Z", + "currentCost": 0.007717756209100049 + } + } + ], + "real_api_run_timestamp_wall_seconds": 3207.0, + "real_network_bytes_from_routes": 4399426793, + "transfer_route_count": 146, + "dependency_route_count": 114, + "cross_cloud_hpc_relay_count": 20, + "relay_server_network_bytes": 3779753473, + "routes": [ + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000011", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.00933074951171875, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000049", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.03038191795349121, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000005", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.015977859497070312, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000006", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.03550577163696289, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000007", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.02649378776550293, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000008", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.0601658821105957, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000009", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.020616769790649414, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000010", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.03507423400878906, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000024", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.0119476318359375, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000025", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.001220703125, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000026", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.029796600341796875, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000027", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.03660869598388672, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000028", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.0225217342376709, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000029", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.014779806137084961, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000043", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.01770782470703125, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000044", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.03433370590209961, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000045", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.03493070602416992, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000046", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.009405374526977539, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000047", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.022412538528442383, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000048", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.029168367385864258, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mprojectid0000001", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.018802642822265625, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mprojectid0000002", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.0280611515045166, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mprojectid0000003", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.025228500366210938, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mprojectid0000004", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.008282661437988281, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mprojectid0000020", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 6.4798197746276855, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mprojectid0000021", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 6.494693279266357, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mprojectid0000022", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 6.492795705795288, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mprojectid0000023", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 6.500110626220703, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mprojectid0000039", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 6.51016640663147, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mprojectid0000040", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 6.48828649520874, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mprojectid0000041", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 6.514239072799683, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mprojectid0000042", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 6.515958309173584, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000018", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000013", + "strategy": "runtime-local", + "duration_seconds": 68.57136988639832, + "logical_bytes": 740899066, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 1.4417e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 66.79656049 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000018", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000014", + "strategy": "runtime-local", + "duration_seconds": 68.57136988639832, + "logical_bytes": 106346880, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 1.4417e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 66.79656049 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000018", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000015", + "strategy": "runtime-local", + "duration_seconds": 68.57136988639832, + "logical_bytes": 106306560, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 1.4417e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 66.79656049 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000018", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000016", + "strategy": "runtime-local", + "duration_seconds": 68.57136988639832, + "logical_bytes": 106346880, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 1.4417e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 66.79656049 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000018", + "producer_activity_id": "montage-independent-work-v1-mimgtblid0000017", + "strategy": "runtime-local", + "duration_seconds": 68.57136988639832, + "logical_bytes": 2606, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 1, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 1.4417e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 66.79656049 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000037", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000032", + "strategy": "runtime-local", + "duration_seconds": 68.89127993583679, + "logical_bytes": 732938742, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.6625e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 67.094164362 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000037", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000033", + "strategy": "runtime-local", + "duration_seconds": 68.89127993583679, + "logical_bytes": 105114240, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.6625e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 67.094164362 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000037", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000034", + "strategy": "runtime-local", + "duration_seconds": 68.89127993583679, + "logical_bytes": 105027840, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.6625e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 67.094164362 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000037", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000035", + "strategy": "runtime-local", + "duration_seconds": 68.89127993583679, + "logical_bytes": 105073920, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.6625e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 67.094164362 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000037", + "producer_activity_id": "montage-independent-work-v1-mimgtblid0000036", + "strategy": "runtime-local", + "duration_seconds": 68.89127993583679, + "logical_bytes": 2600, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 1, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.6625e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 67.094164362 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000056", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000051", + "strategy": "runtime-local", + "duration_seconds": 220.91178274154663, + "logical_bytes": 739032822, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 3.575e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 219.086229683 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000056", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000052", + "strategy": "runtime-local", + "duration_seconds": 220.91178274154663, + "logical_bytes": 106058880, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 3.575e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 219.086229683 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000056", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000053", + "strategy": "runtime-local", + "duration_seconds": 220.91178274154663, + "logical_bytes": 106018560, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 3.575e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 219.086229683 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000056", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000054", + "strategy": "runtime-local", + "duration_seconds": 220.91178274154663, + "logical_bytes": 106018560, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 3.575e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 219.086229683 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000056", + "producer_activity_id": "montage-independent-work-v1-mimgtblid0000055", + "strategy": "runtime-local", + "duration_seconds": 220.91178274154663, + "logical_bytes": 2594, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 1, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 3.575e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 219.086229683 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000013", + "producer_activity_id": "montage-independent-work-v1-mbgmodelid0000012", + "strategy": "runtime-local", + "duration_seconds": 118.78154349327087, + "logical_bytes": 634511866, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 98.948641753 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.957826634 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000013", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000001", + "strategy": "gateway", + "duration_seconds": 117.90843868255615, + "logical_bytes": null, + "network_bytes": 105617907, + "relay_server_network_bytes": 105617907, + "archive_bytes": 52802443, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 5.3667e-05, + "rsync_execution_seconds": 4.989966169, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 91.370362125 + }, + { + "name": "archive-validation", + "durationSeconds": 0.558243667 + }, + { + "name": "archive-upload", + "durationSeconds": 4.990300461 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.163310667 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 98.948641753 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.957826634 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000014", + "producer_activity_id": "montage-independent-work-v1-mbgmodelid0000012", + "strategy": "runtime-local", + "duration_seconds": 110.96577215194702, + "logical_bytes": 634511866, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 90.862677833 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.211367968 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000014", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000002", + "strategy": "gateway", + "duration_seconds": 110.07551312446594, + "logical_bytes": null, + "network_bytes": 107170830, + "relay_server_network_bytes": 107170830, + "archive_bytes": 53578808, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 3.1708e-05, + "rsync_execution_seconds": 7.867882253, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 80.24741412 + }, + { + "name": "archive-validation", + "durationSeconds": 0.575396417 + }, + { + "name": "archive-upload", + "durationSeconds": 7.868127711 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.287437334 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 90.862677833 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.211367968 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000015", + "producer_activity_id": "montage-independent-work-v1-mbgmodelid0000012", + "strategy": "runtime-local", + "duration_seconds": 84.1312563419342, + "logical_bytes": 634511866, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 65.281515779 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.967336258 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000015", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000003", + "strategy": "gateway", + "duration_seconds": 83.25016713142395, + "logical_bytes": null, + "network_bytes": 109907434, + "relay_server_network_bytes": 109907434, + "archive_bytes": 54946946, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 5.542e-06, + "rsync_execution_seconds": 5.153627503, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 57.551059192 + }, + { + "name": "archive-validation", + "durationSeconds": 0.532593375 + }, + { + "name": "archive-upload", + "durationSeconds": 5.15370492 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.158808168 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 65.281515779 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.967336258 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000016", + "producer_activity_id": "montage-independent-work-v1-mbgmodelid0000012", + "strategy": "runtime-local", + "duration_seconds": 100.25711560249329, + "logical_bytes": 634511866, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 80.69182120400001 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.681738967 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000016", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000004", + "strategy": "gateway", + "duration_seconds": 99.37529993057251, + "logical_bytes": null, + "network_bytes": 109611286, + "relay_server_network_bytes": 109611286, + "archive_bytes": 54798888, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 9.775e-05, + "rsync_execution_seconds": 5.190558044, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 72.834803492 + }, + { + "name": "archive-validation", + "durationSeconds": 0.562451084 + }, + { + "name": "archive-upload", + "durationSeconds": 5.191014877 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.212048168 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 80.69182120400001 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.681738967 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000032", + "producer_activity_id": "montage-independent-work-v1-mbgmodelid0000031", + "strategy": "runtime-local", + "duration_seconds": 121.99184679985046, + "logical_bytes": 627864822, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 103.125955463 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.98732655 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000032", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000020", + "strategy": "gateway", + "duration_seconds": 121.11549353599548, + "logical_bytes": null, + "network_bytes": 102529126, + "relay_server_network_bytes": 102529126, + "archive_bytes": 51258241, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 8.1042e-05, + "rsync_execution_seconds": 4.934585543, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 95.584607961 + }, + { + "name": "archive-validation", + "durationSeconds": 0.552470208 + }, + { + "name": "archive-upload", + "durationSeconds": 4.935003877 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.159437876 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 103.125955463 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.98732655 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000033", + "producer_activity_id": "montage-independent-work-v1-mbgmodelid0000031", + "strategy": "runtime-local", + "duration_seconds": 70.03583884239197, + "logical_bytes": 627864822, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 50.352573273 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.809427634 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000033", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000021", + "strategy": "gateway", + "duration_seconds": 69.16299414634705, + "logical_bytes": null, + "network_bytes": 104335080, + "relay_server_network_bytes": 104335080, + "archive_bytes": 52161109, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 3.1292e-05, + "rsync_execution_seconds": 5.046314544, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 42.699166019 + }, + { + "name": "archive-validation", + "durationSeconds": 0.53448275 + }, + { + "name": "archive-upload", + "durationSeconds": 5.046675044 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.188282209 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 50.352573273 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.809427634 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000034", + "producer_activity_id": "montage-independent-work-v1-mbgmodelid0000031", + "strategy": "runtime-local", + "duration_seconds": 50.394211292266846, + "logical_bytes": 627864822, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 31.64435614 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.8304088 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000034", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000022", + "strategy": "gateway", + "duration_seconds": 49.47566747665405, + "logical_bytes": null, + "network_bytes": 106653299, + "relay_server_network_bytes": 106653299, + "archive_bytes": 53320075, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 7.875e-05, + "rsync_execution_seconds": 5.028231253, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 23.984320886 + }, + { + "name": "archive-validation", + "durationSeconds": 0.514497417 + }, + { + "name": "archive-upload", + "durationSeconds": 5.028529627 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.189124417 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 31.64435614 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.8304088 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000035", + "producer_activity_id": "montage-independent-work-v1-mbgmodelid0000031", + "strategy": "runtime-local", + "duration_seconds": 112.71284008026123, + "logical_bytes": 627864822, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 93.800998334 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.021403008 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000035", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000023", + "strategy": "gateway", + "duration_seconds": 111.82394909858704, + "logical_bytes": null, + "network_bytes": 106193023, + "relay_server_network_bytes": 106193023, + "archive_bytes": 53089965, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 3.4792e-05, + "rsync_execution_seconds": 5.071248169, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 86.115549372 + }, + { + "name": "archive-validation", + "durationSeconds": 0.527643167 + }, + { + "name": "archive-upload", + "durationSeconds": 5.071584585 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.195763792 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 93.800998334 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.021403008 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000051", + "producer_activity_id": "montage-independent-work-v1-mbgmodelid0000050", + "strategy": "runtime-local", + "duration_seconds": 104.91257047653198, + "logical_bytes": 633014262, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 84.734250664 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.283210883 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000051", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000039", + "strategy": "gateway", + "duration_seconds": 104.01954793930054, + "logical_bytes": null, + "network_bytes": 103997683, + "relay_server_network_bytes": 103997683, + "archive_bytes": 51992431, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 7.3917e-05, + "rsync_execution_seconds": 5.01576417, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 76.648116785 + }, + { + "name": "archive-validation", + "durationSeconds": 0.519891126 + }, + { + "name": "archive-upload", + "durationSeconds": 5.016239211 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.647790834 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 84.734250664 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.283210883 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000052", + "producer_activity_id": "montage-independent-work-v1-mbgmodelid0000050", + "strategy": "runtime-local", + "duration_seconds": 58.700331687927246, + "logical_bytes": 633014262, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 38.781405101 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.031397841 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000052", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000040", + "strategy": "gateway", + "duration_seconds": 57.813836336135864, + "logical_bytes": null, + "network_bytes": 106265678, + "relay_server_network_bytes": 106265678, + "archive_bytes": 53126289, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 1.0542e-05, + "rsync_execution_seconds": 5.078452795, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 30.65702593 + }, + { + "name": "archive-validation", + "durationSeconds": 0.562282084 + }, + { + "name": "archive-upload", + "durationSeconds": 5.078544211 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.616229251 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 38.781405101 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.031397841 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000053", + "producer_activity_id": "montage-independent-work-v1-mbgmodelid0000050", + "strategy": "runtime-local", + "duration_seconds": 58.162331342697144, + "logical_bytes": 633014262, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 37.534052683 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.754857759 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000053", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000041", + "strategy": "gateway", + "duration_seconds": 57.28966474533081, + "logical_bytes": null, + "network_bytes": 108718484, + "relay_server_network_bytes": 108718484, + "archive_bytes": 54352543, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 5.8583e-05, + "rsync_execution_seconds": 5.02759121, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 29.348639596 + }, + { + "name": "archive-validation", + "durationSeconds": 0.519940709 + }, + { + "name": "archive-upload", + "durationSeconds": 5.027941585 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.644368501 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 37.534052683 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.754857759 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000054", + "producer_activity_id": "montage-independent-work-v1-mbgmodelid0000050", + "strategy": "runtime-local", + "duration_seconds": 138.9047348499298, + "logical_bytes": 633014262, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 118.818895803 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.209875383 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000054", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000042", + "strategy": "gateway", + "duration_seconds": 138.03144693374634, + "logical_bytes": null, + "network_bytes": 108668676, + "relay_server_network_bytes": 108668676, + "archive_bytes": 54327643, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 9.208e-06, + "rsync_execution_seconds": 5.267272378, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 110.353939257 + }, + { + "name": "archive-validation", + "durationSeconds": 0.530202125 + }, + { + "name": "archive-upload", + "durationSeconds": 5.267380294 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.702265709 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 118.818895803 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.209875383 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbgmodelid0000012", + "producer_activity_id": "montage-independent-work-v1-mconcatfitid0000011", + "strategy": "gateway", + "duration_seconds": 314.0487687587738, + "logical_bytes": 543960288, + "network_bytes": 559032552, + "relay_server_network_bytes": 559032552, + "archive_bytes": 279482093, + "files": 27, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 1.7709e-05, + "rsync_execution_seconds": 23.754565511, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 264.406634161 + }, + { + "name": "archive-validation", + "durationSeconds": 2.844069335 + }, + { + "name": "archive-upload", + "durationSeconds": 23.754725053 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 5.555778961 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 299.776869345 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 14.266392464 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbgmodelid0000031", + "producer_activity_id": "montage-independent-work-v1-mconcatfitid0000030", + "strategy": "runtime-local", + "duration_seconds": 15.83263635635376, + "logical_bytes": 627864546, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 72, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.1292e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 15.232841298 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbgmodelid0000050", + "producer_activity_id": "montage-independent-work-v1-mconcatfitid0000049", + "strategy": "gateway", + "duration_seconds": 332.3163990974426, + "logical_bytes": 542462684, + "network_bytes": 555568726, + "relay_server_network_bytes": 555568726, + "archive_bytes": 277750392, + "files": 27, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 9.0e-06, + "rsync_execution_seconds": 22.50596676, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 281.700515004 + }, + { + "name": "archive-validation", + "durationSeconds": 2.785885585 + }, + { + "name": "archive-upload", + "durationSeconds": 22.506065968 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 7.398465045 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 317.233699561 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 15.077407549 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000011", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000005", + "strategy": "direct-runtime", + "duration_seconds": 23.301818370819092, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 7, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.6709e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.266011092 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000011", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000006", + "strategy": "direct-runtime", + "duration_seconds": 23.301818370819092, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.6709e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.266011092 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000011", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000007", + "strategy": "direct-runtime", + "duration_seconds": 23.301818370819092, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.6709e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.266011092 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000011", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000008", + "strategy": "direct-runtime", + "duration_seconds": 23.301818370819092, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.6709e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.266011092 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000011", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000009", + "strategy": "direct-runtime", + "duration_seconds": 23.301818370819092, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.6709e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.266011092 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000011", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000010", + "strategy": "direct-runtime", + "duration_seconds": 23.301818370819092, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.6709e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.266011092 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000030", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000024", + "strategy": "gateway", + "duration_seconds": 483.66852021217346, + "logical_bytes": 241246371, + "network_bytes": 240230880, + "relay_server_network_bytes": 240230880, + "archive_bytes": 120100713, + "files": 7, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 2.275e-05, + "rsync_execution_seconds": 14.941312506, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 153.336095904 + }, + { + "name": "archive-validation", + "durationSeconds": 1.213918084 + }, + { + "name": "archive-upload", + "durationSeconds": 14.941538382 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 3.022311001 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 459.508177377 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 24.153126054 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000030", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000025", + "strategy": "gateway", + "duration_seconds": 483.6683597564697, + "logical_bytes": 136166688, + "network_bytes": 242510991, + "relay_server_network_bytes": 242510991, + "archive_bytes": 121240633, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 4.2542e-05, + "rsync_execution_seconds": 10.134279213, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 443.228494495 + }, + { + "name": "archive-validation", + "durationSeconds": 1.297942459 + }, + { + "name": "archive-upload", + "durationSeconds": 10.134520046 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 3.964075877 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 459.508177377 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 24.153126054 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000030", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000026", + "strategy": "gateway", + "duration_seconds": 483.6684458255768, + "logical_bytes": 114215320, + "network_bytes": 218669206, + "relay_server_network_bytes": 218669206, + "archive_bytes": 109321192, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 7.667e-06, + "rsync_execution_seconds": 15.45195109, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 148.769738568 + }, + { + "name": "archive-validation", + "durationSeconds": 1.148273584 + }, + { + "name": "archive-upload", + "durationSeconds": 15.452046298 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 3.90661646 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 459.508177377 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 24.153126054 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000030", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000027", + "strategy": "gateway", + "duration_seconds": 483.6684205532074, + "logical_bytes": 9175961, + "network_bytes": 220934982, + "relay_server_network_bytes": 220934982, + "archive_bytes": 110453944, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 3.1375e-05, + "rsync_execution_seconds": 10.058888922, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 426.40998032 + }, + { + "name": "archive-validation", + "durationSeconds": 1.175054917 + }, + { + "name": "archive-upload", + "durationSeconds": 10.059085672 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 2.193105334 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 459.508177377 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 24.153126054 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000030", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000028", + "strategy": "gateway", + "duration_seconds": 483.6683704853058, + "logical_bytes": 18271004, + "network_bytes": 230333020, + "relay_server_network_bytes": 230333020, + "archive_bytes": 115152387, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 1.8125e-05, + "rsync_execution_seconds": 9.623536547, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 443.221197161 + }, + { + "name": "archive-validation", + "durationSeconds": 1.207280626 + }, + { + "name": "archive-upload", + "durationSeconds": 9.623734004 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 3.589447127 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 459.508177377 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 24.153126054 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000030", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000029", + "strategy": "gateway", + "duration_seconds": 483.6688690185547, + "logical_bytes": 18236443, + "network_bytes": 232804610, + "relay_server_network_bytes": 232804610, + "archive_bytes": 116388035, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 1.6792e-05, + "rsync_execution_seconds": 15.902585841, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 149.022678902 + }, + { + "name": "archive-validation", + "durationSeconds": 1.195927751 + }, + { + "name": "archive-upload", + "durationSeconds": 15.90274809 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 4.336680127 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 459.508177377 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 24.153126054 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000049", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000043", + "strategy": "direct-runtime", + "duration_seconds": 50.45422673225403, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 7, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 4.7e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 20.51674826 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000049", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000044", + "strategy": "direct-runtime", + "duration_seconds": 50.45422673225403, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 4.7e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 20.51674826 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000049", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000045", + "strategy": "direct-runtime", + "duration_seconds": 50.45422673225403, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 4.7e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 20.51674826 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000049", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000046", + "strategy": "direct-runtime", + "duration_seconds": 50.45422673225403, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 4.7e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 20.51674826 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000049", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000047", + "strategy": "direct-runtime", + "duration_seconds": 50.45422673225403, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 4.7e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 20.51674826 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000049", + "producer_activity_id": "montage-independent-work-v1-mdifffitid0000048", + "strategy": "direct-runtime", + "duration_seconds": 50.45422673225403, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 4.7e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 20.51674826 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000005", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000001", + "strategy": "direct-runtime", + "duration_seconds": 15.11195158958435, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.8625e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 10.438356588 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000005", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000002", + "strategy": "direct-runtime", + "duration_seconds": 15.11195158958435, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.8625e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 10.438356588 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000006", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000001", + "strategy": "direct-runtime", + "duration_seconds": 23.94624924659729, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 1.8167e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.231984342 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000006", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000003", + "strategy": "direct-runtime", + "duration_seconds": 23.94624924659729, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 1.8167e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.231984342 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000007", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000001", + "strategy": "direct-runtime", + "duration_seconds": 15.099162101745605, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.375e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 10.228315838 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000007", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000004", + "strategy": "direct-runtime", + "duration_seconds": 15.099162101745605, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.375e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 10.228315838 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000008", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000002", + "strategy": "direct-runtime", + "duration_seconds": 23.93864607810974, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 1.9375e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.443692842 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000008", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000003", + "strategy": "direct-runtime", + "duration_seconds": 23.93864607810974, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 1.9375e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.443692842 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000009", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000002", + "strategy": "direct-runtime", + "duration_seconds": 15.117977619171143, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 0.00099725 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 10.863442880000001 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000009", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000004", + "strategy": "direct-runtime", + "duration_seconds": 15.117977619171143, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 0.00099725 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 10.863442880000001 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000010", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000003", + "strategy": "direct-runtime", + "duration_seconds": 23.949808835983276, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 6.5291e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.866558468 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000010", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000004", + "strategy": "direct-runtime", + "duration_seconds": 23.949808835983276, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 6.5291e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.866558468 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000024", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000020", + "strategy": "direct-runtime", + "duration_seconds": 28.57111096382141, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 0.0001265 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.392542217 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000024", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000021", + "strategy": "direct-runtime", + "duration_seconds": 28.57111096382141, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 0.0001265 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.392542217 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000025", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000020", + "strategy": "direct-runtime", + "duration_seconds": 9.966228485107422, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.3167e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 5.776185626 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000025", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000022", + "strategy": "direct-runtime", + "duration_seconds": 9.966228485107422, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.3167e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 5.776185626 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000026", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000020", + "strategy": "direct-runtime", + "duration_seconds": 17.757861614227295, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 4.1291e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 9.244146546 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000026", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000023", + "strategy": "direct-runtime", + "duration_seconds": 17.757861614227295, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 4.1291e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 9.244146546 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000027", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000021", + "strategy": "direct-runtime", + "duration_seconds": 30.311562538146973, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 0.000156084 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 21.22848251 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000027", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000022", + "strategy": "direct-runtime", + "duration_seconds": 30.311562538146973, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 0.000156084 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 21.22848251 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000028", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000021", + "strategy": "direct-runtime", + "duration_seconds": 17.536539554595947, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 0.000476375 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.564116379 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000028", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000023", + "strategy": "direct-runtime", + "duration_seconds": 17.536539554595947, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 0.000476375 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.564116379 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000029", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000022", + "strategy": "direct-runtime", + "duration_seconds": 17.77143907546997, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.9417e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 11.319485464 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000029", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000023", + "strategy": "direct-runtime", + "duration_seconds": 17.77143907546997, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.9417e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 11.319485464 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000043", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000039", + "strategy": "direct-runtime", + "duration_seconds": 14.651428461074829, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.75e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 9.563589005 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000043", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000040", + "strategy": "direct-runtime", + "duration_seconds": 14.651428461074829, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 2.75e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 9.563589005 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000044", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000039", + "strategy": "direct-runtime", + "duration_seconds": 15.919742107391357, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 5.6584e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 9.36306013 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000044", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000041", + "strategy": "direct-runtime", + "duration_seconds": 15.919742107391357, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 5.6584e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 9.36306013 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000045", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000039", + "strategy": "direct-runtime", + "duration_seconds": 23.115092515945435, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 3.7125e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 15.923956549 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000045", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000042", + "strategy": "direct-runtime", + "duration_seconds": 23.115092515945435, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 3.7125e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 15.923956549 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000046", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000040", + "strategy": "direct-runtime", + "duration_seconds": 8.380683660507202, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 4.6709e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 5.614199461 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000046", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000041", + "strategy": "direct-runtime", + "duration_seconds": 8.380683660507202, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 4.6709e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 5.614199461 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000047", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000040", + "strategy": "direct-runtime", + "duration_seconds": 24.906448364257812, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 5.175e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.020641134 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000047", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000042", + "strategy": "direct-runtime", + "duration_seconds": 24.906448364257812, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 5.175e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.020641134 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000048", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000041", + "strategy": "direct-runtime", + "duration_seconds": 15.064369678497314, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 1.9291e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.716522129 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000048", + "producer_activity_id": "montage-independent-work-v1-mprojectid0000042", + "strategy": "direct-runtime", + "duration_seconds": 15.064369678497314, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 1.9291e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.716522129 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mimgtblid0000017", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000013", + "strategy": "runtime-local", + "duration_seconds": 53.429725646972656, + "logical_bytes": 740899066, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 7.75e-06 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 51.926005982 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mimgtblid0000017", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000014", + "strategy": "runtime-local", + "duration_seconds": 53.429725646972656, + "logical_bytes": 106346880, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 7.75e-06 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 51.926005982 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mimgtblid0000017", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000015", + "strategy": "runtime-local", + "duration_seconds": 53.429725646972656, + "logical_bytes": 106306560, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 7.75e-06 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 51.926005982 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mimgtblid0000017", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000016", + "strategy": "runtime-local", + "duration_seconds": 53.429725646972656, + "logical_bytes": 106346880, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 7.75e-06 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 51.926005982 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mimgtblid0000036", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000032", + "strategy": "runtime-local", + "duration_seconds": 54.787358045578, + "logical_bytes": 732938742, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 7.1667e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 53.298720149 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mimgtblid0000036", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000033", + "strategy": "runtime-local", + "duration_seconds": 54.787358045578, + "logical_bytes": 105114240, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 7.1667e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 53.298720149 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mimgtblid0000036", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000034", + "strategy": "runtime-local", + "duration_seconds": 54.787358045578, + "logical_bytes": 105027840, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 7.1667e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 53.298720149 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mimgtblid0000036", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000035", + "strategy": "runtime-local", + "duration_seconds": 54.787358045578, + "logical_bytes": 105073920, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 7.1667e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 53.298720149 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mimgtblid0000055", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000051", + "strategy": "runtime-local", + "duration_seconds": 54.88259744644165, + "logical_bytes": 739032822, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 1.2375e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 53.377830025 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mimgtblid0000055", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000052", + "strategy": "runtime-local", + "duration_seconds": 54.88259744644165, + "logical_bytes": 106058880, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 1.2375e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 53.377830025 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mimgtblid0000055", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000053", + "strategy": "runtime-local", + "duration_seconds": 54.88259744644165, + "logical_bytes": 106018560, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 1.2375e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 53.377830025 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mimgtblid0000055", + "producer_activity_id": "montage-independent-work-v1-mbackgroundid0000054", + "strategy": "runtime-local", + "duration_seconds": 54.88259744644165, + "logical_bytes": 106018560, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 1.2375e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 53.377830025 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mviewerid0000019", + "producer_activity_id": "montage-independent-work-v1-maddid0000018", + "strategy": "runtime-local", + "duration_seconds": 31.595834493637085, + "logical_bytes": 1111747752, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 84, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 4.25e-06 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 31.011059181 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mviewerid0000038", + "producer_activity_id": "montage-independent-work-v1-maddid0000037", + "strategy": "runtime-local", + "duration_seconds": 30.052446603775024, + "logical_bytes": 1100003102, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 84, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 3.3709e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 29.459673139 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mviewerid0000057", + "producer_activity_id": "montage-independent-work-v1-maddid0000056", + "strategy": "runtime-local", + "duration_seconds": 105.5463137626648, + "logical_bytes": 1108977176, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 84, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 3.4041e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 104.951331755 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mviewerid0000058", + "producer_activity_id": "montage-independent-work-v1-maddid0000018", + "strategy": "runtime-local", + "duration_seconds": 395.95423579216003, + "logical_bytes": 1111747752, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 84, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 303.802673804 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 90.971308457 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mviewerid0000058", + "producer_activity_id": "montage-independent-work-v1-maddid0000037", + "strategy": "runtime-local", + "duration_seconds": 395.95423579216003, + "logical_bytes": 1009451800, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 39, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 303.802673804 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 90.971308457 + } + ] + }, + { + "consumer_activity_id": "montage-independent-work-v1-mviewerid0000058", + "producer_activity_id": "montage-independent-work-v1-maddid0000056", + "strategy": "direct-runtime", + "duration_seconds": 394.7801208496094, + "logical_bytes": 1018425874, + "network_bytes": 619673320, + "relay_server_network_bytes": 0, + "archive_bytes": 619673320, + "files": 39, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "source_pack_preflight_seconds": null, + "source_pack_preflight_ssh_channel_wait_seconds": null, + "source_pack_preflight_service_seconds": null, + "source_pack_ssh_channel_wait_seconds": null, + "source_pack_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.210240794 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 250.851621405 + }, + { + "name": "peer-private-upload", + "durationSeconds": 28.724556513 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 19.263423801 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 303.802673804 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 90.971308457 + } + ] + } + ], + "shared_preparations": [ + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000018", + "barrier_seconds": 1.4417e-05, + "merge_seconds": 66.79656049, + "merge_check_seconds": 25.7431940460001, + "merge_copy_seconds": 33.813945880999654, + "merge_verify_seconds": 6.73655051500009, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000037", + "barrier_seconds": 2.6625e-05, + "merge_seconds": 67.094164362, + "merge_check_seconds": 25.456530666999925, + "merge_copy_seconds": 34.58182319599996, + "merge_verify_seconds": 6.551817965000055, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-maddid0000056", + "barrier_seconds": 3.575e-05, + "merge_seconds": 219.086229683, + "merge_check_seconds": 121.27468780899994, + "merge_copy_seconds": 62.06325912700004, + "merge_verify_seconds": 35.192216692999864, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000013", + "barrier_seconds": 98.948641753, + "merge_seconds": 18.957826634, + "merge_check_seconds": 4.609278946999893, + "merge_copy_seconds": 10.169980699999996, + "merge_verify_seconds": 3.6801974079999127, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000014", + "barrier_seconds": 90.862677833, + "merge_seconds": 19.211367968, + "merge_check_seconds": 4.605828757999916, + "merge_copy_seconds": 10.434541308000007, + "merge_verify_seconds": 3.645531559000119, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000015", + "barrier_seconds": 65.281515779, + "merge_seconds": 17.967336258, + "merge_check_seconds": 3.6671020270000554, + "merge_copy_seconds": 10.02192975999992, + "merge_verify_seconds": 3.706648684999891, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000016", + "barrier_seconds": 80.69182120400001, + "merge_seconds": 18.681738967, + "merge_check_seconds": 4.383543464000013, + "merge_copy_seconds": 10.249773944000026, + "merge_verify_seconds": 3.543404862999978, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000032", + "barrier_seconds": 103.125955463, + "merge_seconds": 17.98732655, + "merge_check_seconds": 4.44896342200002, + "merge_copy_seconds": 9.856643633999965, + "merge_verify_seconds": 3.1742867120000255, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000033", + "barrier_seconds": 50.352573273, + "merge_seconds": 18.809427634, + "merge_check_seconds": 4.494645716000036, + "merge_copy_seconds": 9.804468757999985, + "merge_verify_seconds": 4.000663433, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000034", + "barrier_seconds": 31.64435614, + "merge_seconds": 17.8304088, + "merge_check_seconds": 4.580872204000002, + "merge_copy_seconds": 9.761975659999962, + "merge_verify_seconds": 2.9794699060000767, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000035", + "barrier_seconds": 93.800998334, + "merge_seconds": 18.021403008, + "merge_check_seconds": 4.534394609999936, + "merge_copy_seconds": 9.961295709999945, + "merge_verify_seconds": 2.982342827000025, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000051", + "barrier_seconds": 84.734250664, + "merge_seconds": 19.283210883, + "merge_check_seconds": 4.608209593999959, + "merge_copy_seconds": 10.383413054000016, + "merge_verify_seconds": 3.7832397060000176, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000052", + "barrier_seconds": 38.781405101, + "merge_seconds": 19.031397841, + "merge_check_seconds": 4.397083030999966, + "merge_copy_seconds": 10.247258582000029, + "merge_verify_seconds": 3.784764866000046, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000053", + "barrier_seconds": 37.534052683, + "merge_seconds": 19.754857759, + "merge_check_seconds": 4.961447119000013, + "merge_copy_seconds": 10.419525247000024, + "merge_verify_seconds": 3.7381198379999887, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbackgroundid0000054", + "barrier_seconds": 118.818895803, + "merge_seconds": 19.209875383, + "merge_check_seconds": 4.67022387399993, + "merge_copy_seconds": 10.126042358999939, + "merge_verify_seconds": 3.881001246000096, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbgmodelid0000012", + "barrier_seconds": 299.776869345, + "merge_seconds": 14.266392464, + "merge_check_seconds": 2.5070188539998526, + "merge_copy_seconds": 8.130230359999814, + "merge_verify_seconds": 3.0046015890000035, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbgmodelid0000031", + "barrier_seconds": 2.1292e-05, + "merge_seconds": 15.232841298, + "merge_check_seconds": 2.53756751800006, + "merge_copy_seconds": 9.150769504999971, + "merge_verify_seconds": 3.020684265, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mbgmodelid0000050", + "barrier_seconds": 317.233699561, + "merge_seconds": 15.077407549, + "merge_check_seconds": 2.962898128000006, + "merge_copy_seconds": 8.074467681999977, + "merge_verify_seconds": 3.463600305, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000011", + "barrier_seconds": 2.6709e-05, + "merge_seconds": 17.266011092, + "merge_check_seconds": 5.660585141740739, + "merge_copy_seconds": 5.871328831650317, + "merge_verify_seconds": 4.222140168771148, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000030", + "barrier_seconds": 459.508177377, + "merge_seconds": 24.153126054, + "merge_check_seconds": 8.702594806999969, + "merge_copy_seconds": 11.903956375999996, + "merge_verify_seconds": 3.0493237859999454, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mconcatfitid0000049", + "barrier_seconds": 4.7e-05, + "merge_seconds": 20.51674826, + "merge_check_seconds": 6.752671428956091, + "merge_copy_seconds": 6.220852295868099, + "merge_verify_seconds": 5.104944341816008, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000005", + "barrier_seconds": 2.8625e-05, + "merge_seconds": 10.438356588, + "merge_check_seconds": 1.6053264392539859, + "merge_copy_seconds": 1.3583475621417165, + "merge_verify_seconds": 5.045419662259519, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000006", + "barrier_seconds": 1.8167e-05, + "merge_seconds": 18.231984342, + "merge_check_seconds": 0.9334692480042577, + "merge_copy_seconds": 1.471917700022459, + "merge_verify_seconds": 5.521449169144034, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000007", + "barrier_seconds": 2.375e-05, + "merge_seconds": 10.228315838, + "merge_check_seconds": 1.5226588305085897, + "merge_copy_seconds": 1.683165967464447, + "merge_verify_seconds": 4.40347358584404, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000008", + "barrier_seconds": 1.9375e-05, + "merge_seconds": 18.443692842, + "merge_check_seconds": 0.9344661701470613, + "merge_copy_seconds": 1.473826639354229, + "merge_verify_seconds": 5.511647284030914, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000009", + "barrier_seconds": 0.00099725, + "merge_seconds": 10.863442880000001, + "merge_check_seconds": 2.1177490828558803, + "merge_copy_seconds": 1.6813630983233452, + "merge_verify_seconds": 4.266745909117162, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000010", + "barrier_seconds": 6.5291e-05, + "merge_seconds": 18.866558468, + "merge_check_seconds": 1.270184824243188, + "merge_copy_seconds": 1.6228833757340908, + "merge_verify_seconds": 5.061655658297241, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000024", + "barrier_seconds": 0.0001265, + "merge_seconds": 19.392542217, + "merge_check_seconds": 0.677861861884594, + "merge_copy_seconds": 1.7391378777101636, + "merge_verify_seconds": 2.1950975181534886, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000025", + "barrier_seconds": 2.3167e-05, + "merge_seconds": 5.776185626, + "merge_check_seconds": 1.6192353293299675, + "merge_copy_seconds": 1.3252540081739426, + "merge_verify_seconds": 1.823850715532899, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000026", + "barrier_seconds": 4.1291e-05, + "merge_seconds": 9.244146546, + "merge_check_seconds": 1.490591885522008, + "merge_copy_seconds": 1.878188787959516, + "merge_verify_seconds": 4.153233711607754, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000027", + "barrier_seconds": 0.000156084, + "merge_seconds": 21.22848251, + "merge_check_seconds": 3.194015149027109, + "merge_copy_seconds": 1.272336933761835, + "merge_verify_seconds": 1.8843000615015626, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000028", + "barrier_seconds": 0.000476375, + "merge_seconds": 8.564116379, + "merge_check_seconds": 1.7330228621140122, + "merge_copy_seconds": 1.444510099478066, + "merge_verify_seconds": 3.93396717030555, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000029", + "barrier_seconds": 2.9417e-05, + "merge_seconds": 11.319485464, + "merge_check_seconds": 1.6993928514420986, + "merge_copy_seconds": 1.825788558460772, + "merge_verify_seconds": 4.218742440454662, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000043", + "barrier_seconds": 2.75e-05, + "merge_seconds": 9.563589005, + "merge_check_seconds": 1.3277165601029992, + "merge_copy_seconds": 1.3203403782099485, + "merge_verify_seconds": 4.341508507728577, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000044", + "barrier_seconds": 5.6584e-05, + "merge_seconds": 9.36306013, + "merge_check_seconds": 0.7266873372718692, + "merge_copy_seconds": 1.6805207170546055, + "merge_verify_seconds": 2.676728562451899, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000045", + "barrier_seconds": 3.7125e-05, + "merge_seconds": 15.923956549, + "merge_check_seconds": 0.6875405348837376, + "merge_copy_seconds": 1.614873018115759, + "merge_verify_seconds": 3.519151414744556, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000046", + "barrier_seconds": 4.6709e-05, + "merge_seconds": 5.614199461, + "merge_check_seconds": 1.6306515401229262, + "merge_copy_seconds": 1.2712753061205149, + "merge_verify_seconds": 1.6743122031912208, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000047", + "barrier_seconds": 5.175e-05, + "merge_seconds": 18.020641134, + "merge_check_seconds": 0.6849064817652106, + "merge_copy_seconds": 1.3377375425770879, + "merge_verify_seconds": 1.9327417211607099, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mdifffitid0000048", + "barrier_seconds": 1.9291e-05, + "merge_seconds": 8.716522129, + "merge_check_seconds": 1.3195640128105879, + "merge_copy_seconds": 1.4095240253955126, + "merge_verify_seconds": 4.322235651314259, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mimgtblid0000017", + "barrier_seconds": 7.75e-06, + "merge_seconds": 51.926005982, + "merge_check_seconds": 17.38217044000021, + "merge_copy_seconds": 27.39036550999981, + "merge_verify_seconds": 6.6541218470001695, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mimgtblid0000036", + "barrier_seconds": 7.1667e-05, + "merge_seconds": 53.298720149, + "merge_check_seconds": 18.728920129000016, + "merge_copy_seconds": 27.380674623000004, + "merge_verify_seconds": 6.684228208999912, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mimgtblid0000055", + "barrier_seconds": 1.2375e-05, + "merge_seconds": 53.377830025, + "merge_check_seconds": 18.955533912999954, + "merge_copy_seconds": 27.65483295599995, + "merge_verify_seconds": 6.2337451810000175, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mviewerid0000019", + "barrier_seconds": 4.25e-06, + "merge_seconds": 31.011059181, + "merge_check_seconds": 6.565343984000265, + "merge_copy_seconds": 17.21528402900003, + "merge_verify_seconds": 6.7385543499999585, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mviewerid0000038", + "barrier_seconds": 3.3709e-05, + "merge_seconds": 29.459673139, + "merge_check_seconds": 5.738225310999951, + "merge_copy_seconds": 16.94268330900013, + "merge_verify_seconds": 6.247800998000002, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mviewerid0000057", + "barrier_seconds": 3.4041e-05, + "merge_seconds": 104.951331755, + "merge_check_seconds": 34.67292259200008, + "merge_copy_seconds": 32.80020113799992, + "merge_verify_seconds": 36.95691540500002, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + }, + { + "consumer_activity_id": "montage-independent-work-v1-mviewerid0000058", + "barrier_seconds": 303.802673804, + "merge_seconds": 90.971308457, + "merge_check_seconds": 21.092318261999935, + "merge_copy_seconds": 49.24239904699971, + "merge_verify_seconds": 20.070955631999823, + "merge_ssh_channel_wait_seconds": null, + "merge_execution_seconds": null, + "merge_command_overhead_seconds": null + } + ], + "activities": [ + { + "activity_id": "montage-independent-work-v1-maddid0000018", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 8.217379093170166, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 0.9227643013000488, + "validated_handle_runtime_seconds": 0.9227643013000488, + "terminal_observation_delay_seconds": 50.20883560180664, + "preparation_elapsed_seconds": 68.57136988639832, + "overlapping_transfer_work_seconds": 342.8568494319916, + "launch_wait_seconds": 11.600524663925171, + "queue_seconds": 80.21431493759155, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-maddid0000037", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 8.217379093170166, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 0.9206876754760742, + "validated_handle_runtime_seconds": 0.9206876754760742, + "terminal_observation_delay_seconds": 1.2228929996490479, + "preparation_elapsed_seconds": 68.89127993583679, + "overlapping_transfer_work_seconds": 344.45639967918396, + "launch_wait_seconds": 11.094663858413696, + "queue_seconds": 80.00277495384216, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-maddid0000056", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 8.217379093170166, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 8.156338691711426, + "validated_handle_runtime_seconds": 8.156338691711426, + "terminal_observation_delay_seconds": 2.9332380294799805, + "preparation_elapsed_seconds": 220.91178274154663, + "overlapping_transfer_work_seconds": 1104.5589137077332, + "launch_wait_seconds": 45.30784821510315, + "queue_seconds": 271.6000134944916, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mbackgroundid0000013", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 2.1703304052352905, + "planned_transfer_seconds": 26.051685557105884, + "observed_runtime_seconds": 1.1718418598175049, + "validated_handle_runtime_seconds": 1.1718418598175049, + "terminal_observation_delay_seconds": 1.2681190967559814, + "preparation_elapsed_seconds": 118.78154349327087, + "overlapping_transfer_work_seconds": 236.68998217582703, + "launch_wait_seconds": 5.745228290557861, + "queue_seconds": 463.29780197143555, + "network_bytes": 105617907 + }, + { + "activity_id": "montage-independent-work-v1-mbackgroundid0000014", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 2.1703304052352905, + "planned_transfer_seconds": 26.042708154629054, + "observed_runtime_seconds": 1.141388177871704, + "validated_handle_runtime_seconds": 1.141388177871704, + "terminal_observation_delay_seconds": 1.103311538696289, + "preparation_elapsed_seconds": 110.96577215194702, + "overlapping_transfer_work_seconds": 221.04128527641296, + "launch_wait_seconds": 5.497469425201416, + "queue_seconds": 218.9492540359497, + "network_bytes": 107170830 + }, + { + "activity_id": "montage-independent-work-v1-mbackgroundid0000015", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 2.1703304052352905, + "planned_transfer_seconds": 26.042708154629054, + "observed_runtime_seconds": 0.9362833499908447, + "validated_handle_runtime_seconds": 0.9362833499908447, + "terminal_observation_delay_seconds": 7.1560447216033936, + "preparation_elapsed_seconds": 84.1312563419342, + "overlapping_transfer_work_seconds": 167.38142347335815, + "launch_wait_seconds": 5.180033445358276, + "queue_seconds": 89.32843828201294, + "network_bytes": 109907434 + }, + { + "activity_id": "montage-independent-work-v1-mbackgroundid0000016", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 2.1703304052352905, + "planned_transfer_seconds": 26.042708154629054, + "observed_runtime_seconds": 0.7738721370697021, + "validated_handle_runtime_seconds": 0.7738721370697021, + "terminal_observation_delay_seconds": 1.2404987812042236, + "preparation_elapsed_seconds": 100.25711560249329, + "overlapping_transfer_work_seconds": 199.6324155330658, + "launch_wait_seconds": 5.374896764755249, + "queue_seconds": 331.92769408226013, + "network_bytes": 109611286 + }, + { + "activity_id": "montage-independent-work-v1-mbackgroundid0000032", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 2.1703304052352905, + "planned_transfer_seconds": 25.72935425740644, + "observed_runtime_seconds": 1.029090166091919, + "validated_handle_runtime_seconds": 1.029090166091919, + "terminal_observation_delay_seconds": 1.1272201538085938, + "preparation_elapsed_seconds": 121.99184679985046, + "overlapping_transfer_work_seconds": 243.10734033584595, + "launch_wait_seconds": 9.309908866882324, + "queue_seconds": 219.6455512046814, + "network_bytes": 102529126 + }, + { + "activity_id": "montage-independent-work-v1-mbackgroundid0000033", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 2.1703304052352905, + "planned_transfer_seconds": 25.73833165988333, + "observed_runtime_seconds": 1.9942986965179443, + "validated_handle_runtime_seconds": 1.9942986965179443, + "terminal_observation_delay_seconds": 1.639662504196167, + "preparation_elapsed_seconds": 70.03583884239197, + "overlapping_transfer_work_seconds": 139.198832988739, + "launch_wait_seconds": 9.719558715820312, + "queue_seconds": 79.76926589012146, + "network_bytes": 104335080 + }, + { + "activity_id": "montage-independent-work-v1-mbackgroundid0000034", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 2.1703304052352905, + "planned_transfer_seconds": 25.719094124667663, + "observed_runtime_seconds": 0.921323299407959, + "validated_handle_runtime_seconds": 0.921323299407959, + "terminal_observation_delay_seconds": 1.2212011814117432, + "preparation_elapsed_seconds": 50.394211292266846, + "overlapping_transfer_work_seconds": 99.8698787689209, + "launch_wait_seconds": 5.849879026412964, + "queue_seconds": 408.44285678863525, + "network_bytes": 106653299 + }, + { + "activity_id": "montage-independent-work-v1-mbackgroundid0000035", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 2.1703304052352905, + "planned_transfer_seconds": 25.72935425740644, + "observed_runtime_seconds": 0.7800111770629883, + "validated_handle_runtime_seconds": 0.7800111770629883, + "terminal_observation_delay_seconds": 1.250084638595581, + "preparation_elapsed_seconds": 112.71284008026123, + "overlapping_transfer_work_seconds": 224.53678917884827, + "launch_wait_seconds": 5.558659553527832, + "queue_seconds": 345.0438234806061, + "network_bytes": 106193023 + }, + { + "activity_id": "montage-independent-work-v1-mbackgroundid0000051", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 2.1703304052352905, + "planned_transfer_seconds": 27.677167267855225, + "observed_runtime_seconds": 1.1500000953674316, + "validated_handle_runtime_seconds": 1.1500000953674316, + "terminal_observation_delay_seconds": 1.2671995162963867, + "preparation_elapsed_seconds": 104.91257047653198, + "overlapping_transfer_work_seconds": 208.93211841583252, + "launch_wait_seconds": 8.629173755645752, + "queue_seconds": 329.3676462173462, + "network_bytes": 103997683 + }, + { + "activity_id": "montage-independent-work-v1-mbackgroundid0000052", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 2.1703304052352905, + "planned_transfer_seconds": 27.68614467033194, + "observed_runtime_seconds": 5.974401950836182, + "validated_handle_runtime_seconds": 5.974401950836182, + "terminal_observation_delay_seconds": 1.63820219039917, + "preparation_elapsed_seconds": 58.700331687927246, + "overlapping_transfer_work_seconds": 116.51416802406311, + "launch_wait_seconds": 7.372749090194702, + "queue_seconds": 66.09259963035583, + "network_bytes": 106265678 + }, + { + "activity_id": "montage-independent-work-v1-mbackgroundid0000053", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 2.1703304052352905, + "planned_transfer_seconds": 27.68614467033194, + "observed_runtime_seconds": 9.189680099487305, + "validated_handle_runtime_seconds": 9.189680099487305, + "terminal_observation_delay_seconds": 1.8798363208770752, + "preparation_elapsed_seconds": 58.162331342697144, + "overlapping_transfer_work_seconds": 115.45199608802795, + "launch_wait_seconds": 20.833706378936768, + "queue_seconds": 178.47882795333862, + "network_bytes": 108718484 + }, + { + "activity_id": "montage-independent-work-v1-mbackgroundid0000054", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 2.1703304052352905, + "planned_transfer_seconds": 27.677167267855225, + "observed_runtime_seconds": 1.0543017387390137, + "validated_handle_runtime_seconds": 1.0543017387390137, + "terminal_observation_delay_seconds": 1.285024642944336, + "preparation_elapsed_seconds": 138.9047348499298, + "overlapping_transfer_work_seconds": 276.93618178367615, + "launch_wait_seconds": 7.164134740829468, + "queue_seconds": 482.69294691085815, + "network_bytes": 108668676 + }, + { + "activity_id": "montage-independent-work-v1-mbgmodelid0000012", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 0.5460176467895508, + "planned_transfer_seconds": 124.5302772908974, + "observed_runtime_seconds": 1.7851626873016357, + "validated_handle_runtime_seconds": 1.7851626873016357, + "terminal_observation_delay_seconds": 1.8481450080871582, + "preparation_elapsed_seconds": 314.0487687587738, + "overlapping_transfer_work_seconds": 314.0487687587738, + "launch_wait_seconds": 5.954580783843994, + "queue_seconds": 958.2402384281158, + "network_bytes": 559032552 + }, + { + "activity_id": "montage-independent-work-v1-mbgmodelid0000031", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 0.5460176467895508, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 0.42560553550720215, + "validated_handle_runtime_seconds": 0.42560553550720215, + "terminal_observation_delay_seconds": 1.7464392185211182, + "preparation_elapsed_seconds": 15.83263635635376, + "overlapping_transfer_work_seconds": 15.83263635635376, + "launch_wait_seconds": 5.980073928833008, + "queue_seconds": 23.388925313949585, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mbgmodelid0000050", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 0.5460176467895508, + "planned_transfer_seconds": 120.89033034304163, + "observed_runtime_seconds": 8.379759788513184, + "validated_handle_runtime_seconds": 8.379759788513184, + "terminal_observation_delay_seconds": 1.4407100677490234, + "preparation_elapsed_seconds": 332.3163990974426, + "overlapping_transfer_work_seconds": 332.3163990974426, + "launch_wait_seconds": 33.47814321517944, + "queue_seconds": 445.2871491909027, + "network_bytes": 555568726 + }, + { + "activity_id": "montage-independent-work-v1-mconcatfitid0000011", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.8328795433044434, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 2.2540571689605713, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 23.32234477996826, + "overlapping_transfer_work_seconds": 139.82024097442627, + "launch_wait_seconds": 11.442723274230957, + "queue_seconds": 0.6575043201446533, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mconcatfitid0000030", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 3.633373498916626, + "planned_transfer_seconds": 401.20912584299106, + "observed_runtime_seconds": 0.5490996837615967, + "validated_handle_runtime_seconds": 0.5490996837615967, + "terminal_observation_delay_seconds": 2.3131189346313477, + "preparation_elapsed_seconds": 483.6688690185547, + "overlapping_transfer_work_seconds": 2902.010985851288, + "launch_wait_seconds": 45.82064604759216, + "queue_seconds": 611.7899680137634, + "network_bytes": 1385483689 + }, + { + "activity_id": "montage-independent-work-v1-mconcatfitid0000049", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.8328795433044434, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 2.2926177978515625, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 50.54956102371216, + "overlapping_transfer_work_seconds": 302.75574231147766, + "launch_wait_seconds": 14.850823640823364, + "queue_seconds": 0.18249106407165527, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000005", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 3.0843896865844727, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 15.160211086273193, + "overlapping_transfer_work_seconds": 30.23988103866577, + "launch_wait_seconds": 18.969791650772095, + "queue_seconds": 0.314586877822876, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000006", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 3.6220107078552246, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 24.02555012702942, + "overlapping_transfer_work_seconds": 47.92800426483154, + "launch_wait_seconds": 10.936261653900146, + "queue_seconds": 2.5218377113342285, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000007", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1.763749361038208, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 15.158862590789795, + "overlapping_transfer_work_seconds": 30.224817991256714, + "launch_wait_seconds": 17.331082582473755, + "queue_seconds": 0.0383601188659668, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000008", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1.6519794464111328, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 24.03105640411377, + "overlapping_transfer_work_seconds": 47.93745803833008, + "launch_wait_seconds": 13.31772756576538, + "queue_seconds": 0.13996219635009766, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000009", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 2.193976402282715, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 15.181671857833862, + "overlapping_transfer_work_seconds": 30.256572008132935, + "launch_wait_seconds": 23.887718439102173, + "queue_seconds": 0.8914909362792969, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000010", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 2.191842555999756, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 24.038909673690796, + "overlapping_transfer_work_seconds": 47.93469190597534, + "launch_wait_seconds": 15.457431554794312, + "queue_seconds": 0.46129417419433594, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000024", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 2.8118035793304443, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 28.597239017486572, + "overlapping_transfer_work_seconds": 57.15416955947876, + "launch_wait_seconds": 13.191643953323364, + "queue_seconds": 1.1532094478607178, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000025", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 2.9652798175811768, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 9.97408938407898, + "overlapping_transfer_work_seconds": 19.933677673339844, + "launch_wait_seconds": 16.557165145874023, + "queue_seconds": 0.6804153919219971, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000026", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1.9498262405395508, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 17.80984354019165, + "overlapping_transfer_work_seconds": 35.54551982879639, + "launch_wait_seconds": 17.61879062652588, + "queue_seconds": 1.5359561443328857, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000027", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1.9190011024475098, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 30.3713595867157, + "overlapping_transfer_work_seconds": 60.65973377227783, + "launch_wait_seconds": 11.028402328491211, + "queue_seconds": 0.09577107429504395, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000028", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 2.3990097045898438, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 17.591498374938965, + "overlapping_transfer_work_seconds": 35.095600843429565, + "launch_wait_seconds": 16.68977403640747, + "queue_seconds": 0.6661584377288818, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000029", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 2.516083002090454, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 17.810863494873047, + "overlapping_transfer_work_seconds": 35.557657957077026, + "launch_wait_seconds": 20.842161178588867, + "queue_seconds": 1.356982707977295, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000043", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 2.8281095027923584, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 14.69545841217041, + "overlapping_transfer_work_seconds": 29.32056474685669, + "launch_wait_seconds": 18.391088247299194, + "queue_seconds": 0.5798122882843018, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000044", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 2.9702842235565186, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 15.97597336769104, + "overlapping_transfer_work_seconds": 31.873817920684814, + "launch_wait_seconds": 24.7003173828125, + "queue_seconds": 0, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000045", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1.715392827987671, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 23.174216270446777, + "overlapping_transfer_work_seconds": 46.26511573791504, + "launch_wait_seconds": 48.057162046432495, + "queue_seconds": 0, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000046", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1.6275734901428223, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 8.397797107696533, + "overlapping_transfer_work_seconds": 16.770772695541382, + "launch_wait_seconds": 14.649502515792847, + "queue_seconds": 0.3853318691253662, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000047", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 2.2387824058532715, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 24.95782232284546, + "overlapping_transfer_work_seconds": 49.83530926704407, + "launch_wait_seconds": 56.85736894607544, + "queue_seconds": 0, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mdifffitid0000048", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 2.2217273712158203, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 2.188443183898926, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 15.117409944534302, + "overlapping_transfer_work_seconds": 30.157907724380493, + "launch_wait_seconds": 15.326874017715454, + "queue_seconds": 0.2148139476776123, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mimgtblid0000017", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 1.4470241069793701, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 0.09377503395080566, + "validated_handle_runtime_seconds": 0.09377503395080566, + "terminal_observation_delay_seconds": 0.5997250080108643, + "preparation_elapsed_seconds": 53.429725646972656, + "overlapping_transfer_work_seconds": 213.71890258789062, + "launch_wait_seconds": 7.682278871536255, + "queue_seconds": 61.1309540271759, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mimgtblid0000036", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 1.4470241069793701, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 0.10982584953308105, + "validated_handle_runtime_seconds": 0.10982584953308105, + "terminal_observation_delay_seconds": 0.5766959190368652, + "preparation_elapsed_seconds": 54.787358045578, + "overlapping_transfer_work_seconds": 219.149432182312, + "launch_wait_seconds": 7.358223915100098, + "queue_seconds": 62.162489891052246, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mimgtblid0000055", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 1.4470241069793701, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1.9788739681243896, + "validated_handle_runtime_seconds": 1.9788739681243896, + "terminal_observation_delay_seconds": 1.269636869430542, + "preparation_elapsed_seconds": 54.88259744644165, + "overlapping_transfer_work_seconds": 219.5303897857666, + "launch_wait_seconds": 23.700952529907227, + "queue_seconds": 78.60034942626953, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mprojectid0000001", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 479.89131402969366, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 479.23669385910034, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 0.018802642822265625, + "overlapping_transfer_work_seconds": 0.018802642822265625, + "launch_wait_seconds": 7.828449964523315, + "queue_seconds": 2.2064943313598633, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mprojectid0000002", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 479.89131402969366, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 477.3460416793823, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 0.0280611515045166, + "overlapping_transfer_work_seconds": 0.0280611515045166, + "launch_wait_seconds": 10.815857410430908, + "queue_seconds": 0.3500363826751709, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mprojectid0000003", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 479.89131402969366, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 480.9227578639984, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 0.025228500366210938, + "overlapping_transfer_work_seconds": 0.025228500366210938, + "launch_wait_seconds": 10.81418228149414, + "queue_seconds": 0.3503448963165283, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mprojectid0000004", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 479.89131402969366, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 479.13104367256165, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 0.008282661437988281, + "overlapping_transfer_work_seconds": 0.008282661437988281, + "launch_wait_seconds": 9.8861722946167, + "queue_seconds": 0.05523824691772461, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mprojectid0000020", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 299.55654418468475, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 296.3911814689636, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 6.4798197746276855, + "overlapping_transfer_work_seconds": 6.4798197746276855, + "launch_wait_seconds": 9.186479568481445, + "queue_seconds": 0.02379465103149414, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mprojectid0000021", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 299.55654418468475, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 301.64098715782166, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 6.494693279266357, + "overlapping_transfer_work_seconds": 6.494693279266357, + "launch_wait_seconds": 11.344988584518433, + "queue_seconds": 0.8770179748535156, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mprojectid0000022", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 299.55654418468475, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 296.09501814842224, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 6.492795705795288, + "overlapping_transfer_work_seconds": 6.492795705795288, + "launch_wait_seconds": 11.342601776123047, + "queue_seconds": 0.8775508403778076, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mprojectid0000023", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 299.55654418468475, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 306.816041469574, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 6.500110626220703, + "overlapping_transfer_work_seconds": 6.500110626220703, + "launch_wait_seconds": 7.109496116638184, + "queue_seconds": 0.20971441268920898, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mprojectid0000039", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 432.90292227268213, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 431.76940989494324, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 6.51016640663147, + "overlapping_transfer_work_seconds": 6.51016640663147, + "launch_wait_seconds": 8.574131488800049, + "queue_seconds": 0.6379446983337402, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mprojectid0000040", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 432.90292227268213, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 427.1354908943176, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 6.48828649520874, + "overlapping_transfer_work_seconds": 6.48828649520874, + "launch_wait_seconds": 8.560758590698242, + "queue_seconds": 0.6385776996612549, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mprojectid0000041", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 432.90292227268213, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 428.1778221130371, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 6.514239072799683, + "overlapping_transfer_work_seconds": 6.514239072799683, + "launch_wait_seconds": 10.035039901733398, + "queue_seconds": 2.1930224895477295, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mprojectid0000042", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 432.90292227268213, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 430.4124708175659, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 6.515958309173584, + "overlapping_transfer_work_seconds": 6.515958309173584, + "launch_wait_seconds": 10.453047752380371, + "queue_seconds": 1.7771687507629395, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mviewerid0000019", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 14.030364871025085, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1.7597286701202393, + "validated_handle_runtime_seconds": 1.7597286701202393, + "terminal_observation_delay_seconds": 1.6040945053100586, + "preparation_elapsed_seconds": 31.595834493637085, + "overlapping_transfer_work_seconds": 31.595834493637085, + "launch_wait_seconds": 8.27393913269043, + "queue_seconds": 39.88155460357666, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mviewerid0000038", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 14.030364871025085, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1.3308792114257812, + "validated_handle_runtime_seconds": 1.3308792114257812, + "terminal_observation_delay_seconds": 0.3191654682159424, + "preparation_elapsed_seconds": 30.052446603775024, + "overlapping_transfer_work_seconds": 30.052446603775024, + "launch_wait_seconds": 7.376959562301636, + "queue_seconds": 37.443498611450195, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mviewerid0000057", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 14.030364871025085, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 22.134554862976074, + "validated_handle_runtime_seconds": 22.134554862976074, + "terminal_observation_delay_seconds": 1.393564224243164, + "preparation_elapsed_seconds": 105.5463137626648, + "overlapping_transfer_work_seconds": 105.5463137626648, + "launch_wait_seconds": 38.763895988464355, + "queue_seconds": 144.32117009162903, + "network_bytes": 0 + }, + { + "activity_id": "montage-independent-work-v1-mviewerid0000058", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 42.091094613075256, + "planned_transfer_seconds": 145.07456697523935, + "observed_runtime_seconds": 3.15958833694458, + "validated_handle_runtime_seconds": 3.15958833694458, + "terminal_observation_delay_seconds": 0.3745901584625244, + "preparation_elapsed_seconds": 395.95423579216003, + "overlapping_transfer_work_seconds": 1186.6885924339294, + "launch_wait_seconds": 26.03114151954651, + "queue_seconds": 480.12689185142517, + "network_bytes": 619673320 + } + ], + "limitations": [ + "No multiple attempts observed; this alone does not prove an uninterrupted benchmark.", + "Runtime finish semantics depend on the driver; observation delay is exported only for exact-attempt handles with validated terminal timestamps.", + "API aggregate counters can contain older attempts; audit uses latest distinct IDs.", + "Transfer work is summed overlapping route time, not wall-clock preparation.", + "Plan assignment work sums are not makespan; API timestamp wall and recovered makespan can differ.", + "Cloud lifecycle cost is a list-rate estimate with explicit exclusions, not the provider invoice. The task/window API cost and plan retention-window cost have different bases.", + "e2-medium interference extrapolated to HPC/micro; micro speedup 0.25 is approximate.", + "Whole-workspace real traffic differs from estimated plan edge volumes.", + "Zero reported interference does not prove absence of physical contention." + ], + "product": { + "filename": "mosaic-color.png", + "bytes": 4920451, + "sha256": "63fa4d8447274bdb8a44f6be8cf5a23c0a2f0baa479bd2ae1ec3b541df4e19f2" + } +} diff --git a/experiments/montage-micro-hpc-e2/combined-observation-sensitivity-forecast-20261011.json b/experiments/montage-micro-hpc-e2/combined-observation-sensitivity-forecast-20261011.json new file mode 100644 index 00000000..54593493 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/combined-observation-sensitivity-forecast-20261011.json @@ -0,0 +1,18 @@ +{ + "issue":"AKO-158", + "role":"exploratory-observed-wall-sensitivity-not-service-calibration", + "fixedPlacementSimGridSeconds":3743.3807417053167, + "replannedPrismSeconds":3621.1737574989843, + "replannedSimGridSeconds":3743.3807417053167, + "simulatedActivities":58, + "scientificComputeWorkSeconds":5108.837997198105, + "publicationWorkSecondsNotMakespan":1133.254865169525, + "independentValidation":false, + "limitations":[ + "Input observation and publication use reference wall coefficients, not queue-free service fits.", + "Aggregate publication work matches by weighted construction, not independent validation.", + "Individual dominant publication errors must be checked; summed work cannot replace that check.", + "Final-workspace inherited-input pruning must be separated from snapshot hashing before accepting the profile.", + "No real run was launched using this exploratory forecast." + ] +} diff --git a/experiments/montage-micro-hpc-e2/compute-model-v2.json b/experiments/montage-micro-hpc-e2/compute-model-v2.json new file mode 100644 index 00000000..f3f716bf --- /dev/null +++ b/experiments/montage-micro-hpc-e2/compute-model-v2.json @@ -0,0 +1,1785 @@ +{ + "schemaVersion": 1, + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "activityFamilyTypeIds": { + "madd": "montage-reference-work-v1-madd", + "mbackground": "montage-reference-work-v1-mbackground", + "mbgmodel": "montage-reference-work-v1-mbgmodel", + "mconcatfit": "montage-reference-work-v1-mconcatfit", + "mdifffit": "montage-reference-work-v1-mdifffit", + "mimgtbl": "montage-reference-work-v1-mimgtbl", + "mproject": "montage-reference-work-v1-mproject", + "mviewer": "montage-reference-work-v1-mviewer" + }, + "activityProfiles": [ + { + "id": "montage-reference-work-v1-madd-montage-gcp-fog-e2-micro-01", + "activityTypeId": "montage-reference-work-v1-madd", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-madd-montage-gcp-fog-e2-micro-02", + "activityTypeId": "montage-reference-work-v1-madd", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-madd-montage-gcp-fog-e2-micro-03", + "activityTypeId": "montage-reference-work-v1-madd", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-madd-montage-gcp-fog-e2-micro-04", + "activityTypeId": "montage-reference-work-v1-madd", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-madd-montage-gcp-fog-e2-micro-05", + "activityTypeId": "montage-reference-work-v1-madd", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-madd-montage-gcp-fog-e2-micro-06", + "activityTypeId": "montage-reference-work-v1-madd", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-madd-montage-gcp-fog-e2-micro-07", + "activityTypeId": "montage-reference-work-v1-madd", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-madd-montage-gcp-fog-e2-micro-08", + "activityTypeId": "montage-reference-work-v1-madd", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-madd-montage-gcp-fog-e2-micro-09", + "activityTypeId": "montage-reference-work-v1-madd", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-madd-montage-gcp-fog-e2-micro-10", + "activityTypeId": "montage-reference-work-v1-madd", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbackground-montage-gcp-e2-medium-1", + "activityTypeId": "montage-reference-work-v1-mbackground", + "resourceId": "montage-gcp-e2-medium-1", + "runtimeSeconds": 0.32499454332434613, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02318926479505456, + 0.6267998218536377 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbackground-montage-gcp-e2-medium-2", + "activityTypeId": "montage-reference-work-v1-mbackground", + "resourceId": "montage-gcp-e2-medium-2", + "runtimeSeconds": 0.32499454332434613, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02318926479505456, + 0.6267998218536377 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbackground-montage-gcp-fog-e2-micro-01", + "activityTypeId": "montage-reference-work-v1-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbackground-montage-gcp-fog-e2-micro-02", + "activityTypeId": "montage-reference-work-v1-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbackground-montage-gcp-fog-e2-micro-03", + "activityTypeId": "montage-reference-work-v1-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbackground-montage-gcp-fog-e2-micro-04", + "activityTypeId": "montage-reference-work-v1-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbackground-montage-gcp-fog-e2-micro-05", + "activityTypeId": "montage-reference-work-v1-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbackground-montage-gcp-fog-e2-micro-06", + "activityTypeId": "montage-reference-work-v1-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbackground-montage-gcp-fog-e2-micro-07", + "activityTypeId": "montage-reference-work-v1-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbackground-montage-gcp-fog-e2-micro-08", + "activityTypeId": "montage-reference-work-v1-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbackground-montage-gcp-fog-e2-micro-09", + "activityTypeId": "montage-reference-work-v1-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbackground-montage-gcp-fog-e2-micro-10", + "activityTypeId": "montage-reference-work-v1-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbackground-plafrim-execution-ssh-node-bora001", + "activityTypeId": "montage-reference-work-v1-mbackground", + "resourceId": "plafrim-execution-ssh-node-bora001", + "runtimeSeconds": 2.042281486093998, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.1621987372636795, + 3.9223642349243164 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 2 + } + }, + { + "id": "montage-reference-work-v1-mbackground-plafrim-execution-ssh-node-diablo03", + "activityTypeId": "montage-reference-work-v1-mbackground", + "resourceId": "plafrim-execution-ssh-node-diablo03", + "runtimeSeconds": 2.042281486093998, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.1621987372636795, + 3.9223642349243164 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 2 + } + }, + { + "id": "montage-reference-work-v1-mbgmodel-montage-gcp-fog-e2-micro-01", + "activityTypeId": "montage-reference-work-v1-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbgmodel-montage-gcp-fog-e2-micro-02", + "activityTypeId": "montage-reference-work-v1-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbgmodel-montage-gcp-fog-e2-micro-03", + "activityTypeId": "montage-reference-work-v1-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbgmodel-montage-gcp-fog-e2-micro-04", + "activityTypeId": "montage-reference-work-v1-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbgmodel-montage-gcp-fog-e2-micro-05", + "activityTypeId": "montage-reference-work-v1-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbgmodel-montage-gcp-fog-e2-micro-06", + "activityTypeId": "montage-reference-work-v1-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbgmodel-montage-gcp-fog-e2-micro-07", + "activityTypeId": "montage-reference-work-v1-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbgmodel-montage-gcp-fog-e2-micro-08", + "activityTypeId": "montage-reference-work-v1-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbgmodel-montage-gcp-fog-e2-micro-09", + "activityTypeId": "montage-reference-work-v1-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbgmodel-montage-gcp-fog-e2-micro-10", + "activityTypeId": "montage-reference-work-v1-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbgmodel-montage-gcp-e2-medium-1", + "activityTypeId": "montage-reference-work-v1-mbgmodel", + "resourceId": "montage-gcp-e2-medium-1", + "runtimeSeconds": 0.025992941856384278, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 1, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.025992941856384278, + 0.025992941856384278 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mbgmodel-montage-gcp-e2-medium-2", + "activityTypeId": "montage-reference-work-v1-mbgmodel", + "resourceId": "montage-gcp-e2-medium-2", + "runtimeSeconds": 0.025992941856384278, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 1, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.025992941856384278, + 0.025992941856384278 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mconcatfit-plafrim-execution-ssh-node-bora001", + "activityTypeId": "montage-reference-work-v1-mconcatfit", + "resourceId": "plafrim-execution-ssh-node-bora001", + "runtimeSeconds": 0.16663997313555548, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 1, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.16663997313555548, + 0.16663997313555548 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 1 + } + }, + { + "id": "montage-reference-work-v1-mconcatfit-plafrim-execution-ssh-node-diablo03", + "activityTypeId": "montage-reference-work-v1-mconcatfit", + "resourceId": "plafrim-execution-ssh-node-diablo03", + "runtimeSeconds": 0.16663997313555548, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 1, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.16663997313555548, + 0.16663997313555548 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 1 + } + }, + { + "id": "montage-reference-work-v1-mconcatfit-montage-gcp-fog-e2-micro-01", + "activityTypeId": "montage-reference-work-v1-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mconcatfit-montage-gcp-fog-e2-micro-02", + "activityTypeId": "montage-reference-work-v1-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mconcatfit-montage-gcp-fog-e2-micro-03", + "activityTypeId": "montage-reference-work-v1-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mconcatfit-montage-gcp-fog-e2-micro-04", + "activityTypeId": "montage-reference-work-v1-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mconcatfit-montage-gcp-fog-e2-micro-05", + "activityTypeId": "montage-reference-work-v1-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mconcatfit-montage-gcp-fog-e2-micro-06", + "activityTypeId": "montage-reference-work-v1-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mconcatfit-montage-gcp-fog-e2-micro-07", + "activityTypeId": "montage-reference-work-v1-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mconcatfit-montage-gcp-fog-e2-micro-08", + "activityTypeId": "montage-reference-work-v1-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mconcatfit-montage-gcp-fog-e2-micro-09", + "activityTypeId": "montage-reference-work-v1-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mconcatfit-montage-gcp-fog-e2-micro-10", + "activityTypeId": "montage-reference-work-v1-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mdifffit-montage-gcp-fog-e2-micro-01", + "activityTypeId": "montage-reference-work-v1-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mdifffit-montage-gcp-fog-e2-micro-02", + "activityTypeId": "montage-reference-work-v1-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mdifffit-montage-gcp-fog-e2-micro-03", + "activityTypeId": "montage-reference-work-v1-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mdifffit-montage-gcp-fog-e2-micro-04", + "activityTypeId": "montage-reference-work-v1-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mdifffit-montage-gcp-fog-e2-micro-05", + "activityTypeId": "montage-reference-work-v1-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mdifffit-montage-gcp-fog-e2-micro-06", + "activityTypeId": "montage-reference-work-v1-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mdifffit-montage-gcp-fog-e2-micro-07", + "activityTypeId": "montage-reference-work-v1-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mdifffit-montage-gcp-fog-e2-micro-08", + "activityTypeId": "montage-reference-work-v1-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mdifffit-montage-gcp-fog-e2-micro-09", + "activityTypeId": "montage-reference-work-v1-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mdifffit-montage-gcp-fog-e2-micro-10", + "activityTypeId": "montage-reference-work-v1-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mdifffit-plafrim-execution-ssh-node-bora001", + "activityTypeId": "montage-reference-work-v1-mdifffit", + "resourceId": "plafrim-execution-ssh-node-bora001", + "runtimeSeconds": 0.15401040880303635, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.08495262265205383, + 0.20197521556507458 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 3 + } + }, + { + "id": "montage-reference-work-v1-mdifffit-plafrim-execution-ssh-node-diablo03", + "activityTypeId": "montage-reference-work-v1-mdifffit", + "resourceId": "plafrim-execution-ssh-node-diablo03", + "runtimeSeconds": 0.15401040880303635, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.08495262265205383, + 0.20197521556507458 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 3 + } + }, + { + "id": "montage-reference-work-v1-mdifffit-montage-gcp-e2-medium-1", + "activityTypeId": "montage-reference-work-v1-mdifffit", + "resourceId": "montage-gcp-e2-medium-1", + "runtimeSeconds": 0.3221574563246507, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 1, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.3221574563246507, + 0.3221574563246507 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mdifffit-montage-gcp-e2-medium-2", + "activityTypeId": "montage-reference-work-v1-mdifffit", + "resourceId": "montage-gcp-e2-medium-2", + "runtimeSeconds": 0.3221574563246507, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 1, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.3221574563246507, + 0.3221574563246507 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mimgtbl-montage-gcp-fog-e2-micro-01", + "activityTypeId": "montage-reference-work-v1-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mimgtbl-montage-gcp-fog-e2-micro-02", + "activityTypeId": "montage-reference-work-v1-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mimgtbl-montage-gcp-fog-e2-micro-03", + "activityTypeId": "montage-reference-work-v1-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mimgtbl-montage-gcp-fog-e2-micro-04", + "activityTypeId": "montage-reference-work-v1-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mimgtbl-montage-gcp-fog-e2-micro-05", + "activityTypeId": "montage-reference-work-v1-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mimgtbl-montage-gcp-fog-e2-micro-06", + "activityTypeId": "montage-reference-work-v1-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mimgtbl-montage-gcp-fog-e2-micro-07", + "activityTypeId": "montage-reference-work-v1-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mimgtbl-montage-gcp-fog-e2-micro-08", + "activityTypeId": "montage-reference-work-v1-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mimgtbl-montage-gcp-fog-e2-micro-09", + "activityTypeId": "montage-reference-work-v1-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mimgtbl-montage-gcp-fog-e2-micro-10", + "activityTypeId": "montage-reference-work-v1-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mproject-montage-gcp-e2-medium-1", + "activityTypeId": "montage-reference-work-v1-mproject", + "resourceId": "montage-gcp-e2-medium-1", + "runtimeSeconds": 0.5422618222257024, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 4, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.482687460737569, + 0.6058569879823626 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mproject-montage-gcp-e2-medium-2", + "activityTypeId": "montage-reference-work-v1-mproject", + "resourceId": "montage-gcp-e2-medium-2", + "runtimeSeconds": 0.5422618222257024, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 4, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.482687460737569, + 0.6058569879823626 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mproject-montage-gcp-fog-e2-micro-01", + "activityTypeId": "montage-reference-work-v1-mproject", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mproject-montage-gcp-fog-e2-micro-02", + "activityTypeId": "montage-reference-work-v1-mproject", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mproject-montage-gcp-fog-e2-micro-03", + "activityTypeId": "montage-reference-work-v1-mproject", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mproject-montage-gcp-fog-e2-micro-04", + "activityTypeId": "montage-reference-work-v1-mproject", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mproject-montage-gcp-fog-e2-micro-05", + "activityTypeId": "montage-reference-work-v1-mproject", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mproject-montage-gcp-fog-e2-micro-06", + "activityTypeId": "montage-reference-work-v1-mproject", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mproject-montage-gcp-fog-e2-micro-07", + "activityTypeId": "montage-reference-work-v1-mproject", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mproject-montage-gcp-fog-e2-micro-08", + "activityTypeId": "montage-reference-work-v1-mproject", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mproject-montage-gcp-fog-e2-micro-09", + "activityTypeId": "montage-reference-work-v1-mproject", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mproject-montage-gcp-fog-e2-micro-10", + "activityTypeId": "montage-reference-work-v1-mproject", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mproject-plafrim-execution-ssh-node-bora001", + "activityTypeId": "montage-reference-work-v1-mproject", + "resourceId": "plafrim-execution-ssh-node-bora001", + "runtimeSeconds": 0.280406867515572, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.2784816492421198, + 0.2823320857890241 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 2 + } + }, + { + "id": "montage-reference-work-v1-mproject-plafrim-execution-ssh-node-diablo03", + "activityTypeId": "montage-reference-work-v1-mproject", + "resourceId": "plafrim-execution-ssh-node-diablo03", + "runtimeSeconds": 0.280406867515572, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.2784816492421198, + 0.2823320857890241 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 2 + } + }, + { + "id": "montage-reference-work-v1-mviewer-montage-gcp-e2-medium-1", + "activityTypeId": "montage-reference-work-v1-mviewer", + "resourceId": "montage-gcp-e2-medium-1", + "runtimeSeconds": 0.24281440161857284, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.07336857739616842, + 0.41226022584097727 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mviewer-montage-gcp-e2-medium-2", + "activityTypeId": "montage-reference-work-v1-mviewer", + "resourceId": "montage-gcp-e2-medium-2", + "runtimeSeconds": 0.24281440161857284, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.07336857739616842, + 0.41226022584097727 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mviewer-montage-gcp-fog-e2-micro-01", + "activityTypeId": "montage-reference-work-v1-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mviewer-montage-gcp-fog-e2-micro-02", + "activityTypeId": "montage-reference-work-v1-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mviewer-montage-gcp-fog-e2-micro-03", + "activityTypeId": "montage-reference-work-v1-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mviewer-montage-gcp-fog-e2-micro-04", + "activityTypeId": "montage-reference-work-v1-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mviewer-montage-gcp-fog-e2-micro-05", + "activityTypeId": "montage-reference-work-v1-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mviewer-montage-gcp-fog-e2-micro-06", + "activityTypeId": "montage-reference-work-v1-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mviewer-montage-gcp-fog-e2-micro-07", + "activityTypeId": "montage-reference-work-v1-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mviewer-montage-gcp-fog-e2-micro-08", + "activityTypeId": "montage-reference-work-v1-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mviewer-montage-gcp-fog-e2-micro-09", + "activityTypeId": "montage-reference-work-v1-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "montage-reference-work-v1-mviewer-montage-gcp-fog-e2-micro-10", + "activityTypeId": "montage-reference-work-v1-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + } + ], + "limitations": [ + "Class-level historical wall rates; no per-activity replay.", + "Unobserved family/class pairs have no measured profile.", + "Includes physical sharing: additional interference factors require separate evidence." + ] +} diff --git a/experiments/montage-micro-hpc-e2/docker-image-bootstrap.yaml b/experiments/montage-micro-hpc-e2/docker-image-bootstrap.yaml new file mode 100644 index 00000000..9ff9b9d3 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/docker-image-bootstrap.yaml @@ -0,0 +1,31 @@ +--- +- name: Validate preinstalled Montage worker image and prepare fresh workspace + hosts: all + become: true + gather_facts: false + tasks: + - name: Require Docker already installed + ansible.builtin.command: docker --version + changed_when: false + - name: Require whole-workspace runtime tools + ansible.builtin.shell: command -v rsync && command -v tar && command -v python3 && command -v sha256sum + changed_when: false + - name: Enable preinstalled Docker service + ansible.builtin.service: + name: docker + state: started + enabled: true + - name: Grant runtime user Docker access + ansible.builtin.user: + name: "{{ ansible_user }}" + groups: docker + append: true + - name: Require baked image manifest + ansible.builtin.command: test -s /etc/akoflow-worker-image.json + changed_when: false + - name: Create isolated worker workspace + ansible.builtin.file: + path: "{{ akoflow_workspace_path }}" + state: directory + owner: "{{ ansible_user }}" + mode: "0770" diff --git a/experiments/montage-micro-hpc-e2/docker-image-prepare.sh b/experiments/montage-micro-hpc-e2/docker-image-prepare.sh new file mode 100644 index 00000000..5cf0c5b0 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/docker-image-prepare.sh @@ -0,0 +1,53 @@ +#!/bin/sh +set -eu +# Execute only through console API on the disposable AKO-136 builder. +test "$(id -u)" = 0 +test -f /etc/akoflow-disposable-image-builder +test "$(cat /etc/akoflow-disposable-image-builder)" = AKO-136 +test -d /akoflow/workspace +test -z "$(find /akoflow/workspace -mindepth 1 -maxdepth 1 -print -quit)" +test -z "$(docker ps -aq)" +docker version >/dev/null +for tool in rsync tar python3 sha256sum; do command -v "$tool" >/dev/null; done +systemctl is-active --quiet docker +systemctl is-active --quiet google-guest-agent + +python3 - <<'PY' +import json, subprocess +from pathlib import Path +packages = subprocess.check_output([ + 'dpkg-query', '-W', '-f=${Package}=${Version}\n', + 'docker.io', 'rsync', 'tar', 'python3', 'google-guest-agent' +], text=True).splitlines() +Path('/etc/akoflow-worker-image.json').write_text(json.dumps({ + 'schemaVersion': 1, 'issue': 'AKO-136', 'os': 'ubuntu-24.04', + 'packages': sorted(packages), 'workspaceEmpty': True, + 'dockerEnabled': True, 'uniqueHostKeysOnBoot': True +}, sort_keys=True) + '\n') +Path('/etc/systemd/system/akoflow-image-hostkeys.service').write_text('''[Unit] +Description=Generate unique SSH host keys for a cloned Akoflow worker +After=local-fs.target +Before=ssh.service google-guest-agent.service google-startup-scripts.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/ssh-keygen -A +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target +''') +PY +systemctl daemon-reload +systemctl enable akoflow-image-hostkeys.service +cloud-init clean --logs --machine-id --seed +rm -f -- /etc/akoflow-disposable-image-builder +# No remote activity data exists: explicit host identity files of this builder only. +for key in /etc/ssh/ssh_host_rsa_key /etc/ssh/ssh_host_rsa_key.pub \ + /etc/ssh/ssh_host_ecdsa_key /etc/ssh/ssh_host_ecdsa_key.pub \ + /etc/ssh/ssh_host_ed25519_key /etc/ssh/ssh_host_ed25519_key.pub; do + rm -f -- "$key" +done +sync +sha256sum /etc/akoflow-worker-image.json +printf 'IMAGE_PREPARED\n' diff --git a/experiments/montage-micro-hpc-e2/docker-image-real-v2-boot-proof.json b/experiments/montage-micro-hpc-e2/docker-image-real-v2-boot-proof.json new file mode 100644 index 00000000..648c67e6 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/docker-image-real-v2-boot-proof.json @@ -0,0 +1,23 @@ +{ + "issue": "AKO-136", + "runId": "montage-docker-image-real-20261010-v2", + "imageName": "akoflow-montage-docker-20261010-v1", + "immutableProviderImageId": "6709737537929682262", + "evidence": "Read-only Compute API instance and actual boot-disk sourceImage comparison", + "scope": "All twelve VM generations in the execution plan: eight initial and four later lifecycle allocations", + "diskSizeGiB": 30, + "instances": [ + {"id": "cloud-instance-d6a26713-f981-4f6d-875b-8219483d631b", "createdAt": "2026-10-10T15:04:19.704-07:00", "imageMatches": true}, + {"id": "cloud-instance-2aff9f25-3032-43be-87b7-a032c1e55605", "createdAt": "2026-10-10T15:04:14.475-07:00", "imageMatches": true}, + {"id": "cloud-instance-89e10a65-888d-4520-bea0-4b55ea2ad649", "createdAt": "2026-10-10T15:04:14.744-07:00", "imageMatches": true}, + {"id": "cloud-instance-96a445f1-6b98-4a24-88e6-48e77a91a682", "createdAt": "2026-10-10T15:04:08.006-07:00", "imageMatches": true}, + {"id": "cloud-instance-e9fdb058-a382-47fc-a5ca-9342e724179c", "createdAt": "2026-10-10T15:02:58.555-07:00", "imageMatches": true}, + {"id": "cloud-instance-9ea48fa9-cc99-427c-8388-edad043e3f65", "createdAt": "2026-10-10T15:02:59.589-07:00", "imageMatches": true}, + {"id": "cloud-instance-781541a5-1c95-4eba-804f-6a081fa38951", "createdAt": "2026-10-10T15:02:58.731-07:00", "imageMatches": true}, + {"id": "cloud-instance-4abc60f3-a9ce-4205-b12a-de58ca26a949", "createdAt": "2026-10-10T15:02:59.741-07:00", "imageMatches": true}, + {"id": "cloud-instance-9d21c71f-287c-4223-b6ad-fac6ab2c4420", "createdAt": "2026-10-10T15:18:33.633-07:00", "imageMatches": true}, + {"id": "cloud-instance-8f7323e2-de19-4eff-bbe0-e4372d11e6a5", "createdAt": "2026-10-10T15:37:29.536-07:00", "imageMatches": true}, + {"id": "cloud-instance-dd166459-34ff-497e-a60f-f28e37a9f4a2", "createdAt": "2026-10-10T15:38:11.071-07:00", "imageMatches": true}, + {"id": "cloud-instance-84fe60a9-715b-458b-b0f8-6fcdaba5a0e0", "createdAt": "2026-10-10T15:43:46.588-07:00", "imageMatches": true} + ] +} diff --git a/experiments/montage-micro-hpc-e2/docker-image-real-v2-completion.json b/experiments/montage-micro-hpc-e2/docker-image-real-v2-completion.json new file mode 100644 index 00000000..a20dd190 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/docker-image-real-v2-completion.json @@ -0,0 +1,46 @@ +{ + "runId": "montage-docker-image-real-20261010-v2", + "status": "completed", + "activitiesCompleted": 58, + "failedActivities": 0, + "activityAttempts": 1, + "successfulRuntimeHandles": 58, + "dockerImageBootVerifiedInstances": 12, + "topology": {"e2Micro": 10, "e2Medium": 2, "hpcAssignments": 8}, + "audit": { + "missingDependencyRoutes": 0, + "transferBarrierViolations": 0, + "unreleasedConsumerLeases": 0, + "overlappingActivitiesOnMicro": 0, + "cloudHpcRoutesWithoutRelay": 0 + }, + "makespanSeconds": {"plan": 3617.9382322026677, "simgrid": 3617.4981106317523, "realReported": 6924.248950242996, "realRunTimestampWall": 6949.0}, + "estimatedCostsUSD": { + "planRetentionWindows": 0.17934833120896376, + "simgridTaskWindows": 0.12258985813176235, + "realReportedTaskWindows": 0.3014926393284796, + "realComputeAndBootDiskLifecycle": 0.35320463779140077 + }, + "cleanup": { + "runInstancesDestroyed": 12, + "apiNonDestroyedInstances": 0, + "apiActiveCloudOperations": 0, + "gcpManagedInstancesRemaining": 0, + "gcpManagedBootDisksRemaining": 0, + "runFirewallRulesRemaining": 0, + "unrelatedHistoricalFirewallRulesObserved": 3, + "hpcUserSqueueJobs": 0, + "hpcDiagnosticConnectionRestoredExactly": true, + "reusableDockerImageRetainedIntentionally": true + }, + "product": {"filename": "docker-image-real-v2-mosaic-color.png", "width": 1800, "height": 1800, "bytes": 4920451, "sha256": "63fa4d8447274bdb8a44f6be8cf5a23c0a2f0baa479bd2ae1ec3b541df4e19f2"}, + "limitations": [ + "Costs use distinct lifecycle/window bases and must not be compared as identical billing scopes.", + "Cloud lifecycle estimate excludes network egress, public IPv4, reusable image storage, discounts and tax; it is not a provider invoice.", + "No PlaFRIM monetary tariff is calibrated.", + "Micro speedup 0.25 is approximate; e2-medium interference is extrapolated to micro/HPC.", + "Whole-workspace real traffic is larger than planned dependency edge volumes; overlapping work totals are not makespan.", + "Zero reported real interference does not establish absence of physical contention.", + "Historical firewall rules outside the run were observed but not modified." + ] +} diff --git a/experiments/montage-micro-hpc-e2/docker-image-real-v2-mosaic-color.png b/experiments/montage-micro-hpc-e2/docker-image-real-v2-mosaic-color.png new file mode 100644 index 00000000..57f93ff3 Binary files /dev/null and b/experiments/montage-micro-hpc-e2/docker-image-real-v2-mosaic-color.png differ diff --git a/experiments/montage-micro-hpc-e2/docker-image-real-v2-results.json b/experiments/montage-micro-hpc-e2/docker-image-real-v2-results.json new file mode 100644 index 00000000..41340b7f --- /dev/null +++ b/experiments/montage-micro-hpc-e2/docker-image-real-v2-results.json @@ -0,0 +1,6007 @@ +{ + "schema_version": 1, + "run_id": "montage-docker-image-real-20261010-v2", + "attempt_history": "single_observed_attempt_per_activity", + "audit": { + "expected_count": 58, + "missing": [ + + ], + "unexpected": [ + + ], + "not_completed": [ + + ], + "incomplete_transfer_count": 0, + "barrier_violation_count": 0, + "unreleased_lease_count": 0, + "single_lane_micro_overlap_count": 0, + "missing_dependency_route_count": 0, + "cross_cloud_hpc_non_relay_count": 0 + }, + "plan": { + "makespanSeconds": 3617.9382322026677, + "cost": 0.17934833120896376 + }, + "plan_assignment_compute_work_seconds": 27553.49476684796, + "plan_assignment_transfer_work_seconds": 28.176017932812783, + "simulation": { + "makespanSeconds": 3617.4981106317523, + "cost": 0.12258985813176235, + "breakdown": { + "computeSeconds": 27537.919503930363, + "transferSeconds": 37.30724763034755, + "queueSeconds": 4373.90484996361, + "interferenceSeconds": 3334.9195039303604, + "overheadSeconds": 0 + }, + "transferredBytes": 880135168 + }, + "real_api_reported": { + "makespanSeconds": 6924.248950242996, + "cost": 0.3014926393284796, + "breakdown": { + "computeSeconds": 18749.882414340973, + "transferSeconds": 34311.56208539009, + "queueSeconds": 18366.654416322708, + "interferenceSeconds": 0, + "overheadSeconds": 46.631765842437744 + }, + "transferredBytes": 26793117698 + }, + "real_cloud_lifecycle_estimate": { + "costBasis": "VM compute and boot-disk lifecycle at on-demand marginal rates; not a provider invoice", + "excludedCosts": [ + "network-egress", + "public-ipv4", + "image-storage", + "discounts-and-tax" + ], + "status": "estimated", + "currency": "USD", + "estimatedCost": 0.35320463779140077, + "knownCost": 0.35320463779140077, + "asOf": "2026-10-11T00:02:51.980619209Z" + }, + "real_cloud_generations": [ + { + "instance_id": "cloud-instance-2aff9f25-3032-43be-87b7-a032c1e55605", + "status": "destroyed", + "billing": { + "pricingStatus": "estimated", + "priceCurrency": "USD", + "priceSource": "gcp-public-catalog", + "priceObservedAt": "2026-10-10T21:57:24.147279919Z", + "priceRegion": "us-central1", + "priceMachineType": "e2-micro", + "startedAt": "2026-10-10T15:04:14.475-07:00", + "finishedAt": "2026-10-11T00:01:17.487960429Z", + "accumulatedComputeSeconds": 6920.74158009, + "computePricePerSecond": 2.3267854166666665e-06, + "diskPricePerSecond": 1.1415525114155251e-06, + "asOf": "2026-10-11T00:02:51.980619209Z", + "currentCost": 0.02412021866375354 + } + }, + { + "instance_id": "cloud-instance-4abc60f3-a9ce-4205-b12a-de58ca26a949", + "status": "destroyed", + "billing": { + "pricingStatus": "estimated", + "priceCurrency": "USD", + "priceSource": "gcp-public-catalog", + "priceObservedAt": "2026-10-10T21:57:24.147279919Z", + "priceRegion": "us-central1", + "priceMachineType": "e2-medium", + "startedAt": "2026-10-10T15:02:59.741-07:00", + "finishedAt": "2026-10-11T00:01:50.27568293Z", + "accumulatedComputeSeconds": 6962.18495638, + "computePricePerSecond": 9.307141666666666e-06, + "diskPricePerSecond": 1.1415525114155251e-06, + "asOf": "2026-10-11T00:02:51.980619209Z", + "currentCost": 0.07293792147359839 + } + }, + { + "instance_id": "cloud-instance-781541a5-1c95-4eba-804f-6a081fa38951", + "status": "destroyed", + "billing": { + "pricingStatus": "estimated", + "priceCurrency": "USD", + "priceSource": "gcp-public-catalog", + "priceObservedAt": "2026-10-10T21:57:24.147279919Z", + "priceRegion": "us-central1", + "priceMachineType": "e2-micro", + "startedAt": "2026-10-10T15:02:58.731-07:00", + "finishedAt": "2026-10-11T00:01:38.4587148Z", + "accumulatedComputeSeconds": 6279.593405258, + "computePricePerSecond": 2.3267854166666665e-06, + "diskPricePerSecond": 1.1415525114155251e-06, + "asOf": "2026-10-11T00:02:51.980619209Z", + "currentCost": 0.022738809411375144 + } + }, + { + "instance_id": "cloud-instance-84fe60a9-715b-458b-b0f8-6fcdaba5a0e0", + "status": "destroyed", + "billing": { + "pricingStatus": "estimated", + "priceCurrency": "USD", + "priceSource": "gcp-public-catalog", + "priceObservedAt": "2026-10-10T21:57:24.147279919Z", + "priceRegion": "us-central1", + "priceMachineType": "e2-micro", + "startedAt": "2026-10-10T15:43:46.588-07:00", + "finishedAt": "2026-10-11T00:00:57.252897961Z", + "accumulatedComputeSeconds": 3534.903372385, + "computePricePerSecond": 2.3267854166666665e-06, + "diskPricePerSecond": 1.1415525114155251e-06, + "asOf": "2026-10-11T00:02:51.980619209Z", + "currentCost": 0.013511108759982333 + } + }, + { + "instance_id": "cloud-instance-89e10a65-888d-4520-bea0-4b55ea2ad649", + "status": "destroyed", + "billing": { + "pricingStatus": "estimated", + "priceCurrency": "USD", + "priceSource": "gcp-public-catalog", + "priceObservedAt": "2026-10-10T21:57:24.147279919Z", + "priceRegion": "us-central1", + "priceMachineType": "e2-micro", + "startedAt": "2026-10-10T15:04:14.744-07:00", + "finishedAt": "2026-10-11T00:01:29.958159712Z", + "accumulatedComputeSeconds": 6920.891548757, + "computePricePerSecond": 2.3267854166666665e-06, + "diskPricePerSecond": 1.1415525114155251e-06, + "asOf": "2026-10-11T00:02:51.980619209Z", + "currentCost": 0.024134495918344663 + } + }, + { + "instance_id": "cloud-instance-8f7323e2-de19-4eff-bbe0-e4372d11e6a5", + "status": "destroyed", + "billing": { + "pricingStatus": "estimated", + "priceCurrency": "USD", + "priceSource": "gcp-public-catalog", + "priceObservedAt": "2026-10-10T21:57:24.147279919Z", + "priceRegion": "us-central1", + "priceMachineType": "e2-micro", + "startedAt": "2026-10-10T15:37:29.536-07:00", + "finishedAt": "2026-10-11T00:00:57.252127794Z", + "accumulatedComputeSeconds": 4351.093916379, + "computePricePerSecond": 2.3267854166666665e-06, + "diskPricePerSecond": 1.1415525114155251e-06, + "asOf": "2026-10-11T00:02:51.980619209Z", + "currentCost": 0.01584063279331698 + } + }, + { + "instance_id": "cloud-instance-96a445f1-6b98-4a24-88e6-48e77a91a682", + "status": "destroyed", + "billing": { + "pricingStatus": "estimated", + "priceCurrency": "USD", + "priceSource": "gcp-public-catalog", + "priceObservedAt": "2026-10-10T21:57:24.147279919Z", + "priceRegion": "us-central1", + "priceMachineType": "e2-micro", + "startedAt": "2026-10-10T15:04:08.006-07:00", + "finishedAt": "2026-10-11T00:01:37.336157591Z", + "accumulatedComputeSeconds": 6354.843278714, + "computePricePerSecond": 2.3267854166666665e-06, + "diskPricePerSecond": 1.1415525114155251e-06, + "asOf": "2026-10-11T00:02:51.980619209Z", + "currentCost": 0.022833537211309124 + } + }, + { + "instance_id": "cloud-instance-9d21c71f-287c-4223-b6ad-fac6ab2c4420", + "status": "destroyed", + "billing": { + "pricingStatus": "estimated", + "priceCurrency": "USD", + "priceSource": "gcp-public-catalog", + "priceObservedAt": "2026-10-10T21:57:24.147279919Z", + "priceRegion": "us-central1", + "priceMachineType": "e2-micro", + "startedAt": "2026-10-10T15:18:33.633-07:00", + "finishedAt": "2026-10-11T00:01:14.949106886Z", + "accumulatedComputeSeconds": 6061.865630673, + "computePricePerSecond": 2.3267854166666665e-06, + "diskPricePerSecond": 1.1415525114155251e-06, + "asOf": "2026-10-11T00:02:51.980619209Z", + "currentCost": 0.02113812642268346 + } + }, + { + "instance_id": "cloud-instance-9ea48fa9-cc99-427c-8388-edad043e3f65", + "status": "destroyed", + "billing": { + "pricingStatus": "estimated", + "priceCurrency": "USD", + "priceSource": "gcp-public-catalog", + "priceObservedAt": "2026-10-10T21:57:24.147279919Z", + "priceRegion": "us-central1", + "priceMachineType": "e2-micro", + "startedAt": "2026-10-10T15:02:59.589-07:00", + "finishedAt": "2026-10-11T00:01:38.411514133Z", + "accumulatedComputeSeconds": 6277.491584424, + "computePricePerSecond": 2.3267854166666665e-06, + "diskPricePerSecond": 1.1415525114155251e-06, + "asOf": "2026-10-11T00:02:51.980619209Z", + "currentCost": 0.0227328855912154 + } + }, + { + "instance_id": "cloud-instance-d6a26713-f981-4f6d-875b-8219483d631b", + "status": "destroyed", + "billing": { + "pricingStatus": "estimated", + "priceCurrency": "USD", + "priceSource": "gcp-public-catalog", + "priceObservedAt": "2026-10-10T21:57:24.147279919Z", + "priceRegion": "us-central1", + "priceMachineType": "e2-micro", + "startedAt": "2026-10-10T15:04:19.704-07:00", + "finishedAt": "2026-10-11T00:01:19.96724593Z", + "accumulatedComputeSeconds": 6406.797562841, + "computePricePerSecond": 2.3267854166666665e-06, + "diskPricePerSecond": 1.1415525114155251e-06, + "asOf": "2026-10-11T00:02:51.980619209Z", + "currentCost": 0.022921242275943478 + } + }, + { + "instance_id": "cloud-instance-dd166459-34ff-497e-a60f-f28e37a9f4a2", + "status": "destroyed", + "billing": { + "pricingStatus": "estimated", + "priceCurrency": "USD", + "priceSource": "gcp-public-catalog", + "priceObservedAt": "2026-10-10T21:57:24.147279919Z", + "priceRegion": "us-central1", + "priceMachineType": "e2-micro", + "startedAt": "2026-10-10T15:38:11.071-07:00", + "finishedAt": "2026-10-11T00:00:57.394872794Z", + "accumulatedComputeSeconds": 4607.231701795, + "computePricePerSecond": 2.3267854166666665e-06, + "diskPricePerSecond": 1.1415525114155251e-06, + "asOf": "2026-10-11T00:02:51.980619209Z", + "currentCost": 0.01638935902443182 + } + }, + { + "instance_id": "cloud-instance-e9fdb058-a382-47fc-a5ca-9342e724179c", + "status": "destroyed", + "billing": { + "pricingStatus": "estimated", + "priceCurrency": "USD", + "priceSource": "gcp-public-catalog", + "priceObservedAt": "2026-10-10T21:57:24.147279919Z", + "priceRegion": "us-central1", + "priceMachineType": "e2-medium", + "startedAt": "2026-10-10T15:02:58.555-07:00", + "finishedAt": "2026-10-11T00:01:57.448176711Z", + "accumulatedComputeSeconds": 7065.206608636, + "computePricePerSecond": 9.307141666666666e-06, + "diskPricePerSecond": 1.1415525114155251e-06, + "asOf": "2026-10-11T00:02:51.980619209Z", + "currentCost": 0.0739063002454464 + } + } + ], + "real_api_run_timestamp_wall_seconds": 6949.0, + "real_network_bytes_from_routes": 33143371363, + "transfer_route_count": 122, + "dependency_route_count": 114, + "cross_cloud_hpc_relay_count": 22, + "relay_server_network_bytes": 9927073037, + "routes": [ + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.012964010238647461, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.0023751258850097656, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.0016813278198242188, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.002091646194458008, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.003881216049194336, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000048", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.010619878768920898, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000039", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 6.485845327377319, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000042", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 6.489797830581665, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013", + "strategy": "direct-runtime", + "duration_seconds": 998.9238812923431, + "logical_bytes": 740939386, + "network_bytes": 426325775, + "relay_server_network_bytes": 0, + "archive_bytes": 426325775, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 13.712441924 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 22.321715386 + }, + { + "name": "peer-private-upload", + "durationSeconds": 50.113527148 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 237.244030355 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 762.31935372 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 236.587417607 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014", + "strategy": "direct-runtime", + "duration_seconds": 998.9254643917084, + "logical_bytes": 106346880, + "network_bytes": 426222183, + "relay_server_network_bytes": 0, + "archive_bytes": 426222183, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 14.107132715 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 28.006995429 + }, + { + "name": "peer-private-upload", + "durationSeconds": 72.944541991 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 467.73053142 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 762.31935372 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 236.587417607 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "strategy": "gateway", + "duration_seconds": 998.9247858524323, + "logical_bytes": 106346880, + "network_bytes": 852807399, + "relay_server_network_bytes": 852807399, + "archive_bytes": 426351589, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 7.708e-06, + "rsync_execution_seconds": 88.282822748, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 56.999876277 + }, + { + "name": "archive-validation", + "durationSeconds": 3.852883043 + }, + { + "name": "archive-upload", + "durationSeconds": 88.28292704 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 209.987703305 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 762.31935372 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 236.587417607 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016", + "strategy": "direct-runtime", + "duration_seconds": 998.9249968528748, + "logical_bytes": 106346880, + "network_bytes": 426294585, + "relay_server_network_bytes": 0, + "archive_bytes": 426294585, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 14.119102382 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 28.842641555 + }, + { + "name": "peer-private-upload", + "durationSeconds": 55.617819192 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 244.180149525 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 762.31935372 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 236.587417607 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "strategy": "direct-runtime", + "duration_seconds": 998.9256546497345, + "logical_bytes": 2606, + "network_bytes": 596359538, + "relay_server_network_bytes": 0, + "archive_bytes": 596359538, + "files": 1, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.514343876 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 203.389851175 + }, + { + "name": "peer-private-upload", + "durationSeconds": 204.790461176 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 347.259798242 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 762.31935372 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 236.587417607 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032", + "strategy": "direct-runtime", + "duration_seconds": 870.5849549770355, + "logical_bytes": 732938742, + "network_bytes": 419365583, + "relay_server_network_bytes": 0, + "archive_bytes": 419365583, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.472338209 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 125.651768557 + }, + { + "name": "peer-private-upload", + "durationSeconds": 42.47936181 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 12.180462672 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 804.122080071 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 66.450399947 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033", + "strategy": "direct-runtime", + "duration_seconds": 870.5858879089355, + "logical_bytes": 105114240, + "network_bytes": 418940522, + "relay_server_network_bytes": 0, + "archive_bytes": 418940522, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.44606396 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 136.133556812 + }, + { + "name": "peer-private-upload", + "durationSeconds": 15.939571923 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 13.964348673 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 804.122080071 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 66.450399947 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034", + "strategy": "direct-runtime", + "duration_seconds": 870.5855317115784, + "logical_bytes": 105027840, + "network_bytes": 419212118, + "relay_server_network_bytes": 0, + "archive_bytes": 419212118, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.412250835 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 18.593731342 + }, + { + "name": "peer-private-upload", + "durationSeconds": 6.908585211 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 12.271324006 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 804.122080071 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 66.450399947 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035", + "strategy": "direct-runtime", + "duration_seconds": 870.5852825641632, + "logical_bytes": 105073920, + "network_bytes": 418994616, + "relay_server_network_bytes": 0, + "archive_bytes": 418994616, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 5.73280242 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 304.010986636 + }, + { + "name": "peer-private-upload", + "durationSeconds": 20.299676467 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 12.235476714 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 804.122080071 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 66.450399947 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "strategy": "direct-runtime", + "duration_seconds": 870.5863614082336, + "logical_bytes": 2600, + "network_bytes": 587177478, + "relay_server_network_bytes": 0, + "archive_bytes": 587177478, + "files": 1, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.566003709 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 724.592180869 + }, + { + "name": "peer-private-upload", + "durationSeconds": 56.750840525 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 18.041546049 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 804.122080071 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 66.450399947 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "strategy": "gateway", + "duration_seconds": 607.8986115455627, + "logical_bytes": 739073142, + "network_bytes": 848057754, + "relay_server_network_bytes": 848057754, + "archive_bytes": 423977058, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 2.75e-05, + "rsync_execution_seconds": 66.87356228, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 60.704215862 + }, + { + "name": "archive-validation", + "durationSeconds": 3.854221543 + }, + { + "name": "archive-upload", + "durationSeconds": 66.873718071 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 151.045887901 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 364.322760332 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 243.573326361 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052", + "strategy": "direct-runtime", + "duration_seconds": 607.8988606929779, + "logical_bytes": 106058880, + "network_bytes": 423742384, + "relay_server_network_bytes": 0, + "archive_bytes": 423742384, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.331554917 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 27.582921763 + }, + { + "name": "peer-private-upload", + "durationSeconds": 6.753221753 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 17.674082592 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 364.322760332 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 243.573326361 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053", + "strategy": "direct-runtime", + "duration_seconds": 607.8987860679626, + "logical_bytes": 106058880, + "network_bytes": 423976160, + "relay_server_network_bytes": 0, + "archive_bytes": 423976160, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.496523835 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 153.340793487 + }, + { + "name": "peer-private-upload", + "durationSeconds": 69.981841363 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 136.494977644 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 364.322760332 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 243.573326361 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054", + "strategy": "direct-runtime", + "duration_seconds": 607.898609161377, + "logical_bytes": 106018560, + "network_bytes": 423789984, + "relay_server_network_bytes": 0, + "archive_bytes": 423789984, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.450065334 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 52.378699941 + }, + { + "name": "peer-private-upload", + "durationSeconds": 24.046561595 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 191.248107044 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 364.322760332 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 243.573326361 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "strategy": "direct-runtime", + "duration_seconds": 607.8989915847778, + "logical_bytes": 2594, + "network_bytes": 591839049, + "relay_server_network_bytes": 0, + "archive_bytes": 591839049, + "files": 1, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.360760667 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 38.04543835 + }, + { + "name": "peer-private-upload", + "durationSeconds": 8.269827837 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 55.418071692 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 364.322760332 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 243.573326361 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "strategy": "direct-runtime", + "duration_seconds": 133.3755121231079, + "logical_bytes": 634552186, + "network_bytes": 369609876, + "relay_server_network_bytes": 0, + "archive_bytes": 369609876, + "files": 73, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.347021292 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 52.145165899 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 14.740183174 + }, + { + "name": "peer-private-upload", + "durationSeconds": 49.236930189 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 7.572826046 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 127.879948059 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 5.494888086 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000001", + "strategy": "runtime-local", + "duration_seconds": 136.59450793266296, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 127.879948059 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 5.494888086 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "strategy": "direct-runtime", + "duration_seconds": 140.25654578208923, + "logical_bytes": 634552186, + "network_bytes": 369609876, + "relay_server_network_bytes": 0, + "archive_bytes": 369609876, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.339557626 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 52.311870149 + }, + { + "name": "peer-private-upload", + "durationSeconds": 54.475972775 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 9.18253513 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 120.300814096 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.955273884 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000002", + "strategy": "runtime-local", + "duration_seconds": 143.2702453136444, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 120.300814096 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.955273884 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "strategy": "gateway", + "duration_seconds": 197.9253568649292, + "logical_bytes": null, + "network_bytes": 739310116, + "relay_server_network_bytes": 739310116, + "archive_bytes": 369609876, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 1.7417e-05, + "rsync_execution_seconds": 108.393497508, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 52.234585523 + }, + { + "name": "archive-validation", + "durationSeconds": 3.341227627 + }, + { + "name": "archive-upload", + "durationSeconds": 108.393672425 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 10.001879504 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 176.197660539 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 21.729738135 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000003", + "strategy": "gateway", + "duration_seconds": 197.92564368247986, + "logical_bytes": null, + "network_bytes": 290209787, + "relay_server_network_bytes": 290209787, + "archive_bytes": 145087119, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 2.8917e-05, + "rsync_execution_seconds": 24.609660678, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 11.782335589 + }, + { + "name": "archive-validation", + "durationSeconds": 1.175621292 + }, + { + "name": "archive-upload", + "durationSeconds": 24.609935552 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 4.040316127 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 176.197660539 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 21.729738135 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "strategy": "runtime-local", + "duration_seconds": 86.61581134796143, + "logical_bytes": 634552186, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 21.875078136 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 63.846415445 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000004", + "strategy": "direct-runtime", + "duration_seconds": 85.72192597389221, + "logical_bytes": null, + "network_bytes": 144915686, + "relay_server_network_bytes": 0, + "archive_bytes": 144915686, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.3587470010000002 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 6.582292337 + }, + { + "name": "peer-private-upload", + "durationSeconds": 4.040368252 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 4.847127252 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 21.875078136 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 63.846415445 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "strategy": "direct-runtime", + "duration_seconds": 447.05177545547485, + "logical_bytes": 627864822, + "network_bytes": 363111632, + "relay_server_network_bytes": 0, + "archive_bytes": 363111632, + "files": 73, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 12.736953714 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 289.265290049 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 50.339275231 + }, + { + "name": "peer-private-upload", + "durationSeconds": 62.74233478 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 8.950516921 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 426.913107196 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 20.136638884 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000020", + "strategy": "runtime-local", + "duration_seconds": 450.92823028564453, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 426.913107196 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 20.136638884 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "strategy": "direct-runtime", + "duration_seconds": 445.9578585624695, + "logical_bytes": 627864822, + "network_bytes": 363111632, + "relay_server_network_bytes": 0, + "archive_bytes": 363111632, + "files": 73, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 12.500091714 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 289.767463508 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 50.339177523 + }, + { + "name": "peer-private-upload", + "durationSeconds": 61.247130862 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 8.99675367 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 426.329521362 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.626282426 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000021", + "strategy": "runtime-local", + "duration_seconds": 448.97535967826843, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 426.329521362 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.626282426 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "strategy": "direct-runtime", + "duration_seconds": 450.80903339385986, + "logical_bytes": 627864822, + "network_bytes": 363111632, + "relay_server_network_bytes": 0, + "archive_bytes": 363111632, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 5.747049585 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 328.018076776 + }, + { + "name": "peer-private-upload", + "durationSeconds": 98.32883392 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 5.661688502 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 446.630253996 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 4.176778294 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000022", + "strategy": "runtime-local", + "duration_seconds": 451.7024304866791, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 446.630253996 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 4.176778294 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "strategy": "direct-runtime", + "duration_seconds": 447.05297017097473, + "logical_bytes": 627864822, + "network_bytes": 363111632, + "relay_server_network_bytes": 0, + "archive_bytes": 363111632, + "files": 73, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 6.504088211 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 283.261264505 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 50.339336731 + }, + { + "name": "peer-private-upload", + "durationSeconds": 65.826755406 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 10.626078755 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 427.158639905 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.892257176 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000023", + "strategy": "direct-runtime", + "duration_seconds": 447.05300521850586, + "logical_bytes": null, + "network_bytes": 143205735, + "relay_server_network_bytes": 0, + "archive_bytes": 143205735, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.391981043 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 6.54429567 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.811567085 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 3.844895335 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 427.158639905 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.892257176 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "strategy": "gateway", + "duration_seconds": 179.1990737915039, + "logical_bytes": null, + "network_bytes": 735856246, + "relay_server_network_bytes": 735856246, + "archive_bytes": 367883152, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 2.1416e-05, + "rsync_execution_seconds": 109.214267425, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 30.418417181 + }, + { + "name": "archive-validation", + "durationSeconds": 3.350000627 + }, + { + "name": "archive-upload", + "durationSeconds": 109.214450842 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 10.328649046 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 155.425048863 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 23.773206261 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000039", + "strategy": "direct-runtime", + "duration_seconds": 181.24683332443237, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 155.425048863 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 23.773206261 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "strategy": "direct-runtime", + "duration_seconds": 63.02906584739685, + "logical_bytes": 633054582, + "network_bytes": 367883152, + "relay_server_network_bytes": 0, + "archive_bytes": 367883152, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.750968752 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 22.237315094 + }, + { + "name": "peer-private-upload", + "durationSeconds": 6.156749336 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 10.404075629 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 43.290751521 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.738239801 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000040", + "strategy": "direct-runtime", + "duration_seconds": 63.02917814254761, + "logical_bytes": null, + "network_bytes": 143242585, + "relay_server_network_bytes": 0, + "archive_bytes": 143242585, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.770040918 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 10.644484922 + }, + { + "name": "peer-private-upload", + "durationSeconds": 5.459716461 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 2.919044919 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 43.290751521 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.738239801 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "strategy": "direct-runtime", + "duration_seconds": 63.27666711807251, + "logical_bytes": 633054582, + "network_bytes": 367883152, + "relay_server_network_bytes": 0, + "archive_bytes": 367883152, + "files": 73, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 13.273675548 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 6.185884962 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 0.826967876 + }, + { + "name": "peer-private-upload", + "durationSeconds": 8.032206503 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 10.848455797 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 43.474779104 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.801646759 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000041", + "strategy": "runtime-local", + "duration_seconds": 66.85424399375916, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 43.474779104 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.801646759 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "strategy": "direct-runtime", + "duration_seconds": 216.37109184265137, + "logical_bytes": 633054582, + "network_bytes": 367883152, + "relay_server_network_bytes": 0, + "archive_bytes": 367883152, + "files": 73, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 5.653891627 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 3.5541e-05 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 0.82791425 + }, + { + "name": "peer-private-upload", + "durationSeconds": 7.4791347120000005 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 61.750358403999996 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 78.157551328 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 138.21304698 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000042", + "strategy": "gateway", + "duration_seconds": 216.37158632278442, + "logical_bytes": null, + "network_bytes": 108668684, + "relay_server_network_bytes": 108668684, + "archive_bytes": 54327647, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 8.667e-06, + "rsync_execution_seconds": 7.012489586, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 11.220321047 + }, + { + "name": "archive-validation", + "durationSeconds": 0.502052333 + }, + { + "name": "archive-upload", + "durationSeconds": 7.012597503 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 2.265445085 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 78.157551328 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 138.21304698 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "strategy": "gateway", + "duration_seconds": 123.7168481349945, + "logical_bytes": 634551910, + "network_bytes": 739326469, + "relay_server_network_bytes": 739326469, + "archive_bytes": 369618052, + "files": 72, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 1.475e-05, + "rsync_execution_seconds": 33.194828307, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 62.59573207 + }, + { + "name": "archive-validation", + "durationSeconds": 3.845870169 + }, + { + "name": "archive-upload", + "durationSeconds": 33.195026598 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 6.438998877 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 106.968484673 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 16.747609883 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "strategy": "direct-runtime", + "duration_seconds": 306.39686155319214, + "logical_bytes": 627864546, + "network_bytes": 363111387, + "relay_server_network_bytes": 0, + "archive_bytes": 363111387, + "files": 72, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.57820496 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 255.545136741 + }, + { + "name": "peer-private-upload", + "durationSeconds": 15.645166464999999 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 12.808673423 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 289.389497839 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.000019174 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "strategy": "direct-runtime", + "duration_seconds": 254.1288230419159, + "logical_bytes": 633054306, + "network_bytes": 367882958, + "relay_server_network_bytes": 0, + "archive_bytes": 367882958, + "files": 72, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.079382876 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 222.943990226 + }, + { + "name": "peer-private-upload", + "durationSeconds": 15.555421632 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 7.189723295 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 250.521262947 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 3.603590835 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000005", + "strategy": "gateway", + "duration_seconds": 306.98727464675903, + "logical_bytes": null, + "network_bytes": 426426241, + "relay_server_network_bytes": 426426241, + "archive_bytes": 213187034, + "files": 52, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 1.4167e-05, + "rsync_execution_seconds": 171.383548828, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 17.5730373 + }, + { + "name": "archive-validation", + "durationSeconds": 1.798117542 + }, + { + "name": "archive-upload", + "durationSeconds": 171.383783244 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 9.791392046 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 255.813322575 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 51.168297815 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000006", + "strategy": "gateway", + "duration_seconds": 306.98746943473816, + "logical_bytes": null, + "network_bytes": 429244852, + "relay_server_network_bytes": 429244852, + "archive_bytes": 214596167, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 2.0208e-05, + "rsync_execution_seconds": 44.844207688, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 199.5754098 + }, + { + "name": "archive-validation", + "durationSeconds": 1.862231376 + }, + { + "name": "archive-upload", + "durationSeconds": 44.844450521 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 7.420635628 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 255.813322575 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 51.168297815 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000007", + "strategy": "gateway", + "duration_seconds": 306.98651814460754, + "logical_bytes": null, + "network_bytes": 405449391, + "relay_server_network_bytes": 405449391, + "archive_bytes": 202699889, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 6.959e-06, + "rsync_execution_seconds": 166.405013284, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 15.812291341 + }, + { + "name": "archive-validation", + "durationSeconds": 1.670531668 + }, + { + "name": "archive-upload", + "durationSeconds": 166.405118867 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 6.85216142 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 255.813322575 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 51.168297815 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000008", + "strategy": "gateway", + "duration_seconds": 306.9874219894409, + "logical_bytes": null, + "network_bytes": 407423115, + "relay_server_network_bytes": 407423115, + "archive_bytes": 203686627, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 8.75e-06, + "rsync_execution_seconds": 168.683223451, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 16.422759091 + }, + { + "name": "archive-validation", + "durationSeconds": 1.7085479590000001 + }, + { + "name": "archive-upload", + "durationSeconds": 168.68332016 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 7.57468617 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 255.813322575 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 51.168297815 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009", + "strategy": "direct-runtime", + "duration_seconds": 310.1767842769623, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 255.813322575 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 51.168297815 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000010", + "strategy": "gateway", + "duration_seconds": 306.9877817630768, + "logical_bytes": null, + "network_bytes": 419888998, + "relay_server_network_bytes": 419888998, + "archive_bytes": 209918808, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 3.1375e-05, + "rsync_execution_seconds": 166.27197891, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 24.971724928 + }, + { + "name": "archive-validation", + "durationSeconds": 1.789384418 + }, + { + "name": "archive-upload", + "durationSeconds": 166.272189243 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 8.116375962 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 255.813322575 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 51.168297815 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000024", + "strategy": "direct-runtime", + "duration_seconds": 208.68960785865784, + "logical_bytes": 331797673, + "network_bytes": 210219373, + "relay_server_network_bytes": 0, + "archive_bytes": 210219373, + "files": 52, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.512963959 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 12.391321881 + }, + { + "name": "peer-private-upload", + "durationSeconds": 5.912847461 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 14.827411048 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 64.370666196 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 144.315873941 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000025", + "strategy": "direct-runtime", + "duration_seconds": 208.69022679328918, + "logical_bytes": 136166688, + "network_bytes": 211360503, + "relay_server_network_bytes": 0, + "archive_bytes": 211360503, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.471483377 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 17.809785134 + }, + { + "name": "peer-private-upload", + "durationSeconds": 8.428078378 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 10.833673796 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 64.370666196 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 144.315873941 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000026", + "strategy": "direct-runtime", + "duration_seconds": 208.68938875198364, + "logical_bytes": 114215320, + "network_bytes": 199439603, + "relay_server_network_bytes": 0, + "archive_bytes": 199439603, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.491786543 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 47.297451689 + }, + { + "name": "peer-private-upload", + "durationSeconds": 5.969831627 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 6.8825074619999995 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 64.370666196 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 144.315873941 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000027", + "strategy": "runtime-local", + "duration_seconds": 212.1812081336975, + "logical_bytes": 9175961, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 64.370666196 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 144.315873941 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000028", + "strategy": "direct-runtime", + "duration_seconds": 208.6896357536316, + "logical_bytes": 18271004, + "network_bytes": 205275229, + "relay_server_network_bytes": 0, + "archive_bytes": 205275229, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.421861376 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 12.413380672 + }, + { + "name": "peer-private-upload", + "durationSeconds": 5.941380336 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 13.883482048 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 64.370666196 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 144.315873941 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000029", + "strategy": "direct-runtime", + "duration_seconds": 208.6904113292694, + "logical_bytes": 18236443, + "network_bytes": 206507103, + "relay_server_network_bytes": 0, + "archive_bytes": 206507103, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.433204418 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 9.961742588 + }, + { + "name": "peer-private-upload", + "durationSeconds": 4.200545252 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 11.77177813 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 64.370666196 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 144.315873941 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000043", + "strategy": "runtime-local", + "duration_seconds": 244.90794491767883, + "logical_bytes": 334124710, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 52, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 130.021792976 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 114.001598677 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000044", + "strategy": "direct-runtime", + "duration_seconds": 244.024028301239, + "logical_bytes": 137623968, + "network_bytes": 213395231, + "relay_server_network_bytes": 0, + "archive_bytes": 213395231, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.373935334 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 13.01958534 + }, + { + "name": "peer-private-upload", + "durationSeconds": 5.67773971 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 8.432273879 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 130.021792976 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 114.001598677 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045", + "strategy": "gateway", + "duration_seconds": 244.0240502357483, + "logical_bytes": 115252123, + "network_bytes": 222728466, + "relay_server_network_bytes": 222728466, + "archive_bytes": 111350574, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 6.75e-06, + "rsync_execution_seconds": 10.661502838, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 31.049852556 + }, + { + "name": "archive-validation", + "durationSeconds": 1.240911 + }, + { + "name": "archive-upload", + "durationSeconds": 10.661592213 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 4.455706627 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 130.021792976 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 114.001598677 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000046", + "strategy": "direct-runtime", + "duration_seconds": 244.02390885353088, + "logical_bytes": 9256599, + "network_bytes": 202642890, + "relay_server_network_bytes": 0, + "archive_bytes": 202642890, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.426779668 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 16.688241382 + }, + { + "name": "peer-private-upload", + "durationSeconds": 6.671223295 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 8.667050754 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 130.021792976 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 114.001598677 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000047", + "strategy": "direct-runtime", + "duration_seconds": 244.0238709449768, + "logical_bytes": 18403484, + "network_bytes": 207576101, + "relay_server_network_bytes": 0, + "archive_bytes": 207576101, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.371296751 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 43.19736202 + }, + { + "name": "peer-private-upload", + "durationSeconds": 18.289526925 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 64.446825238 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 130.021792976 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 114.001598677 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000048", + "strategy": "gateway", + "duration_seconds": 244.02444171905518, + "logical_bytes": 18391965, + "network_bytes": 417839293, + "relay_server_network_bytes": 417839293, + "archive_bytes": 208894084, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 9.708e-06, + "rsync_execution_seconds": 25.664746137, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 42.532719728000004 + }, + { + "name": "archive-validation", + "durationSeconds": 1.732665959 + }, + { + "name": "archive-upload", + "durationSeconds": 25.664828721 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 50.496567815 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 130.021792976 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 114.001598677 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000005", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000001", + "strategy": "direct-runtime", + "duration_seconds": 115.86401224136353, + "logical_bytes": 196938502, + "network_bytes": 142917128, + "relay_server_network_bytes": 0, + "archive_bytes": 142917128, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.366805418 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 7.339799337 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.649469251 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 3.824633835 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 106.95893384 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.904917379 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000005", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000002", + "strategy": "direct-runtime", + "duration_seconds": 115.86419367790222, + "logical_bytes": 106346880, + "network_bytes": 143696217, + "relay_server_network_bytes": 0, + "archive_bytes": 143696217, + "files": 2, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.365661376 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 53.910338858 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 5.086503753 + }, + { + "name": "peer-private-upload", + "durationSeconds": 38.94090685 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 3.813294752 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 106.95893384 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.904917379 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000006", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000001", + "strategy": "direct-runtime", + "duration_seconds": 33.24743843078613, + "logical_bytes": 196938502, + "network_bytes": 142917128, + "relay_server_network_bytes": 0, + "archive_bytes": 142917128, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.31964421 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 6.67303617 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.527068709 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 5.698039086 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 24.103265927 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 9.144619588 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000006", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000003", + "strategy": "direct-runtime", + "duration_seconds": 33.2477343082428, + "logical_bytes": 106346880, + "network_bytes": 145087119, + "relay_server_network_bytes": 0, + "archive_bytes": 145087119, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.407427584 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 10.739332963 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.98967721 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 5.221601128 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 24.103265927 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 9.144619588 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000007", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000001", + "strategy": "direct-runtime", + "duration_seconds": 132.95035552978516, + "logical_bytes": 196938502, + "network_bytes": 142917128, + "relay_server_network_bytes": 0, + "archive_bytes": 142917128, + "files": 47, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.34048771 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 4.5458e-05 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 0.502257584 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.486318002 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 2.847007043 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 126.483640142 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 6.466507794 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000007", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000004", + "strategy": "direct-runtime", + "duration_seconds": 132.95039653778076, + "logical_bytes": 106346880, + "network_bytes": 144915686, + "relay_server_network_bytes": 0, + "archive_bytes": 144915686, + "files": 2, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 7.200342712 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 76.250675285 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 3.270092252 + }, + { + "name": "peer-private-upload", + "durationSeconds": 29.721611305 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 3.121320752 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 126.483640142 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 6.466507794 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000008", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000002", + "strategy": "direct-runtime", + "duration_seconds": 101.02718091011047, + "logical_bytes": 196898182, + "network_bytes": 143696217, + "relay_server_network_bytes": 0, + "archive_bytes": 143696217, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.366446667 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 54.301691817 + }, + { + "name": "peer-private-upload", + "durationSeconds": 23.538814552 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 4.238682252 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 94.855657501 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 6.171168002 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000008", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000003", + "strategy": "runtime-local", + "duration_seconds": 104.50667762756348, + "logical_bytes": 106346880, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 94.855657501 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 6.171168002 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000002", + "strategy": "gateway", + "duration_seconds": 150.87144494056702, + "logical_bytes": null, + "network_bytes": 287427648, + "relay_server_network_bytes": 287427648, + "archive_bytes": 143696217, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 4.3083e-05, + "rsync_execution_seconds": 34.907802266, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 69.980844157 + }, + { + "name": "archive-validation", + "durationSeconds": 1.150031834 + }, + { + "name": "archive-upload", + "durationSeconds": 34.908253474 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 4.214483544 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 139.913326481 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 10.95796063 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000004", + "strategy": "gateway", + "duration_seconds": 150.8714725971222, + "logical_bytes": null, + "network_bytes": 289866881, + "relay_server_network_bytes": 289866881, + "archive_bytes": 144915686, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 1.7042e-05, + "rsync_execution_seconds": 30.852412014, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 101.804000255 + }, + { + "name": "archive-validation", + "durationSeconds": 1.133418042 + }, + { + "name": "archive-upload", + "durationSeconds": 30.852555015 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 3.683395543 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 139.913326481 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 10.95796063 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000010", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000003", + "strategy": "direct-runtime", + "duration_seconds": 135.12014198303223, + "logical_bytes": 196898182, + "network_bytes": 145087119, + "relay_server_network_bytes": 0, + "archive_bytes": 145087119, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.429816627 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 11.910264547 + }, + { + "name": "peer-private-upload", + "durationSeconds": 5.576698128 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 3.6866393349999997 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 127.476058434 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 7.644051086 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000010", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000004", + "strategy": "direct-runtime", + "duration_seconds": 135.12032461166382, + "logical_bytes": 106346880, + "network_bytes": 144915686, + "relay_server_network_bytes": 0, + "archive_bytes": 144915686, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 7.375919003 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 76.490763953 + }, + { + "name": "peer-private-upload", + "durationSeconds": 33.492711891 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 3.5463492519999997 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 127.476058434 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 7.644051086 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000024", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000020", + "strategy": "runtime-local", + "duration_seconds": 27.11520528793335, + "logical_bytes": 195625222, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 19.265075384 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 6.9538165450000005 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000024", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000021", + "strategy": "direct-runtime", + "duration_seconds": 26.21855616569519, + "logical_bytes": 105114240, + "network_bytes": 142278066, + "relay_server_network_bytes": 0, + "archive_bytes": 142278066, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.391780043 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 6.703116877 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.692133835 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 2.726851626 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 19.265075384 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 6.9538165450000005 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000025", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000020", + "strategy": "direct-runtime", + "duration_seconds": 57.49069285392761, + "logical_bytes": 195625222, + "network_bytes": 141375291, + "relay_server_network_bytes": 0, + "archive_bytes": 141375291, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.424164043 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 7.158202754 + }, + { + "name": "peer-private-upload", + "durationSeconds": 4.057500002 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 11.660461297 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 27.057852971 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 30.437160681 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000025", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000022", + "strategy": "direct-runtime", + "duration_seconds": 57.49071145057678, + "logical_bytes": 105027840, + "network_bytes": 143436735, + "relay_server_network_bytes": 0, + "archive_bytes": 143436735, + "files": 2, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.415187085 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 0.000109167 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 0.495965208 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.697935085 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 3.444823502 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 27.057852971 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 30.437160681 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000026", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000020", + "strategy": "direct-runtime", + "duration_seconds": 96.25195264816284, + "logical_bytes": 195625222, + "network_bytes": 141375291, + "relay_server_network_bytes": 0, + "archive_bytes": 141375291, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.267388085 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 6.372838044 + }, + { + "name": "peer-private-upload", + "durationSeconds": 16.573525549 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 35.771849891 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 66.314490698 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 29.937637097 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000026", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000023", + "strategy": "direct-runtime", + "duration_seconds": 96.25233221054077, + "logical_bytes": 105073920, + "network_bytes": 143205735, + "relay_server_network_bytes": 0, + "archive_bytes": 143205735, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.286490792 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 6.519804461 + }, + { + "name": "peer-private-upload", + "durationSeconds": 16.805902341 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 31.210215056 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 66.314490698 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 29.937637097 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000027", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000021", + "strategy": "direct-runtime", + "duration_seconds": 137.54649186134338, + "logical_bytes": 195665542, + "network_bytes": 142278066, + "relay_server_network_bytes": 0, + "archive_bytes": 142278066, + "files": 47, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 5.238544461 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 0.000196 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 0.474361584 + }, + { + "name": "peer-private-upload", + "durationSeconds": 17.487504675 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 67.23080249 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 101.110295047 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 36.436821724 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000027", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000022", + "strategy": "direct-runtime", + "duration_seconds": 137.54758739471436, + "logical_bytes": 105027840, + "network_bytes": 143436735, + "relay_server_network_bytes": 0, + "archive_bytes": 143436735, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 5.245015586 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 6.886743086 + }, + { + "name": "peer-private-upload", + "durationSeconds": 25.567671513 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 59.258530527 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 101.110295047 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 36.436821724 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000028", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000021", + "strategy": "runtime-local", + "duration_seconds": 25.428139448165894, + "logical_bytes": 195665542, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 16.752103425 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 7.780564044 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000028", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000023", + "strategy": "direct-runtime", + "duration_seconds": 24.532938480377197, + "logical_bytes": 105073920, + "network_bytes": 143205735, + "relay_server_network_bytes": 0, + "archive_bytes": 143205735, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.363404585 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 6.590733628 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.399642126 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 2.66509171 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 16.752103425 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 7.780564044 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000029", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000022", + "strategy": "direct-runtime", + "duration_seconds": 37.27013635635376, + "logical_bytes": 195579142, + "network_bytes": 143436735, + "relay_server_network_bytes": 0, + "archive_bytes": 143436735, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 4.352586419 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 15.533482091 + }, + { + "name": "peer-private-upload", + "durationSeconds": 5.817277252 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 5.928769003 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 33.733123931 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 3.536212793 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000029", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000023", + "strategy": "runtime-local", + "duration_seconds": 38.59876275062561, + "logical_bytes": 105073920, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 33.733123931 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 3.536212793 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000043", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000039", + "strategy": "gateway", + "duration_seconds": 55.786330461502075, + "logical_bytes": 106018560, + "network_bytes": 103997679, + "relay_server_network_bytes": 103997679, + "archive_bytes": 51992429, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.00033975, + "rsync_execution_seconds": 6.096574127, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 39.587198226 + }, + { + "name": "archive-validation", + "durationSeconds": 0.538669833 + }, + { + "name": "archive-upload", + "durationSeconds": 6.097390668 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.628090501 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 48.736417897 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 7.049611129 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000043", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000040", + "strategy": "direct-runtime", + "duration_seconds": 55.78641390800476, + "logical_bytes": 196610182, + "network_bytes": 143242585, + "relay_server_network_bytes": 0, + "archive_bytes": 143242585, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.308783501 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 5.576019003 + }, + { + "name": "peer-private-upload", + "durationSeconds": 4.017818627 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 4.068347376 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 48.736417897 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 7.049611129 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000044", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000039", + "strategy": "gateway", + "duration_seconds": 125.47219014167786, + "logical_bytes": 106018560, + "network_bytes": 103997679, + "relay_server_network_bytes": 103997679, + "archive_bytes": 51992429, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 8.042e-06, + "rsync_execution_seconds": 5.755920378, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 13.888867964 + }, + { + "name": "archive-validation", + "durationSeconds": 0.528902667 + }, + { + "name": "archive-upload", + "durationSeconds": 5.756016586 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.272753501 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 119.640673846 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 5.830948001 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000044", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000041", + "strategy": "direct-runtime", + "duration_seconds": 125.47228455543518, + "logical_bytes": 196610182, + "network_bytes": 144488549, + "relay_server_network_bytes": 0, + "archive_bytes": 144488549, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 5.49335071 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 101.535130671 + }, + { + "name": "peer-private-upload", + "durationSeconds": 7.240543045 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 3.007121543 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 119.640673846 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 5.830948001 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000039", + "strategy": "direct-runtime", + "duration_seconds": 8.6780686378479, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 4.3208e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 6.511636503 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000042", + "strategy": "direct-runtime", + "duration_seconds": 8.6780686378479, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 4.3208e-05 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 6.511636503 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000046", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000040", + "strategy": "direct-runtime", + "duration_seconds": 119.90132546424866, + "logical_bytes": 196610182, + "network_bytes": 143242585, + "relay_server_network_bytes": 0, + "archive_bytes": 143242585, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 4.754082294 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 6.812805086 + }, + { + "name": "peer-private-upload", + "durationSeconds": 18.612074634 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 45.524928855 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 81.556893579 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 38.347655976 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000046", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000041", + "strategy": "runtime-local", + "duration_seconds": 121.18112564086914, + "logical_bytes": 106058880, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 81.556893579 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 38.347655976 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000047", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000040", + "strategy": "direct-runtime", + "duration_seconds": 114.06439328193665, + "logical_bytes": 196610182, + "network_bytes": 143242585, + "relay_server_network_bytes": 0, + "archive_bytes": 143242585, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 5.499579877 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 6.813983045 + }, + { + "name": "peer-private-upload", + "durationSeconds": 16.446160507 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 36.715309016 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 92.555965667 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 21.509046051 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000047", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000042", + "strategy": "gateway", + "duration_seconds": 114.06535649299622, + "logical_bytes": 106018560, + "network_bytes": 108668683, + "relay_server_network_bytes": 108668683, + "archive_bytes": 54327647, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 2.5292e-05, + "rsync_execution_seconds": 51.017637481, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 11.898343631 + }, + { + "name": "archive-validation", + "durationSeconds": 0.545374209 + }, + { + "name": "archive-upload", + "durationSeconds": 51.017912939 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 21.742268344 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 92.555965667 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 21.509046051 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000048", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000041", + "strategy": "gateway", + "duration_seconds": 136.0122423171997, + "logical_bytes": null, + "network_bytes": 289012504, + "relay_server_network_bytes": 289012504, + "archive_bytes": 144488549, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 1.225e-05, + "rsync_execution_seconds": 36.136521766, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 85.161567582 + }, + { + "name": "archive-validation", + "durationSeconds": 1.173096001 + }, + { + "name": "archive-upload", + "durationSeconds": 36.13663585 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 3.901508419 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 128.536866059 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 7.47857792 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000048", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000042", + "strategy": "direct-runtime", + "duration_seconds": 138.1003017425537, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 128.536866059 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 7.47857792 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013", + "strategy": "direct-runtime", + "duration_seconds": 409.974862575531, + "logical_bytes": 740939386, + "network_bytes": 426325775, + "relay_server_network_bytes": 0, + "archive_bytes": 426325775, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.380699376 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 23.889355677 + }, + { + "name": "peer-private-upload", + "durationSeconds": 6.88951767 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 140.56709048 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 278.768292128 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 131.204556309 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014", + "strategy": "direct-runtime", + "duration_seconds": 409.9745624065399, + "logical_bytes": 106346880, + "network_bytes": 426222183, + "relay_server_network_bytes": 0, + "archive_bytes": 426222183, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.474778918 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 27.043984721 + }, + { + "name": "peer-private-upload", + "durationSeconds": 33.526433141 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 136.21276277 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 278.768292128 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 131.204556309 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "strategy": "gateway", + "duration_seconds": 409.97449827194214, + "logical_bytes": 106346880, + "network_bytes": 852807399, + "relay_server_network_bytes": 852807399, + "archive_bytes": 426351589, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 1.3708e-05, + "rsync_execution_seconds": 54.438554525, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 102.588095879 + }, + { + "name": "archive-validation", + "durationSeconds": 3.784318668 + }, + { + "name": "archive-upload", + "durationSeconds": 54.438743025 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 48.30779123 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 278.768292128 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 131.204556309 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016", + "strategy": "direct-runtime", + "duration_seconds": 409.9749310016632, + "logical_bytes": 106346880, + "network_bytes": 426294585, + "relay_server_network_bytes": 0, + "archive_bytes": 426294585, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.3998750429999998 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 28.190190513 + }, + { + "name": "peer-private-upload", + "durationSeconds": 50.630167607 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 131.954575936 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 278.768292128 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 131.204556309 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032", + "strategy": "direct-runtime", + "duration_seconds": 439.53393745422363, + "logical_bytes": 732938742, + "network_bytes": 419365583, + "relay_server_network_bytes": 0, + "archive_bytes": 419365583, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.444578959 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 27.464351179 + }, + { + "name": "peer-private-upload", + "durationSeconds": 51.954862982 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 45.158113229 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 386.502753759 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 53.028995149 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033", + "strategy": "direct-runtime", + "duration_seconds": 439.53312063217163, + "logical_bytes": 105114240, + "network_bytes": 418940522, + "relay_server_network_bytes": 0, + "archive_bytes": 418940522, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.437618543 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 25.897815886 + }, + { + "name": "peer-private-upload", + "durationSeconds": 13.714479215 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 17.109355217 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 386.502753759 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 53.028995149 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034", + "strategy": "direct-runtime", + "duration_seconds": 439.5331792831421, + "logical_bytes": 105027840, + "network_bytes": 419212118, + "relay_server_network_bytes": 0, + "archive_bytes": 419212118, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.41725496 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 23.116924051 + }, + { + "name": "peer-private-upload", + "durationSeconds": 6.48005267 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 16.883390341 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 386.502753759 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 53.028995149 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035", + "strategy": "direct-runtime", + "duration_seconds": 439.53410816192627, + "logical_bytes": 105073920, + "network_bytes": 418994616, + "relay_server_network_bytes": 0, + "archive_bytes": 418994616, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.632721252 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 334.301800736 + }, + { + "name": "peer-private-upload", + "durationSeconds": 34.879486223 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 12.931198089 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 386.502753759 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 53.028995149 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "strategy": "gateway", + "duration_seconds": 423.51522946357727, + "logical_bytes": 739073142, + "network_bytes": 848057753, + "relay_server_network_bytes": 848057753, + "archive_bytes": 423977058, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 6.958e-06, + "rsync_execution_seconds": 35.442528557, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 101.971623338 + }, + { + "name": "archive-validation", + "durationSeconds": 3.819012544 + }, + { + "name": "archive-upload", + "durationSeconds": 35.442611224 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 7.516678671 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 371.155632002 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 52.357549732 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052", + "strategy": "direct-runtime", + "duration_seconds": 423.515261888504, + "logical_bytes": 106058880, + "network_bytes": 423742384, + "relay_server_network_bytes": 0, + "archive_bytes": 423742384, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.332714751 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 27.017575429 + }, + { + "name": "peer-private-upload", + "durationSeconds": 9.582688712 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 12.823375881 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 371.155632002 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 52.357549732 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053", + "strategy": "direct-runtime", + "duration_seconds": 423.5150535106659, + "logical_bytes": 106058880, + "network_bytes": 423976160, + "relay_server_network_bytes": 0, + "archive_bytes": 423976160, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.381869918 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 316.872213269 + }, + { + "name": "peer-private-upload", + "durationSeconds": 37.744174892 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 12.147973631 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 371.155632002 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 52.357549732 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054", + "strategy": "direct-runtime", + "duration_seconds": 423.51526927948, + "logical_bytes": 106018560, + "network_bytes": 423789984, + "relay_server_network_bytes": 0, + "archive_bytes": 423789984, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.380404626 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 177.203873705 + }, + { + "name": "peer-private-upload", + "durationSeconds": 21.295400718 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 11.881498964 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 371.155632002 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 52.357549732 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000019", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "strategy": "direct-runtime", + "duration_seconds": 468.19441294670105, + "logical_bytes": 1111828392, + "network_bytes": 624504559, + "relay_server_network_bytes": 0, + "archive_bytes": 624504559, + "files": 84, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.358384792 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 361.993204622 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 54.534579608 + }, + { + "name": "peer-private-upload", + "durationSeconds": 27.623296388 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 12.127251798 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 460.939372668 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 7.251629086 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000038", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "strategy": "direct-runtime", + "duration_seconds": 157.68661236763, + "logical_bytes": 1100003102, + "network_bytes": 615130007, + "relay_server_network_bytes": 0, + "archive_bytes": 615130007, + "files": 84, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.282488667 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 80.93315387 + }, + { + "name": "peer-private-upload", + "durationSeconds": 24.076353261 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 18.331483092 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 127.35759385 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 30.327887263 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000057", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "strategy": "direct-runtime", + "duration_seconds": 495.8601224422455, + "logical_bytes": 1109057816, + "network_bytes": 619720534, + "relay_server_network_bytes": 0, + "archive_bytes": 619720534, + "files": 84, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.412345375 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 382.950141091 + }, + { + "name": "peer-private-upload", + "durationSeconds": 31.382413639 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 22.603871136 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 441.38265795 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 54.465003691 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000058", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "strategy": "direct-runtime", + "duration_seconds": 486.8264842033386, + "logical_bytes": 1111828392, + "network_bytes": 624504559, + "relay_server_network_bytes": 0, + "archive_bytes": 624504559, + "files": 84, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.334044752 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 362.706896956 + }, + { + "name": "peer-private-upload", + "durationSeconds": 54.641161483 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 11.049143505 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 433.703607531 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 53.119289442 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000058", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "strategy": "direct-runtime", + "duration_seconds": 486.8264443874359, + "logical_bytes": 1009451800, + "network_bytes": 615130007, + "relay_server_network_bytes": 0, + "archive_bytes": 615130007, + "files": 39, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.238551043 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 42.660181978 + }, + { + "name": "peer-private-upload", + "durationSeconds": 24.632780803 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 10.862616296 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 433.703607531 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 53.119289442 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000058", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "strategy": "direct-runtime", + "duration_seconds": 486.8266372680664, + "logical_bytes": 1018506514, + "network_bytes": 619720534, + "relay_server_network_bytes": 0, + "archive_bytes": 619720534, + "files": 39, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.329883252 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 7.5167e-05 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 8.123189962 + }, + { + "name": "peer-private-upload", + "durationSeconds": 7.566221212 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 11.679193797 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 433.703607531 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 53.119289442 + } + ] + } + ], + "shared_preparations": [ + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "barrier_seconds": 762.31935372, + "merge_seconds": 236.587417607, + "merge_check_seconds": 145.75149784399946, + "merge_copy_seconds": 53.56884224699934, + "merge_verify_seconds": 35.92816998099988 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "barrier_seconds": 804.122080071, + "merge_seconds": 66.450399947, + "merge_check_seconds": 25.46066779800003, + "merge_copy_seconds": 33.81956537900078, + "merge_verify_seconds": 6.668966102999548 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "barrier_seconds": 364.322760332, + "merge_seconds": 243.573326361, + "merge_check_seconds": 142.49716386900036, + "merge_copy_seconds": 63.3139910280006, + "merge_verify_seconds": 37.1964132789999 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013", + "barrier_seconds": 127.879948059, + "merge_seconds": 5.494888086, + "merge_check_seconds": 1.7062233350006863, + "merge_copy_seconds": 2.787949521999508, + "merge_verify_seconds": 0.5123015320004924 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014", + "barrier_seconds": 120.300814096, + "merge_seconds": 19.955273884, + "merge_check_seconds": 4.951173943999493, + "merge_copy_seconds": 10.93475685500016, + "merge_verify_seconds": 3.5650197659997502 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "barrier_seconds": 176.197660539, + "merge_seconds": 21.729738135, + "merge_check_seconds": 8.317466377280653, + "merge_copy_seconds": 4.891942439600825, + "merge_verify_seconds": 7.56159333884716 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016", + "barrier_seconds": 21.875078136, + "merge_seconds": 63.846415445, + "merge_check_seconds": 9.770585694000147, + "merge_copy_seconds": 40.23239413300007, + "merge_verify_seconds": 13.27519343699987 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032", + "barrier_seconds": 426.913107196, + "merge_seconds": 20.136638884, + "merge_check_seconds": 4.7745690230003675, + "merge_copy_seconds": 10.818428671999754, + "merge_verify_seconds": 3.989548990000003 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033", + "barrier_seconds": 426.329521362, + "merge_seconds": 19.626282426, + "merge_check_seconds": 4.873323572000118, + "merge_copy_seconds": 10.523049770000398, + "merge_verify_seconds": 3.7433378400000947 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034", + "barrier_seconds": 446.630253996, + "merge_seconds": 4.176778294, + "merge_check_seconds": 1.6326302810002744, + "merge_copy_seconds": 1.4865193229998113, + "merge_verify_seconds": 0.5032371529996453 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035", + "barrier_seconds": 427.158639905, + "merge_seconds": 19.892257176, + "merge_check_seconds": 5.0070255440004985, + "merge_copy_seconds": 10.357501779999438, + "merge_verify_seconds": 3.9768748630003756 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "barrier_seconds": 155.425048863, + "merge_seconds": 23.773206261, + "merge_check_seconds": 10.40847041644156, + "merge_copy_seconds": 5.385399823077023, + "merge_verify_seconds": 7.006542180664837 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052", + "barrier_seconds": 43.290751521, + "merge_seconds": 19.738239801, + "merge_check_seconds": 5.041067892999308, + "merge_copy_seconds": 10.515979834999598, + "merge_verify_seconds": 3.67229672799931 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053", + "barrier_seconds": 43.474779104, + "merge_seconds": 19.801646759, + "merge_check_seconds": 5.162643724999725, + "merge_copy_seconds": 10.32359564699982, + "merge_verify_seconds": 3.7847263240000757 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054", + "barrier_seconds": 78.157551328, + "merge_seconds": 138.21304698, + "merge_check_seconds": 53.48430508499996, + "merge_copy_seconds": 38.488857527999926, + "merge_verify_seconds": 44.77779699300004 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "barrier_seconds": 106.968484673, + "merge_seconds": 16.747609883, + "merge_check_seconds": 2.698455793999983, + "merge_copy_seconds": 9.697547818999738, + "merge_verify_seconds": 3.758709139000075 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "barrier_seconds": 289.389497839, + "merge_seconds": 17.000019174, + "merge_check_seconds": 3.825055442000121, + "merge_copy_seconds": 9.022188792999941, + "merge_verify_seconds": 3.6229606209999474 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "barrier_seconds": 250.521262947, + "merge_seconds": 3.603590835, + "merge_check_seconds": 0.5051906540002165, + "merge_copy_seconds": 2.142047676000402, + "merge_verify_seconds": 0.4960838049992162 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "barrier_seconds": 255.813322575, + "merge_seconds": 51.168297815, + "merge_check_seconds": 29.774229486472905, + "merge_copy_seconds": 10.907653647474945, + "merge_verify_seconds": 9.319769462570548 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "barrier_seconds": 64.370666196, + "merge_seconds": 144.315873941, + "merge_check_seconds": 66.03216088599993, + "merge_copy_seconds": 40.50987513599989, + "merge_verify_seconds": 36.49297974000001 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "barrier_seconds": 130.021792976, + "merge_seconds": 114.001598677, + "merge_check_seconds": 64.67154081999979, + "merge_copy_seconds": 26.555731558999923, + "merge_verify_seconds": 22.22765749600012 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000005", + "barrier_seconds": 106.95893384, + "merge_seconds": 8.904917379, + "merge_check_seconds": 2.5484010519999174, + "merge_copy_seconds": 4.098082534000014, + "merge_verify_seconds": 1.711104073999877 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000006", + "barrier_seconds": 24.103265927, + "merge_seconds": 9.144619588, + "merge_check_seconds": 2.620249063000003, + "merge_copy_seconds": 4.151263733999997, + "merge_verify_seconds": 1.8435620770000014 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000007", + "barrier_seconds": 126.483640142, + "merge_seconds": 6.466507794, + "merge_check_seconds": 1.1408798139998453, + "merge_copy_seconds": 3.3527238600001965, + "merge_verify_seconds": 1.4621625760000825 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000008", + "barrier_seconds": 94.855657501, + "merge_seconds": 6.171168002, + "merge_check_seconds": 1.549914395000087, + "merge_copy_seconds": 2.378768721999677, + "merge_verify_seconds": 1.6883353930002158 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009", + "barrier_seconds": 139.913326481, + "merge_seconds": 10.95796063, + "merge_check_seconds": 3.6601696284487844, + "merge_copy_seconds": 2.483007412403822, + "merge_verify_seconds": 2.644647597335279 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000010", + "barrier_seconds": 127.476058434, + "merge_seconds": 7.644051086, + "merge_check_seconds": 2.4743183250002403, + "merge_copy_seconds": 3.054641808999804, + "merge_verify_seconds": 1.59523130999969 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000024", + "barrier_seconds": 19.265075384, + "merge_seconds": 6.9538165450000005, + "merge_check_seconds": 1.7071456700000454, + "merge_copy_seconds": 3.620273917000304, + "merge_verify_seconds": 1.1361159879998013 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000025", + "barrier_seconds": 27.057852971, + "merge_seconds": 30.437160681, + "merge_check_seconds": 11.184759001000003, + "merge_copy_seconds": 8.962502901000008, + "merge_verify_seconds": 9.561906169000011 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000026", + "barrier_seconds": 66.314490698, + "merge_seconds": 29.937637097, + "merge_check_seconds": 8.075572075999844, + "merge_copy_seconds": 13.515580894999857, + "merge_verify_seconds": 4.983909885999765 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000027", + "barrier_seconds": 101.110295047, + "merge_seconds": 36.436821724, + "merge_check_seconds": 15.438960124000005, + "merge_copy_seconds": 8.145220017999975, + "merge_verify_seconds": 12.343458579000014 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000028", + "barrier_seconds": 16.752103425, + "merge_seconds": 7.780564044, + "merge_check_seconds": 1.9067623900000399, + "merge_copy_seconds": 4.1767056619996765, + "merge_verify_seconds": 1.2138205769997512 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000029", + "barrier_seconds": 33.733123931, + "merge_seconds": 3.536212793, + "merge_check_seconds": 0.7207421489999888, + "merge_copy_seconds": 1.7899585740000248, + "merge_verify_seconds": 0.5052870060000032 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000043", + "barrier_seconds": 48.736417897, + "merge_seconds": 7.049611129, + "merge_check_seconds": 1.684984602, + "merge_copy_seconds": 3.4788350139999977, + "merge_verify_seconds": 1.3676855079999939 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000044", + "barrier_seconds": 119.640673846, + "merge_seconds": 5.830948001, + "merge_check_seconds": 1.1038579400000117, + "merge_copy_seconds": 3.1307526440000117, + "merge_verify_seconds": 1.0764374750000059 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045", + "barrier_seconds": 4.3208e-05, + "merge_seconds": 6.511636503, + "merge_check_seconds": 1.7308137826621532, + "merge_copy_seconds": 1.2903437670320272, + "merge_verify_seconds": 2.5150117529556155 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000046", + "barrier_seconds": 81.556893579, + "merge_seconds": 38.347655976, + "merge_check_seconds": 13.23874066899998, + "merge_copy_seconds": 10.045166034999966, + "merge_verify_seconds": 13.448123979999764 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000047", + "barrier_seconds": 92.555965667, + "merge_seconds": 21.509046051, + "merge_check_seconds": 7.494330901000012, + "merge_copy_seconds": 7.787287515000116, + "merge_verify_seconds": 4.465058804999899 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000048", + "barrier_seconds": 128.536866059, + "merge_seconds": 7.47857792, + "merge_check_seconds": 1.710929460823536, + "merge_copy_seconds": 2.0938274255022407, + "merge_verify_seconds": 2.732031609863043 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "barrier_seconds": 278.768292128, + "merge_seconds": 131.204556309, + "merge_check_seconds": 55.52439420599967, + "merge_copy_seconds": 57.700238978000016, + "merge_verify_seconds": 16.76125427399984 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "barrier_seconds": 386.502753759, + "merge_seconds": 53.028995149, + "merge_check_seconds": 18.695452503000524, + "merge_copy_seconds": 27.288482144000227, + "merge_verify_seconds": 6.4975390220006375 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "barrier_seconds": 371.155632002, + "merge_seconds": 52.357549732, + "merge_check_seconds": 18.724648188000174, + "merge_copy_seconds": 27.0868461150003, + "merge_verify_seconds": 6.045547994999652 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000019", + "barrier_seconds": 460.939372668, + "merge_seconds": 7.251629086, + "merge_check_seconds": 0.8757597130006616, + "merge_copy_seconds": 5.020064412000465, + "merge_verify_seconds": 0.8792642919997888 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000038", + "barrier_seconds": 127.35759385, + "merge_seconds": 30.327887263, + "merge_check_seconds": 6.825928993999696, + "merge_copy_seconds": 16.770350947000225, + "merge_verify_seconds": 6.250859729000695 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000057", + "barrier_seconds": 441.38265795, + "merge_seconds": 54.465003691, + "merge_check_seconds": 7.319902485999592, + "merge_copy_seconds": 17.104031484999723, + "merge_verify_seconds": 29.535274692999337 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000058", + "barrier_seconds": 433.703607531, + "merge_seconds": 53.119289442, + "merge_check_seconds": 8.54865721799979, + "merge_copy_seconds": 30.175133962000473, + "merge_verify_seconds": 13.92480018400056 + } + ], + "activities": [ + { + "activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "resource_id": "montage-gcp-fog-e2-micro-03", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 3.1248689280014332, + "observed_runtime_seconds": 8.437352895736694, + "validated_handle_runtime_seconds": 8.437352895736694, + "terminal_observation_delay_seconds": 1.5530600547790527, + "preparation_elapsed_seconds": 998.9256546497345, + "overlapping_transfer_work_seconds": 4994.624783039093, + "launch_wait_seconds": 40.950018882751465, + "queue_seconds": 1091.0935900211334, + "network_bytes": 2728009480 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "resource_id": "montage-gcp-fog-e2-micro-09", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 1.2850327680016562, + "observed_runtime_seconds": 0.9578084945678711, + "validated_handle_runtime_seconds": 0.9578084945678711, + "terminal_observation_delay_seconds": 1.1367733478546143, + "preparation_elapsed_seconds": 870.5863614082336, + "overlapping_transfer_work_seconds": 4352.928018569946, + "launch_wait_seconds": 20.97837543487549, + "queue_seconds": 892.2211546897888, + "network_bytes": 2263690317 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "resource_id": "montage-gcp-fog-e2-micro-01", + "attempt": 1, + "planned_compute_seconds": 8, + "planned_transfer_seconds": 2.964032768000834, + "observed_runtime_seconds": 8.217379093170166, + "validated_handle_runtime_seconds": 8.217379093170166, + "terminal_observation_delay_seconds": 1.2964222431182861, + "preparation_elapsed_seconds": 607.8989915847778, + "overlapping_transfer_work_seconds": 3039.493859052658, + "launch_wait_seconds": 35.49031400680542, + "queue_seconds": 654.8143384456635, + "network_bytes": 2711405331 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013", + "resource_id": "montage-gcp-e2-medium-2", + "attempt": 1, + "planned_compute_seconds": 1, + "planned_transfer_seconds": 0.0010655360001692316, + "observed_runtime_seconds": 0.6267998218536377, + "validated_handle_runtime_seconds": 0.6267998218536377, + "terminal_observation_delay_seconds": 1.1867237091064453, + "preparation_elapsed_seconds": 136.59450793266296, + "overlapping_transfer_work_seconds": 269.9700200557709, + "launch_wait_seconds": 6.143299579620361, + "queue_seconds": 142.79235744476318, + "network_bytes": 369609876 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 4, + "planned_transfer_seconds": 0.0010655360001692316, + "observed_runtime_seconds": 3.014694929122925, + "validated_handle_runtime_seconds": 3.014694929122925, + "terminal_observation_delay_seconds": 0.6340243816375732, + "preparation_elapsed_seconds": 143.2702453136444, + "overlapping_transfer_work_seconds": 283.52679109573364, + "launch_wait_seconds": 6.416366815567017, + "queue_seconds": 149.77146005630493, + "network_bytes": 369609876 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 1, + "planned_transfer_seconds": 0.6411638399999902, + "observed_runtime_seconds": 3.9223642349243164, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 199.32968091964722, + "overlapping_transfer_work_seconds": 395.8639645576477, + "launch_wait_seconds": 8.174291849136353, + "queue_seconds": 0.19284296035766602, + "network_bytes": 1029519903 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016", + "resource_id": "montage-gcp-fog-e2-micro-04", + "attempt": 1, + "planned_compute_seconds": 4, + "planned_transfer_seconds": 0.29466666666712626, + "observed_runtime_seconds": 13.19267725944519, + "validated_handle_runtime_seconds": 13.19267725944519, + "terminal_observation_delay_seconds": 1.0526487827301025, + "preparation_elapsed_seconds": 86.61581134796143, + "overlapping_transfer_work_seconds": 172.33773732185364, + "launch_wait_seconds": 20.98622727394104, + "queue_seconds": 107.66598224639893, + "network_bytes": 144915686 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032", + "resource_id": "montage-gcp-fog-e2-micro-06", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.0011310720001347363, + "observed_runtime_seconds": 0.9790773391723633, + "validated_handle_runtime_seconds": 0.9790773391723633, + "terminal_observation_delay_seconds": 4.559584856033325, + "preparation_elapsed_seconds": 450.92823028564453, + "overlapping_transfer_work_seconds": 897.9800057411194, + "launch_wait_seconds": 7.266067028045654, + "queue_seconds": 518.0898294448853, + "network_bytes": 363111632 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 100, + "planned_transfer_seconds": 0.0011310720001347363, + "observed_runtime_seconds": 3.0126373767852783, + "validated_handle_runtime_seconds": 3.0126373767852783, + "terminal_observation_delay_seconds": 0.720634937286377, + "preparation_elapsed_seconds": 448.97535967826843, + "overlapping_transfer_work_seconds": 894.9332182407379, + "launch_wait_seconds": 6.433155059814453, + "queue_seconds": 515.3042397499084, + "network_bytes": 363111632 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034", + "resource_id": "montage-gcp-e2-medium-2", + "attempt": 1, + "planned_compute_seconds": 23, + "planned_transfer_seconds": 0.0011310720001347363, + "observed_runtime_seconds": 0.5333530902862549, + "validated_handle_runtime_seconds": 0.5333530902862549, + "terminal_observation_delay_seconds": 1.1956729888916016, + "preparation_elapsed_seconds": 451.7024304866791, + "overlapping_transfer_work_seconds": 902.5114638805389, + "launch_wait_seconds": 5.893486976623535, + "queue_seconds": 489.2012503147125, + "network_bytes": 363111632 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035", + "resource_id": "montage-gcp-fog-e2-micro-08", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.24213107200012018, + "observed_runtime_seconds": 1.0645825862884521, + "validated_handle_runtime_seconds": 1.0645825862884521, + "terminal_observation_delay_seconds": 1.3099606037139893, + "preparation_elapsed_seconds": 447.05300521850586, + "overlapping_transfer_work_seconds": 894.1059753894806, + "launch_wait_seconds": 10.731779336929321, + "queue_seconds": 526.0460546016693, + "network_bytes": 506317367 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "resource_id": "plafrim-execution-ssh-node-diablo03", + "attempt": 1, + "planned_compute_seconds": 16, + "planned_transfer_seconds": 0.080163839999841, + "observed_runtime_seconds": 2.595179796218872, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 181.26472067832947, + "overlapping_transfer_work_seconds": 360.4482822418213, + "launch_wait_seconds": 6.157193183898926, + "queue_seconds": 0.43082404136657715, + "network_bytes": 735856246 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052", + "resource_id": "montage-gcp-fog-e2-micro-09", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.3220983040005194, + "observed_runtime_seconds": 1.3280234336853027, + "validated_handle_runtime_seconds": 1.3280234336853027, + "terminal_observation_delay_seconds": 9.382233142852783, + "preparation_elapsed_seconds": 63.02917814254761, + "overlapping_transfer_work_seconds": 126.05824398994446, + "launch_wait_seconds": 5.465626001358032, + "queue_seconds": 69.12700462341309, + "network_bytes": 511125737 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053", + "resource_id": "montage-gcp-fog-e2-micro-08", + "attempt": 1, + "planned_compute_seconds": 48, + "planned_transfer_seconds": 0.001098304000151984, + "observed_runtime_seconds": 1.157512903213501, + "validated_handle_runtime_seconds": 1.157512903213501, + "terminal_observation_delay_seconds": 3.688835382461548, + "preparation_elapsed_seconds": 66.85424399375916, + "overlapping_transfer_work_seconds": 130.13091111183167, + "launch_wait_seconds": 8.100330591201782, + "queue_seconds": 74.99752974510193, + "network_bytes": 367883152 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054", + "resource_id": "montage-gcp-fog-e2-micro-05", + "attempt": 1, + "planned_compute_seconds": 48, + "planned_transfer_seconds": 0.8810983040000337, + "observed_runtime_seconds": 23.867364168167114, + "validated_handle_runtime_seconds": 23.867364168167114, + "terminal_observation_delay_seconds": 4.798771619796753, + "preparation_elapsed_seconds": 216.37158632278442, + "overlapping_transfer_work_seconds": 432.7426781654358, + "launch_wait_seconds": 70.07824802398682, + "queue_seconds": 312.5580608844757, + "network_bytes": 476551836 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "resource_id": "montage-gcp-fog-e2-micro-04", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.080163839999841, + "observed_runtime_seconds": 0.47063517570495605, + "validated_handle_runtime_seconds": 0.47063517570495605, + "terminal_observation_delay_seconds": 1.2088592052459717, + "preparation_elapsed_seconds": 123.7168481349945, + "overlapping_transfer_work_seconds": 123.7168481349945, + "launch_wait_seconds": 7.855768918991089, + "queue_seconds": 132.15962028503418, + "network_bytes": 739326469 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "resource_id": "montage-gcp-fog-e2-micro-05", + "attempt": 1, + "planned_compute_seconds": 28, + "planned_transfer_seconds": 0.0010327680001864792, + "observed_runtime_seconds": 0.6214001178741455, + "validated_handle_runtime_seconds": 0.6214001178741455, + "terminal_observation_delay_seconds": 1.0639917850494385, + "preparation_elapsed_seconds": 306.39686155319214, + "overlapping_transfer_work_seconds": 306.39686155319214, + "launch_wait_seconds": 7.416016101837158, + "queue_seconds": 314.42007875442505, + "network_bytes": 363111387 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "resource_id": "montage-gcp-e2-medium-1", + "attempt": 1, + "planned_compute_seconds": 20, + "planned_transfer_seconds": 0.0010327680001864792, + "observed_runtime_seconds": 0.5198588371276855, + "validated_handle_runtime_seconds": 0.5198588371276855, + "terminal_observation_delay_seconds": 1.1290433406829834, + "preparation_elapsed_seconds": 254.1288230419159, + "overlapping_transfer_work_seconds": 254.1288230419159, + "launch_wait_seconds": 5.485957384109497, + "queue_seconds": 260.19727993011475, + "network_bytes": 367882958 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 17, + "planned_transfer_seconds": 0.40215015679905264, + "observed_runtime_seconds": 2.8328795433044434, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 310.2043294906616, + "overlapping_transfer_work_seconds": 1845.1149315834045, + "launch_wait_seconds": 17.3840172290802, + "queue_seconds": 0.8335714340209961, + "network_bytes": 2088432597 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "resource_id": "montage-gcp-fog-e2-micro-10", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.005163840000932396, + "observed_runtime_seconds": 4.70743465423584, + "validated_handle_runtime_seconds": 4.70743465423584, + "terminal_observation_delay_seconds": 1.2800743579864502, + "preparation_elapsed_seconds": 212.1812081336975, + "overlapping_transfer_work_seconds": 1255.6304786205292, + "launch_wait_seconds": 81.23420357704163, + "queue_seconds": 293.4360225200653, + "network_bytes": 1032801811 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "resource_id": "montage-gcp-fog-e2-micro-01", + "attempt": 1, + "planned_compute_seconds": 20, + "planned_transfer_seconds": 0.16342598400024144, + "observed_runtime_seconds": 2.559312343597412, + "validated_handle_runtime_seconds": 2.559312343597412, + "terminal_observation_delay_seconds": 2.537781000137329, + "preparation_elapsed_seconds": 244.90794491767883, + "overlapping_transfer_work_seconds": 1465.028244972229, + "launch_wait_seconds": 24.287164211273193, + "queue_seconds": 276.1662187576294, + "network_bytes": 1264181981 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000005", + "resource_id": "montage-gcp-fog-e2-micro-05", + "attempt": 1, + "planned_compute_seconds": 120, + "planned_transfer_seconds": 0.44249999999988177, + "observed_runtime_seconds": 2.1713368892669678, + "validated_handle_runtime_seconds": 2.1713368892669678, + "terminal_observation_delay_seconds": 2.0016393661499023, + "preparation_elapsed_seconds": 115.86419367790222, + "overlapping_transfer_work_seconds": 231.72820591926575, + "launch_wait_seconds": 4.995863676071167, + "queue_seconds": 127.20878100395203, + "network_bytes": 286613345 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000006", + "resource_id": "montage-gcp-fog-e2-micro-10", + "attempt": 1, + "planned_compute_seconds": 112, + "planned_transfer_seconds": 0.44250000000010914, + "observed_runtime_seconds": 31.231823921203613, + "validated_handle_runtime_seconds": 31.231823921203613, + "terminal_observation_delay_seconds": 1.5106003284454346, + "preparation_elapsed_seconds": 33.2477343082428, + "overlapping_transfer_work_seconds": 66.49517273902893, + "launch_wait_seconds": 202.290536403656, + "queue_seconds": 347.0226411819458, + "network_bytes": 288004247 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000007", + "resource_id": "montage-gcp-fog-e2-micro-04", + "attempt": 1, + "planned_compute_seconds": 96, + "planned_transfer_seconds": 0.5356666666671117, + "observed_runtime_seconds": 0.6820909976959229, + "validated_handle_runtime_seconds": 0.6820909976959229, + "terminal_observation_delay_seconds": 0.9627459049224854, + "preparation_elapsed_seconds": 132.95039653778076, + "overlapping_transfer_work_seconds": 265.9007520675659, + "launch_wait_seconds": 4.108665227890015, + "queue_seconds": 140.74181079864502, + "network_bytes": 287832814 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000008", + "resource_id": "montage-gcp-fog-e2-micro-03", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.20149999999989632, + "observed_runtime_seconds": 0.9904422760009766, + "validated_handle_runtime_seconds": 0.9904422760009766, + "terminal_observation_delay_seconds": 0.7154440879821777, + "preparation_elapsed_seconds": 104.50667762756348, + "overlapping_transfer_work_seconds": 205.53385853767395, + "launch_wait_seconds": 5.166273355484009, + "queue_seconds": 112.15365624427795, + "network_bytes": 143696217 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009", + "resource_id": "plafrim-execution-ssh-node-diablo03", + "attempt": 1, + "planned_compute_seconds": 11, + "planned_transfer_seconds": 1.202000000000453, + "observed_runtime_seconds": 2.2217273712158203, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 152.24401926994324, + "overlapping_transfer_work_seconds": 301.74500918388367, + "launch_wait_seconds": 9.20637059211731, + "queue_seconds": 0.8455982208251953, + "network_bytes": 577294529 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000010", + "resource_id": "montage-gcp-fog-e2-micro-01", + "attempt": 1, + "planned_compute_seconds": 96, + "planned_transfer_seconds": 0.49616666666724996, + "observed_runtime_seconds": 1.3641915321350098, + "validated_handle_runtime_seconds": 1.3641915321350098, + "terminal_observation_delay_seconds": 0.5377531051635742, + "preparation_elapsed_seconds": 135.12032461166382, + "overlapping_transfer_work_seconds": 270.24046659469604, + "launch_wait_seconds": 5.071686029434204, + "queue_seconds": 143.78016424179077, + "network_bytes": 290002805 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000024", + "resource_id": "montage-gcp-fog-e2-micro-06", + "attempt": 1, + "planned_compute_seconds": 64, + "planned_transfer_seconds": 0.1610000000000582, + "observed_runtime_seconds": 6.44399356842041, + "validated_handle_runtime_seconds": 6.44399356842041, + "terminal_observation_delay_seconds": 18.362658739089966, + "preparation_elapsed_seconds": 27.11520528793335, + "overlapping_transfer_work_seconds": 53.33376145362854, + "launch_wait_seconds": 2.9248297214508057, + "queue_seconds": 34.342814207077026, + "network_bytes": 142278066 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000025", + "resource_id": "montage-gcp-fog-e2-micro-05", + "attempt": 1, + "planned_compute_seconds": 80, + "planned_transfer_seconds": 0.40200000000004366, + "observed_runtime_seconds": 52.214394092559814, + "validated_handle_runtime_seconds": 52.214394092559814, + "terminal_observation_delay_seconds": 5.882064342498779, + "preparation_elapsed_seconds": 57.49071145057678, + "overlapping_transfer_work_seconds": 114.9814043045044, + "launch_wait_seconds": 201.20781135559082, + "queue_seconds": 348.135062456131, + "network_bytes": 284812026 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000026", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 64, + "planned_transfer_seconds": 0.40200000000004366, + "observed_runtime_seconds": 17.39228868484497, + "validated_handle_runtime_seconds": 17.39228868484497, + "terminal_observation_delay_seconds": 0.6327497959136963, + "preparation_elapsed_seconds": 96.25233221054077, + "overlapping_transfer_work_seconds": 192.5042848587036, + "launch_wait_seconds": 13.488139152526855, + "queue_seconds": 1166.1034712791443, + "network_bytes": 284581026 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000027", + "resource_id": "montage-gcp-fog-e2-micro-10", + "attempt": 1, + "planned_compute_seconds": 60, + "planned_transfer_seconds": 0.40200000000004366, + "observed_runtime_seconds": 9.441102981567383, + "validated_handle_runtime_seconds": 9.441102981567383, + "terminal_observation_delay_seconds": 1.0607397556304932, + "preparation_elapsed_seconds": 137.54758739471436, + "overlapping_transfer_work_seconds": 275.09407925605774, + "launch_wait_seconds": 13.53244924545288, + "queue_seconds": 488.71105909347534, + "network_bytes": 285714801 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000028", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 79.99999999999977, + "planned_transfer_seconds": 0.24099999999998545, + "observed_runtime_seconds": 0.9196591377258301, + "validated_handle_runtime_seconds": 0.9196591377258301, + "terminal_observation_delay_seconds": 1.0424652099609375, + "preparation_elapsed_seconds": 25.428139448165894, + "overlapping_transfer_work_seconds": 49.96107792854309, + "launch_wait_seconds": 2.972435235977173, + "queue_seconds": 31.323758602142334, + "network_bytes": 143205735 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000029", + "resource_id": "montage-gcp-e2-medium-1", + "attempt": 1, + "planned_compute_seconds": 13, + "planned_transfer_seconds": 0.24099999999998545, + "observed_runtime_seconds": 4.188046932220459, + "validated_handle_runtime_seconds": 4.188046932220459, + "terminal_observation_delay_seconds": 2.073564052581787, + "preparation_elapsed_seconds": 38.59876275062561, + "overlapping_transfer_work_seconds": 75.86889910697937, + "launch_wait_seconds": 13.927991151809692, + "queue_seconds": 55.673333406448364, + "network_bytes": 143436735 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000043", + "resource_id": "montage-gcp-fog-e2-micro-01", + "attempt": 1, + "planned_compute_seconds": 44, + "planned_transfer_seconds": 1.2010000000002492, + "observed_runtime_seconds": 21.509366750717163, + "validated_handle_runtime_seconds": 21.509366750717163, + "terminal_observation_delay_seconds": 1.5018932819366455, + "preparation_elapsed_seconds": 55.78641390800476, + "overlapping_transfer_work_seconds": 111.57274436950684, + "launch_wait_seconds": 77.60327458381653, + "queue_seconds": 237.82084918022156, + "network_bytes": 247240264 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000044", + "resource_id": "montage-gcp-fog-e2-micro-04", + "attempt": 1, + "planned_compute_seconds": 44, + "planned_transfer_seconds": 0.9620000000002165, + "observed_runtime_seconds": 2.544865131378174, + "validated_handle_runtime_seconds": 2.544865131378174, + "terminal_observation_delay_seconds": 1.4109084606170654, + "preparation_elapsed_seconds": 125.47228455543518, + "overlapping_transfer_work_seconds": 250.94447469711304, + "launch_wait_seconds": 22.9153413772583, + "queue_seconds": 261.1122784614563, + "network_bytes": 248486228 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045", + "resource_id": "plafrim-execution-ssh-node-diablo03", + "attempt": 1, + "planned_compute_seconds": 16, + "planned_transfer_seconds": 0.009000000000014552, + "observed_runtime_seconds": 1.3592419624328613, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 8.688355922698975, + "overlapping_transfer_work_seconds": 17.360018491744995, + "launch_wait_seconds": 4.0061821937561035, + "queue_seconds": 0.7228367328643799, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000046", + "resource_id": "montage-gcp-fog-e2-micro-08", + "attempt": 1, + "planned_compute_seconds": 56, + "planned_transfer_seconds": 0.32100000000036744, + "observed_runtime_seconds": 26.946631908416748, + "validated_handle_runtime_seconds": 26.946631908416748, + "terminal_observation_delay_seconds": 5.301002740859985, + "preparation_elapsed_seconds": 121.18112564086914, + "overlapping_transfer_work_seconds": 241.0824511051178, + "launch_wait_seconds": 25.251282215118408, + "queue_seconds": 149.5040020942688, + "network_bytes": 143242585 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000047", + "resource_id": "montage-gcp-fog-e2-micro-09", + "attempt": 1, + "planned_compute_seconds": 56, + "planned_transfer_seconds": 1.2010000000002492, + "observed_runtime_seconds": 11.818978071212769, + "validated_handle_runtime_seconds": 11.818978071212769, + "terminal_observation_delay_seconds": 2.245002269744873, + "preparation_elapsed_seconds": 114.06535649299622, + "overlapping_transfer_work_seconds": 228.12974977493286, + "launch_wait_seconds": 9.227837324142456, + "queue_seconds": 2748.4885082244873, + "network_bytes": 251911268 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000048", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 19, + "planned_transfer_seconds": 0.4810000000002219, + "observed_runtime_seconds": 2.9261977672576904, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 138.12837743759155, + "overlapping_transfer_work_seconds": 274.12316393852234, + "launch_wait_seconds": 4.544691562652588, + "queue_seconds": 0.017539024353027344, + "network_bytes": 289012504 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "resource_id": "montage-gcp-fog-e2-micro-06", + "attempt": 1, + "planned_compute_seconds": 88, + "planned_transfer_seconds": 3.1238361600012468, + "observed_runtime_seconds": 1.4470241069793701, + "validated_handle_runtime_seconds": 1.4470241069793701, + "terminal_observation_delay_seconds": 5.962459564208984, + "preparation_elapsed_seconds": 409.9749310016632, + "overlapping_transfer_work_seconds": 1639.8988542556763, + "launch_wait_seconds": 24.033706665039062, + "queue_seconds": 821.0685911178589, + "network_bytes": 2131649942 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "resource_id": "montage-gcp-fog-e2-micro-03", + "attempt": 1, + "planned_compute_seconds": 100, + "planned_transfer_seconds": 1.2840000000014697, + "observed_runtime_seconds": 1.9225947856903076, + "validated_handle_runtime_seconds": 1.9225947856903076, + "terminal_observation_delay_seconds": 1.3007850646972656, + "preparation_elapsed_seconds": 439.53410816192627, + "overlapping_transfer_work_seconds": 1758.1343455314636, + "launch_wait_seconds": 18.46054172515869, + "queue_seconds": 460.7046847343445, + "network_bytes": 1676512839 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 2.9630000000006476, + "observed_runtime_seconds": 0.1340923309326172, + "validated_handle_runtime_seconds": 0.1340923309326172, + "terminal_observation_delay_seconds": 0.4895446300506592, + "preparation_elapsed_seconds": 423.51526927948, + "overlapping_transfer_work_seconds": 1694.0608141422272, + "launch_wait_seconds": 9.238798141479492, + "queue_seconds": 601.3800776004791, + "network_bytes": 2119566281 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000001", + "resource_id": "montage-gcp-e2-medium-2", + "attempt": 1, + "planned_compute_seconds": 2500.1603327863495, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1009.9635989665985, + "validated_handle_runtime_seconds": 1009.9635989665985, + "terminal_observation_delay_seconds": 2.6425774097442627, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 104.24532985687256, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000002", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 1656, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 3303.6321551799774, + "validated_handle_runtime_seconds": 3303.6321551799774, + "terminal_observation_delay_seconds": 1.5413219928741455, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 177.97915768623352, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000003", + "resource_id": "montage-gcp-fog-e2-micro-03", + "attempt": 1, + "planned_compute_seconds": 2200, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1800.6897740364075, + "validated_handle_runtime_seconds": 1800.6897740364075, + "terminal_observation_delay_seconds": 1.809779405593872, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 267.1590392589569, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000004", + "resource_id": "montage-gcp-fog-e2-micro-09", + "attempt": 1, + "planned_compute_seconds": 3296, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 3373.3067605495453, + "validated_handle_runtime_seconds": 3373.3067605495453, + "terminal_observation_delay_seconds": 0.6629946231842041, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 172.46223878860474, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000020", + "resource_id": "montage-gcp-fog-e2-micro-06", + "attempt": 1, + "planned_compute_seconds": 2112, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 2212.9325346946716, + "validated_handle_runtime_seconds": 2212.9325346946716, + "terminal_observation_delay_seconds": 1.5519981384277344, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 212.5859682559967, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000021", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 1444, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 2058.7907650470734, + "validated_handle_runtime_seconds": 2058.7907650470734, + "terminal_observation_delay_seconds": 1.5546224117279053, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 100.34812378883362, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000022", + "resource_id": "montage-gcp-e2-medium-2", + "attempt": 1, + "planned_compute_seconds": 1953.1603327863493, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 540.6099560260773, + "validated_handle_runtime_seconds": 540.6099560260773, + "terminal_observation_delay_seconds": 2.2877111434936523, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 104.22805833816528, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000023", + "resource_id": "montage-gcp-e2-medium-1", + "attempt": 1, + "planned_compute_seconds": 1974.0870506376316, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 616.2185587882996, + "validated_handle_runtime_seconds": 616.2185587882996, + "terminal_observation_delay_seconds": 1.8173558712005615, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 95.84681725502014, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000039", + "resource_id": "plafrim-execution-ssh-node-diablo03", + "attempt": 1, + "planned_compute_seconds": 1540, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 434.79141211509705, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 6.485845327377319, + "overlapping_transfer_work_seconds": 6.485845327377319, + "launch_wait_seconds": 5.010369062423706, + "queue_seconds": 0.8524012565612793, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000040", + "resource_id": "montage-gcp-e2-medium-1", + "attempt": 1, + "planned_compute_seconds": 2379.087050637632, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 830.2273955345154, + "validated_handle_runtime_seconds": 830.2273955345154, + "terminal_observation_delay_seconds": 0.6870224475860596, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 95.84815573692322, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000041", + "resource_id": "montage-gcp-fog-e2-micro-08", + "attempt": 1, + "planned_compute_seconds": 3012, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1821.5046372413635, + "validated_handle_runtime_seconds": 1821.5046372413635, + "terminal_observation_delay_seconds": 3.633888006210327, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 198.16842126846313, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000042", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 1546, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 430.53262972831726, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 6.489797830581665, + "overlapping_transfer_work_seconds": 6.489797830581665, + "launch_wait_seconds": 5.231635808944702, + "queue_seconds": 0.46117353439331055, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000019", + "resource_id": "montage-gcp-e2-medium-1", + "attempt": 1, + "planned_compute_seconds": 17, + "planned_transfer_seconds": 0.1610000000000582, + "observed_runtime_seconds": 1.2472658157348633, + "validated_handle_runtime_seconds": 1.2472658157348633, + "terminal_observation_delay_seconds": 0.4268317222595215, + "preparation_elapsed_seconds": 468.19441294670105, + "overlapping_transfer_work_seconds": 468.19441294670105, + "launch_wait_seconds": 5.162675142288208, + "queue_seconds": 489.68685269355774, + "network_bytes": 624504559 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000038", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 32, + "planned_transfer_seconds": 0.1610000000000582, + "observed_runtime_seconds": 1.6501014232635498, + "validated_handle_runtime_seconds": 1.6501014232635498, + "terminal_observation_delay_seconds": 17.32021927833557, + "preparation_elapsed_seconds": 157.68661236763, + "overlapping_transfer_work_seconds": 157.68661236763, + "launch_wait_seconds": 9.076800107955933, + "queue_seconds": 169.12029814720154, + "network_bytes": 615130007 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000057", + "resource_id": "montage-gcp-fog-e2-micro-10", + "attempt": 1, + "planned_compute_seconds": 4, + "planned_transfer_seconds": 0.1610000000000582, + "observed_runtime_seconds": 26.41062831878662, + "validated_handle_runtime_seconds": 26.41062831878662, + "terminal_observation_delay_seconds": 1.5639357566833496, + "preparation_elapsed_seconds": 495.8601224422455, + "overlapping_transfer_work_seconds": 495.8601224422455, + "launch_wait_seconds": 54.45741558074951, + "queue_seconds": 553.5859718322754, + "network_bytes": 619720534 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000058", + "resource_id": "montage-gcp-e2-medium-2", + "attempt": 1, + "planned_compute_seconds": 7, + "planned_transfer_seconds": 0.4830000000001746, + "observed_runtime_seconds": 2.885821580886841, + "validated_handle_runtime_seconds": 2.885821580886841, + "terminal_observation_delay_seconds": 0.4241642951965332, + "preparation_elapsed_seconds": 486.8266372680664, + "overlapping_transfer_work_seconds": 1460.479565858841, + "launch_wait_seconds": 24.22416591644287, + "queue_seconds": 525.6955797672272, + "network_bytes": 1859355100 + } + ], + "limitations": [ + "No multiple attempts observed; this alone does not prove an uninterrupted benchmark.", + "Runtime finish semantics depend on the driver; observation delay is exported only for exact-attempt handles with validated terminal timestamps.", + "API aggregate counters can contain older attempts; audit uses latest distinct IDs.", + "Transfer work is summed overlapping route time, not wall-clock preparation.", + "Plan assignment work sums are not makespan; API timestamp wall and recovered makespan can differ.", + "Cloud lifecycle cost is a list-rate estimate with explicit exclusions, not the provider invoice. The task/window API cost and plan retention-window cost have different bases.", + "e2-medium interference extrapolated to HPC/micro; micro speedup 0.25 is approximate.", + "Whole-workspace real traffic differs from estimated plan edge volumes.", + "Zero reported interference does not prove absence of physical contention." + ], + "product": { + "filename": "mosaic-color-v2.png", + "bytes": 4920451, + "sha256": "63fa4d8447274bdb8a44f6be8cf5a23c0a2f0baa479bd2ae1ec3b541df4e19f2" + } +} diff --git a/experiments/montage-micro-hpc-e2/dominant-transfer-residual-gate-20261011.json b/experiments/montage-micro-hpc-e2/dominant-transfer-residual-gate-20261011.json new file mode 100644 index 00000000..92cabca6 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/dominant-transfer-residual-gate-20261011.json @@ -0,0 +1,22 @@ +{ + "issue":"AKO-158", + "referenceRunId":"montage-prism-transfer-real-20261011-v2", + "role":"recovered-reference-residuals-not-independent-validation", + "dependencyCount":114, + "dominantProducer":"montage-independent-work-v1-mimgtblid0000017", + "dominantConsumer":"montage-independent-work-v1-maddid0000018", + "predictedDependencySeconds":288.8601371575005, + "observedDependencyWallSeconds":1294.134133049, + "durationErrorPercent":-77.67927374908646, + "observedProbeSeconds":6.731303795, + "observedSnapshotAcquireSeconds":746.63980359, + "observedUploadSeconds":206.247699719, + "observedReceiverSeconds":334.515325945, + "dominantTransferGatePassed":false, + "limitations":[ + "Shared consumer barrier and merge were excluded from this dependency comparison.", + "Snapshot acquisition, transport and extraction include possible admission/contention; a slower network alone is not a demonstrated cause.", + "Recovered copy reuse is excluded from fresh transfer-duration comparisons.", + "Improved makespan sensitivity does not establish an adequate transfer model." + ] +} diff --git a/experiments/montage-micro-hpc-e2/execution-calibration-v2.json b/experiments/montage-micro-hpc-e2/execution-calibration-v2.json new file mode 100644 index 00000000..bd5138b1 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/execution-calibration-v2.json @@ -0,0 +1,2457 @@ +{ + "schemaVersion": 1, + "baselineRunId": "montage-docker-image-real-20261010-v2", + "baselineSHA256": "ce8343153d9016ce2149fa5e70b3624ca148c4747e6dd9887dae1fa2b6ab46c6", + "synchronizationContract": "whole-workspace-v1", + "workspaceVolumeBasis": "producer-final-snapshot-manifest", + "computeObservations": [ + { + "activityFamily": "madd", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 3, + "median": 8.217379093170166, + "minimum": 0.9578084945678711, + "maximum": 8.437352895736694 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 3, + "median": 0.9374836550818549, + "minimum": 0.10642316606309679, + "maximum": 4.108689546585083 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mbackground", + "resourceClass": "e2-medium", + "runtimeSeconds": { + "samples": 2, + "median": 0.5800764560699463, + "minimum": 0.5333530902862549, + "maximum": 0.6267998218536377 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 0.32499454332434613, + "minimum": 0.02318926479505456, + "maximum": 0.6267998218536377 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mbackground", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 8, + "median": 2.1703304052352905, + "minimum": 0.9790773391723633, + "maximum": 23.867364168167114 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 8, + "median": 0.13403182718488904, + "minimum": 0.09645940860112508, + "maximum": 13.19267725944519 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mbackground", + "resourceClass": "hpc", + "runtimeSeconds": { + "samples": 2, + "median": 3.2587720155715942, + "minimum": 2.595179796218872, + "maximum": 3.9223642349243164 + }, + "taskRuntimeFallbackSamples": 2, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 2.042281486093998, + "minimum": 0.1621987372636795, + "maximum": 3.9223642349243164 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mbgmodel", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 2, + "median": 0.5460176467895508, + "minimum": 0.47063517570495605, + "maximum": 0.6214001178741455 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 0.07053212135557144, + "minimum": 0.05229279730055067, + "maximum": 0.08877144541059222 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mbgmodel", + "resourceClass": "e2-medium", + "runtimeSeconds": { + "samples": 1, + "median": 0.5198588371276855, + "minimum": 0.5198588371276855, + "maximum": 0.5198588371276855 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 1, + "median": 0.025992941856384278, + "minimum": 0.025992941856384278, + "maximum": 0.025992941856384278 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "hpc", + "runtimeSeconds": { + "samples": 1, + "median": 2.8328795433044434, + "minimum": 2.8328795433044434, + "maximum": 2.8328795433044434 + }, + "taskRuntimeFallbackSamples": 1, + "wallSecondsPerReferenceSecond": { + "samples": 1, + "median": 0.16663997313555548, + "minimum": 0.16663997313555548, + "maximum": 0.16663997313555548 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 2, + "median": 3.633373498916626, + "minimum": 2.559312343597412, + "maximum": 4.70743465423584 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 0.5174553818172878, + "minimum": 0.5118624687194824, + "maximum": 0.5230482949150933 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mdifffit", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 14, + "median": 7.9425482749938965, + "minimum": 0.6820909976959229, + "maximum": 52.214394092559814 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 14, + "median": 0.5160782317320506, + "minimum": 0.02842045823733012, + "maximum": 2.6107197046279906 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mdifffit", + "resourceClass": "hpc", + "runtimeSeconds": { + "samples": 3, + "median": 2.2217273712158203, + "minimum": 1.3592419624328613, + "maximum": 2.9261977672576904 + }, + "taskRuntimeFallbackSamples": 3, + "wallSecondsPerReferenceSecond": { + "samples": 3, + "median": 0.15401040880303635, + "minimum": 0.08495262265205383, + "maximum": 0.20197521556507458 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mdifffit", + "resourceClass": "e2-medium", + "runtimeSeconds": { + "samples": 1, + "median": 4.188046932220459, + "minimum": 4.188046932220459, + "maximum": 4.188046932220459 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 1, + "median": 0.3221574563246507, + "minimum": 0.3221574563246507, + "maximum": 0.3221574563246507 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mimgtbl", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 3, + "median": 1.4470241069793701, + "minimum": 0.1340923309326172, + "maximum": 1.9225947856903076 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 3, + "median": 0.06577382304451683, + "minimum": 0.01489914788140191, + "maximum": 0.0769037914276123 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mproject", + "resourceClass": "e2-medium", + "runtimeSeconds": { + "samples": 4, + "median": 723.2229771614075, + "minimum": 540.6099560260773, + "maximum": 1009.9635989665985 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 4, + "median": 0.5422618222257024, + "minimum": 0.482687460737569, + "maximum": 0.6058569879823626 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mproject", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 6, + "median": 2135.8616498708725, + "minimum": 1800.6897740364075, + "maximum": 3373.3067605495453 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 6, + "median": 4.142489493875232, + "minimum": 2.4189968622063263, + "maximum": 7.97978781444439 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mproject", + "resourceClass": "hpc", + "runtimeSeconds": { + "samples": 2, + "median": 432.66202092170715, + "minimum": 430.53262972831726, + "maximum": 434.79141211509705 + }, + "taskRuntimeFallbackSamples": 2, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 0.280406867515572, + "minimum": 0.2784816492421198, + "maximum": 0.2823320857890241 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mviewer", + "resourceClass": "e2-medium", + "runtimeSeconds": { + "samples": 2, + "median": 2.066543698310852, + "minimum": 1.2472658157348633, + "maximum": 2.885821580886841 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 0.24281440161857284, + "minimum": 0.07336857739616842, + "maximum": 0.41226022584097727 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mviewer", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 2, + "median": 14.030364871025085, + "minimum": 1.6501014232635498, + "maximum": 26.41062831878662 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 13.308445498347282, + "minimum": 0.20626267790794373, + "maximum": 26.41062831878662 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + } + ], + "materializationPhaseObservations": [ + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-micro", + "phase": "peer-authenticated-connectivity-probe", + "seconds": { + "samples": 20, + "median": 2.4162210225000003, + "minimum": 2.286490792, + "maximum": 13.712441924 + }, + "logicalBytes": { + "samples": 18, + "median": 196610182.0, + "minimum": 18236443.0, + "maximum": 740939386.0 + }, + "archiveBytes": { + "samples": 20, + "median": 143339660.0, + "minimum": 142917128.0, + "maximum": 426325775.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 196938502.0, + "minimum": 195579142.0, + "maximum": 740939386.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.7256941966584066, + "minimum": 0.5719966097845853, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 83676394.3242459, + "minimum": 35750036.62047911, + "maximum": 311227613.81359726 + }, + "files": { + "samples": 18, + "median": 47.0, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 60827976.14251813, + "minimum": 26046095.920719363, + "maximum": 179075854.47277406 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-micro", + "phase": "peer-snapshot-acquire", + "seconds": { + "samples": 20, + "median": 6.8133940655, + "minimum": 3.5541e-05, + "maximum": 23.889355677 + }, + "logicalBytes": { + "samples": 18, + "median": 196610182.0, + "minimum": 18236443.0, + "maximum": 740939386.0 + }, + "archiveBytes": { + "samples": 20, + "median": 143339660.0, + "minimum": 142917128.0, + "maximum": 426325775.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 196938502.0, + "minimum": 195579142.0, + "maximum": 740939386.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.7256941966584066, + "minimum": 0.5719966097845853, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 30510114.773531586, + "minimum": 18470614.9184961, + "maximum": 17811951886553.56 + }, + "files": { + "samples": 18, + "median": 47.0, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 21221299.827622443, + "minimum": 13456976.645619227, + "maximum": 10350951070594.525 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-micro", + "phase": "peer-private-upload", + "seconds": { + "samples": 20, + "median": 6.318401003, + "minimum": 3.399642126, + "maximum": 50.113527148 + }, + "logicalBytes": { + "samples": 18, + "median": 196610182.0, + "minimum": 18236443.0, + "maximum": 740939386.0 + }, + "archiveBytes": { + "samples": 20, + "median": 143339660.0, + "minimum": 142917128.0, + "maximum": 426325775.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 196938502.0, + "minimum": 195579142.0, + "maximum": 740939386.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.7256941966584066, + "minimum": 0.5719966097845853, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 54899955.45290358, + "minimum": 7649470.226514639, + "maximum": 113099800.15949468 + }, + "files": { + "samples": 18, + "median": 47.0, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 39840579.06897718, + "minimum": 5610082.049398552, + "maximum": 64692702.25854507 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-micro", + "phase": "peer-receiver-verify-and-extract", + "seconds": { + "samples": 20, + "median": 11.3101169635, + "minimum": 2.66509171, + "maximum": 237.244030355 + }, + "logicalBytes": { + "samples": 18, + "median": 196610182.0, + "minimum": 18236443.0, + "maximum": 740939386.0 + }, + "archiveBytes": { + "samples": 20, + "median": 143339660.0, + "minimum": 142917128.0, + "maximum": 426325775.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 196938502.0, + "minimum": 195579142.0, + "maximum": 740939386.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.7256941966584066, + "minimum": 0.5719966097845853, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 45867943.07818958, + "minimum": 3123110.768651568, + "maximum": 73402810.59221035 + }, + "files": { + "samples": 18, + "median": 47.0, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 29496458.305427484, + "minimum": 1796992.633964562, + "maximum": 53733886.32843708 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-authenticated-connectivity-probe", + "seconds": { + "samples": 45, + "median": 2.444578959, + "minimum": 2.267388085, + "maximum": 14.119102382 + }, + "logicalBytes": { + "samples": 44, + "median": 106346880.0, + "minimum": 2594.0, + "maximum": 1109057816.0 + }, + "archiveBytes": { + "samples": 45, + "median": 363111632.0, + "minimum": 141375291.0, + "maximum": 619720534.0 + }, + "workspaceBytes": { + "samples": 45, + "median": 627864822.0, + "minimum": 195625222.0, + "maximum": 1109057816.0 + }, + "archiveToWorkspaceRatio": { + "samples": 45, + "median": 0.578327721631775, + "minimum": 0.5587810888300886, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 45, + "median": 134248084.80919027, + "minimum": 26694732.13031702, + "maximum": 481931462.75104815 + }, + "files": { + "samples": 44, + "median": 2.5, + "minimum": 1.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 45, + "median": 85519694.35317792, + "minimum": 19647136.301396284, + "maximum": 269499698.2431896 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-snapshot-acquire", + "seconds": { + "samples": 45, + "median": 47.297451689, + "minimum": 0.000196, + "maximum": 724.592180869 + }, + "logicalBytes": { + "samples": 44, + "median": 106346880.0, + "minimum": 2594.0, + "maximum": 1109057816.0 + }, + "archiveBytes": { + "samples": 45, + "median": 363111632.0, + "minimum": 141375291.0, + "maximum": 619720534.0 + }, + "workspaceBytes": { + "samples": 45, + "median": 627864822.0, + "minimum": 195625222.0, + "maximum": 1109057816.0 + }, + "archiveToWorkspaceRatio": { + "samples": 45, + "median": 0.578327721631775, + "minimum": 0.5587810888300886, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 45, + "median": 13591501.74435183, + "minimum": 1446547.9612862365, + "maximum": 998293581632.6531 + }, + "files": { + "samples": 44, + "median": 2.5, + "minimum": 1.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 45, + "median": 7600469.9877143195, + "minimum": 810355.8022055948, + "maximum": 725908500000.0 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-private-upload", + "seconds": { + "samples": 45, + "median": 21.295400718, + "minimum": 3.692133835, + "maximum": 204.790461176 + }, + "logicalBytes": { + "samples": 44, + "median": 106346880.0, + "minimum": 2594.0, + "maximum": 1109057816.0 + }, + "archiveBytes": { + "samples": 45, + "median": 363111632.0, + "minimum": 141375291.0, + "maximum": 619720534.0 + }, + "workspaceBytes": { + "samples": 45, + "median": 627864822.0, + "minimum": 195625222.0, + "maximum": 1109057816.0 + }, + "archiveToWorkspaceRatio": { + "samples": 45, + "median": 0.578327721631775, + "minimum": 0.5587810888300886, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 45, + "median": 27154065.76248039, + "minimum": 5056332.734069341, + "maximum": 127839669.3181365 + }, + "files": { + "samples": 44, + "median": 2.5, + "minimum": 1.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 45, + "median": 17623724.802639503, + "minimum": 2912047.4390039076, + "maximum": 71566066.50891274 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-receiver-verify-and-extract", + "seconds": { + "samples": 45, + "median": 12.808673423, + "minimum": 2.726851626, + "maximum": 467.73053142 + }, + "logicalBytes": { + "samples": 44, + "median": 106346880.0, + "minimum": 2594.0, + "maximum": 1109057816.0 + }, + "archiveBytes": { + "samples": 45, + "median": 363111632.0, + "minimum": 141375291.0, + "maximum": 619720534.0 + }, + "workspaceBytes": { + "samples": 45, + "median": 627864822.0, + "minimum": 195625222.0, + "maximum": 1109057816.0 + }, + "archiveToWorkspaceRatio": { + "samples": 45, + "median": 0.578327721631775, + "minimum": 0.5587810888300886, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 45, + "median": 45018555.18656611, + "minimum": 1584029.7270111442, + "maximum": 71755111.32852522 + }, + "files": { + "samples": 44, + "median": 2.5, + "minimum": 1.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 45, + "median": 28348867.599979248, + "minimum": 911255.8500425805, + "maximum": 52176680.47773714 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "pack-and-receive", + "seconds": { + "samples": 11, + "median": 42.532719728000004, + "minimum": 11.220321047, + "maximum": 102.588095879 + }, + "logicalBytes": { + "samples": 10, + "median": 106346880.0, + "minimum": 18391965.0, + "maximum": 739073142.0 + }, + "archiveBytes": { + "samples": 11, + "median": 208894084.0, + "minimum": 51992429.0, + "maximum": 426351589.0 + }, + "workspaceBytes": { + "samples": 11, + "median": 321020707.0, + "minimum": 106018560.0, + "maximum": 740899066.0 + }, + "archiveToWorkspaceRatio": { + "samples": 11, + "median": 0.5736604862310096, + "minimum": 0.4904087454121241, + "maximum": 0.6507184098874967 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 11, + "median": 7633347.82034076, + "minimum": 2678102.132784162, + "maximum": 12998257.441814132 + }, + "files": { + "samples": 10, + "median": 2.5, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 11, + "median": 4565984.029781632, + "minimum": 1313364.7070242148, + "maximum": 7479868.674241965 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "archive-validation", + "seconds": { + "samples": 11, + "median": 1.732665959, + "minimum": 0.502052333, + "maximum": 3.854221543 + }, + "logicalBytes": { + "samples": 10, + "median": 106346880.0, + "minimum": 18391965.0, + "maximum": 739073142.0 + }, + "archiveBytes": { + "samples": 11, + "median": 208894084.0, + "minimum": 51992429.0, + "maximum": 426351589.0 + }, + "workspaceBytes": { + "samples": 11, + "median": 321020707.0, + "minimum": 106018560.0, + "maximum": 740899066.0 + }, + "archiveToWorkspaceRatio": { + "samples": 11, + "median": 0.5736604862310096, + "minimum": 0.4904087454121241, + "maximum": 0.6507184098874967 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 11, + "median": 193524669.92053956, + "minimum": 164995666.03024346, + "maximum": 211170336.30037928 + }, + "files": { + "samples": 10, + "median": 2.5, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 11, + "median": 108211123.48062727, + "minimum": 89732925.24604906, + "maximum": 120562237.00531535 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "archive-upload", + "seconds": { + "samples": 11, + "median": 33.195026598, + "minimum": 5.756016586, + "maximum": 88.28292704 + }, + "logicalBytes": { + "samples": 10, + "median": 106346880.0, + "minimum": 18391965.0, + "maximum": 739073142.0 + }, + "archiveBytes": { + "samples": 11, + "median": 208894084.0, + "minimum": 51992429.0, + "maximum": 426351589.0 + }, + "workspaceBytes": { + "samples": 11, + "median": 321020707.0, + "minimum": 106018560.0, + "maximum": 740899066.0 + }, + "archiveToWorkspaceRatio": { + "samples": 11, + "median": 0.5736604862310096, + "minimum": 0.4904087454121241, + "maximum": 0.6507184098874967 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 11, + "median": 15118301.022502018, + "minimum": 2078065.4066888622, + "maximum": 20852671.868023537 + }, + "files": { + "samples": 10, + "median": 2.5, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 11, + "median": 8139313.38762742, + "minimum": 1064873.9603471688, + "maximum": 11962353.883026078 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "receiver-verify-and-extract", + "seconds": { + "samples": 11, + "median": 7.516678671, + "minimum": 1.272753501, + "maximum": 209.987703305 + }, + "logicalBytes": { + "samples": 10, + "median": 106346880.0, + "minimum": 18391965.0, + "maximum": 739073142.0 + }, + "archiveBytes": { + "samples": 11, + "median": 208894084.0, + "minimum": 51992429.0, + "maximum": 426351589.0 + }, + "workspaceBytes": { + "samples": 11, + "median": 321020707.0, + "minimum": 106018560.0, + "maximum": 740899066.0 + }, + "archiveToWorkspaceRatio": { + "samples": 11, + "median": 0.5736604862310096, + "minimum": 0.4904087454121241, + "maximum": 0.6507184098874967 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 11, + "median": 46798115.17037942, + "minimum": 3528297.392366206, + "maximum": 98548224.98363979 + }, + "files": { + "samples": 10, + "median": 2.5, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 11, + "median": 23981003.71521475, + "minimum": 2030364.5513029818, + "maximum": 57403030.977419436 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-medium", + "phase": "peer-authenticated-connectivity-probe", + "seconds": { + "samples": 7, + "median": 3.238551043, + "minimum": 2.347021292, + "maximum": 5.747049585 + }, + "logicalBytes": { + "samples": 7, + "median": 1009451800.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveBytes": { + "samples": 7, + "median": 615130007.0, + "minimum": 363111632.0, + "maximum": 624504559.0 + }, + "workspaceBytes": { + "samples": 7, + "median": 1100003102.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveToWorkspaceRatio": { + "samples": 7, + "median": 0.5616915015784198, + "minimum": 0.5587810888300886, + "maximum": 0.5824735682180756 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 7, + "median": 333062072.17141193, + "minimum": 109249939.94113939, + "maximum": 471436381.27734333 + }, + "files": { + "samples": 7, + "median": 73.0, + "minimum": 39.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 7, + "median": 186108787.3359471, + "minimum": 63182268.8545674, + "maximum": 264801808.89836743 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-medium", + "phase": "peer-snapshot-acquire", + "seconds": { + "samples": 7, + "median": 222.943990226, + "minimum": 7.5167e-05, + "maximum": 362.706896956 + }, + "logicalBytes": { + "samples": 7, + "median": 1009451800.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveBytes": { + "samples": 7, + "median": 615130007.0, + "minimum": 363111632.0, + "maximum": 624504559.0 + }, + "workspaceBytes": { + "samples": 7, + "median": 1100003102.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveToWorkspaceRatio": { + "samples": 7, + "median": 0.5616915015784198, + "minimum": 0.5587810888300886, + "maximum": 0.5824735682180756 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 7, + "median": 3071406.804890141, + "minimum": 1914116.5272692032, + "maximum": 14754584006279.354 + }, + "files": { + "samples": 7, + "median": 73.0, + "minimum": 39.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 7, + "median": 1725183.10019692, + "minimum": 1106986.6501533238, + "maximum": 8244582516263.787 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-medium", + "phase": "peer-reused-source-manifest-check", + "seconds": { + "samples": 3, + "median": 14.740183174, + "minimum": 8.123189962, + "maximum": 54.534579608 + }, + "logicalBytes": { + "samples": 3, + "median": 1018506514.0, + "minimum": 634552186.0, + "maximum": 1111828392.0 + }, + "archiveBytes": { + "samples": 3, + "median": 619720534.0, + "minimum": 369609876.0, + "maximum": 624504559.0 + }, + "workspaceBytes": { + "samples": 3, + "median": 1109057816.0, + "minimum": 634552186.0, + "maximum": 1111828392.0 + }, + "archiveToWorkspaceRatio": { + "samples": 3, + "median": 0.5616915015784198, + "minimum": 0.5587810888300886, + "maximum": 0.5824735682180756 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 3, + "median": 43049138.43399705, + "minimum": 20387585.271435726, + "maximum": 136529838.79339692 + }, + "files": { + "samples": 3, + "median": 73.0, + "minimum": 39.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 3, + "median": 25074985.27236416, + "minimum": 11451533.38467081, + "maximum": 76290291.97877079 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-medium", + "phase": "peer-private-upload", + "seconds": { + "samples": 7, + "median": 27.623296388, + "minimum": 7.566221212, + "maximum": 98.32883392 + }, + "logicalBytes": { + "samples": 7, + "median": 1009451800.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveBytes": { + "samples": 7, + "median": 615130007.0, + "minimum": 363111632.0, + "maximum": 624504559.0 + }, + "workspaceBytes": { + "samples": 7, + "median": 1100003102.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveToWorkspaceRatio": { + "samples": 7, + "median": 0.5616915015784198, + "minimum": 0.5587810888300886, + "maximum": 0.5824735682180756 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 7, + "median": 40249663.77593501, + "minimum": 6385358.159650593, + "maximum": 146580146.80314106 + }, + "files": { + "samples": 7, + "median": 73.0, + "minimum": 39.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 7, + "median": 22607894.084331468, + "minimum": 3692829.636273592, + "maximum": 81906214.03153339 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-medium", + "phase": "peer-receiver-verify-and-extract", + "seconds": { + "samples": 7, + "median": 10.862616296, + "minimum": 5.661688502, + "maximum": 12.127251798 + }, + "logicalBytes": { + "samples": 7, + "median": 1009451800.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveBytes": { + "samples": 7, + "median": 615130007.0, + "minimum": 363111632.0, + "maximum": 624504559.0 + }, + "workspaceBytes": { + "samples": 7, + "median": 1100003102.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveToWorkspaceRatio": { + "samples": 7, + "median": 0.5616915015784198, + "minimum": 0.5587810888300886, + "maximum": 0.5824735682180756 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 7, + "median": 94960134.68711175, + "minimum": 83793313.37409675, + "maximum": 110897097.53162962 + }, + "files": { + "samples": 7, + "median": 73.0, + "minimum": 39.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 7, + "median": 53061927.45591616, + "minimum": 48807390.23382553, + "maximum": 64134865.75104411 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-micro", + "targetClass": "hpc", + "phase": "pack-and-receive", + "seconds": { + "samples": 10, + "median": 38.603155225500004, + "minimum": 11.782335589, + "maximum": 199.5754098 + }, + "logicalBytes": null, + "archiveBytes": { + "samples": 10, + "median": 203193258.0, + "minimum": 143696217.0, + "maximum": 369609876.0 + }, + "workspaceBytes": { + "samples": 10, + "median": 312516065.5, + "minimum": 196610182.0, + "maximum": 634552186.0 + }, + "archiveToWorkspaceRatio": { + "samples": 10, + "median": 0.6522022259028002, + "minimum": 0.5824735682180756, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 10, + "median": 12514547.202893108, + "minimum": 1678160.6127510006, + "maximum": 19765033.306060687 + }, + "files": { + "samples": 9, + "median": 5.0, + "minimum": 2.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 10, + "median": 7741110.27791067, + "minimum": 1075263.5668645387, + "maximum": 12819134.471322034 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-micro", + "targetClass": "hpc", + "phase": "archive-validation", + "seconds": { + "samples": 10, + "median": 1.6895398135000002, + "minimum": 1.133418042, + "maximum": 3.341227627 + }, + "logicalBytes": null, + "archiveBytes": { + "samples": 10, + "median": 203193258.0, + "minimum": 143696217.0, + "maximum": 369609876.0 + }, + "workspaceBytes": { + "samples": 10, + "median": 312516065.5, + "minimum": 196610182.0, + "maximum": 634552186.0 + }, + "archiveToWorkspaceRatio": { + "samples": 10, + "median": 0.6522022259028002, + "minimum": 0.5824735682180756, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 10, + "median": 179804359.97152635, + "minimum": 167484361.96237248, + "maximum": 189915880.28073013 + }, + "files": { + "samples": 9, + "median": 5.0, + "minimum": 2.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 10, + "median": 120277383.91281037, + "minimum": 110620980.44839375, + "maximum": 127857225.33963335 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-micro", + "targetClass": "hpc", + "phase": "archive-upload", + "seconds": { + "samples": 10, + "median": 76.619061473, + "minimum": 24.609935552, + "maximum": 171.383783244 + }, + "logicalBytes": null, + "archiveBytes": { + "samples": 10, + "median": 203193258.0, + "minimum": 143696217.0, + "maximum": 369609876.0 + }, + "workspaceBytes": { + "samples": 10, + "median": 312516065.5, + "minimum": 196610182.0, + "maximum": 634552186.0 + }, + "archiveToWorkspaceRatio": { + "samples": 10, + "median": 0.6522022259028002, + "minimum": 0.5824735682180756, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 10, + "median": 5540595.869932596, + "minimum": 1852593.7579577223, + "maximum": 8000759.757536158 + }, + "files": { + "samples": 9, + "median": 5.0, + "minimum": 2.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 10, + "median": 3704139.690952878, + "minimum": 1207508.998558948, + "maximum": 5895469.278797403 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-micro", + "targetClass": "hpc", + "phase": "receiver-verify-and-extract", + "seconds": { + "samples": 10, + "median": 7.1363985240000005, + "minimum": 3.683395543, + "maximum": 10.001879504 + }, + "logicalBytes": null, + "archiveBytes": { + "samples": 10, + "median": 203193258.0, + "minimum": 143696217.0, + "maximum": 369609876.0 + }, + "workspaceBytes": { + "samples": 10, + "median": 312516065.5, + "minimum": 196610182.0, + "maximum": 634552186.0 + }, + "archiveToWorkspaceRatio": { + "samples": 10, + "median": 0.6522022259028002, + "minimum": 0.5824735682180756, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 10, + "median": 46164951.51762764, + "minimum": 34201395.10569445, + "maximum": 63443294.40743881 + }, + "files": { + "samples": 9, + "median": 5.0, + "minimum": 2.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 10, + "median": 31838848.70171778, + "minimum": 21772903.484861646, + "maximum": 39342960.6753477 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-reused-source-manifest-check", + "seconds": { + "samples": 6, + "median": 27.712840638, + "minimum": 0.474361584, + "maximum": 50.339336731 + }, + "logicalBytes": { + "samples": 6, + "median": 411765182.0, + "minimum": 106346880.0, + "maximum": 627864822.0 + }, + "archiveBytes": { + "samples": 6, + "median": 254013659.0, + "minimum": 142278066.0, + "maximum": 363111632.0 + }, + "workspaceBytes": { + "samples": 6, + "median": 412381502.0, + "minimum": 195665542.0, + "maximum": 627864822.0 + }, + "archiveToWorkspaceRatio": { + "samples": 6, + "median": 0.6527385212944301, + "minimum": 0.578327721631775, + "maximum": 0.7359930118603126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 6, + "median": 25591306.483499434, + "minimum": 12472647.888770213, + "maximum": 412481846.3377085 + }, + "files": { + "samples": 6, + "median": 60.0, + "minimum": 2.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 6, + "median": 17731894.794842176, + "minimum": 7213278.0362278465, + "maximum": 299935894.47158945 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-medium", + "targetClass": "hpc", + "phase": "pack-and-receive", + "seconds": { + "samples": 1, + "median": 30.418417181, + "minimum": 30.418417181, + "maximum": 30.418417181 + }, + "logicalBytes": null, + "archiveBytes": { + "samples": 1, + "median": 367883152.0, + "minimum": 367883152.0, + "maximum": 367883152.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 633054582.0, + "minimum": 633054582.0, + "maximum": 633054582.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5811239069429878, + "minimum": 0.5811239069429878, + "maximum": 0.5811239069429878 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 20811555.651732583, + "minimum": 20811555.651732583, + "maximum": 20811555.651732583 + }, + "files": { + "samples": 1, + "median": 73.0, + "minimum": 73.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 12094092.529896256, + "minimum": 12094092.529896256, + "maximum": 12094092.529896256 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-medium", + "targetClass": "hpc", + "phase": "archive-validation", + "seconds": { + "samples": 1, + "median": 3.350000627, + "minimum": 3.350000627, + "maximum": 3.350000627 + }, + "logicalBytes": null, + "archiveBytes": { + "samples": 1, + "median": 367883152.0, + "minimum": 367883152.0, + "maximum": 367883152.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 633054582.0, + "minimum": 633054582.0, + "maximum": 633054582.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5811239069429878, + "minimum": 0.5811239069429878, + "maximum": 0.5811239069429878 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 188971481.64623314, + "minimum": 188971481.64623314, + "maximum": 188971481.64623314 + }, + "files": { + "samples": 1, + "median": 73.0, + "minimum": 73.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 109815845.7150641, + "minimum": 109815845.7150641, + "maximum": 109815845.7150641 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-medium", + "targetClass": "hpc", + "phase": "archive-upload", + "seconds": { + "samples": 1, + "median": 109.214450842, + "minimum": 109.214450842, + "maximum": 109.214450842 + }, + "logicalBytes": null, + "archiveBytes": { + "samples": 1, + "median": 367883152.0, + "minimum": 367883152.0, + "maximum": 367883152.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 633054582.0, + "minimum": 633054582.0, + "maximum": 633054582.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5811239069429878, + "minimum": 0.5811239069429878, + "maximum": 0.5811239069429878 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 5796436.06793241, + "minimum": 5796436.06793241, + "maximum": 5796436.06793241 + }, + "files": { + "samples": 1, + "median": 73.0, + "minimum": 73.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 3368447.5741421315, + "minimum": 3368447.5741421315, + "maximum": 3368447.5741421315 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-medium", + "targetClass": "hpc", + "phase": "receiver-verify-and-extract", + "seconds": { + "samples": 1, + "median": 10.328649046, + "minimum": 10.328649046, + "maximum": 10.328649046 + }, + "logicalBytes": null, + "archiveBytes": { + "samples": 1, + "median": 367883152.0, + "minimum": 367883152.0, + "maximum": 367883152.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 633054582.0, + "minimum": 633054582.0, + "maximum": 633054582.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5811239069429878, + "minimum": 0.5811239069429878, + "maximum": 0.5811239069429878 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 61291131.02600426, + "minimum": 61291131.02600426, + "maximum": 61291131.02600426 + }, + "files": { + "samples": 1, + "median": 73.0, + "minimum": 73.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 35617741.52278617, + "minimum": 35617741.52278617, + "maximum": 35617741.52278617 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-micro", + "phase": "peer-reused-source-manifest-check", + "seconds": { + "samples": 4, + "median": 0.66461273, + "minimum": 0.495965208, + "maximum": 0.82791425 + }, + "logicalBytes": { + "samples": 4, + "median": 414996542.0, + "minimum": 105027840.0, + "maximum": 633054582.0 + }, + "archiveBytes": { + "samples": 4, + "median": 255659943.5, + "minimum": 142917128.0, + "maximum": 367883152.0 + }, + "workspaceBytes": { + "samples": 4, + "median": 414996542.0, + "minimum": 195579142.0, + "maximum": 633054582.0 + }, + "archiveToWorkspaceRatio": { + "samples": 4, + "median": 0.6534090518006972, + "minimum": 0.5811239069429878, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 4, + "median": 579489154.6585486, + "minimum": 392106576.9312505, + "maximum": 765512906.0901998 + }, + "files": { + "samples": 4, + "median": 60.0, + "minimum": 2.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 4, + "median": 366778296.9747619, + "minimum": 284549467.35060155, + "maximum": 444857850.8024174 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-medium", + "phase": "peer-authenticated-connectivity-probe", + "seconds": { + "samples": 1, + "median": 4.352586419, + "minimum": 4.352586419, + "maximum": 4.352586419 + }, + "logicalBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveBytes": { + "samples": 1, + "median": 143436735.0, + "minimum": 143436735.0, + "maximum": 143436735.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.7333948473912417, + "minimum": 0.7333948473912417, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 44934005.479191385, + "minimum": 44934005.479191385, + "maximum": 44934005.479191385 + }, + "files": { + "samples": 1, + "median": 47.0, + "minimum": 47.0, + "maximum": 47.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 32954368.091088787, + "minimum": 32954368.091088787, + "maximum": 32954368.091088787 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-medium", + "phase": "peer-snapshot-acquire", + "seconds": { + "samples": 1, + "median": 15.533482091, + "minimum": 15.533482091, + "maximum": 15.533482091 + }, + "logicalBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveBytes": { + "samples": 1, + "median": 143436735.0, + "minimum": 143436735.0, + "maximum": 143436735.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.7333948473912417, + "minimum": 0.7333948473912417, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 12590811.310318973, + "minimum": 12590811.310318973, + "maximum": 12590811.310318973 + }, + "files": { + "samples": 1, + "median": 47.0, + "minimum": 47.0, + "maximum": 47.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 9234036.139463304, + "minimum": 9234036.139463304, + "maximum": 9234036.139463304 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-medium", + "phase": "peer-private-upload", + "seconds": { + "samples": 1, + "median": 5.817277252, + "minimum": 5.817277252, + "maximum": 5.817277252 + }, + "logicalBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveBytes": { + "samples": 1, + "median": 143436735.0, + "minimum": 143436735.0, + "maximum": 143436735.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.7333948473912417, + "minimum": 0.7333948473912417, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 33620392.07135249, + "minimum": 33620392.07135249, + "maximum": 33620392.07135249 + }, + "files": { + "samples": 1, + "median": 47.0, + "minimum": 47.0, + "maximum": 47.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 24657022.312403273, + "minimum": 24657022.312403273, + "maximum": 24657022.312403273 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-medium", + "phase": "peer-receiver-verify-and-extract", + "seconds": { + "samples": 1, + "median": 5.928769003, + "minimum": 5.928769003, + "maximum": 5.928769003 + }, + "logicalBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveBytes": { + "samples": 1, + "median": 143436735.0, + "minimum": 143436735.0, + "maximum": 143436735.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.7333948473912417, + "minimum": 0.7333948473912417, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 32988153.510625146, + "minimum": 32988153.510625146, + "maximum": 32988153.510625146 + }, + "files": { + "samples": 1, + "median": 47.0, + "minimum": 47.0, + "maximum": 47.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 24193341.809643783, + "minimum": 24193341.809643783, + "maximum": 24193341.809643783 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + } + ], + "consumerMergeObservations": [ + { + "activityFamily": "madd", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 3, + "median": 236.587417607, + "minimum": 66.450399947, + "maximum": 243.573326361 + }, + "checkSeconds": { + "samples": 3, + "median": 142.49716386900036, + "minimum": 25.46066779800003, + "maximum": 145.75149784399946 + }, + "copySeconds": { + "samples": 3, + "median": 53.56884224699934, + "minimum": 33.81956537900078, + "maximum": 63.3139910280006 + }, + "verifySeconds": { + "samples": 3, + "median": 35.92816998099988, + "minimum": 6.668966102999548, + "maximum": 37.1964132789999 + } + }, + { + "activityFamily": "mbackground", + "resourceClass": "e2-medium", + "mergeSeconds": { + "samples": 2, + "median": 4.83583319, + "minimum": 4.176778294, + "maximum": 5.494888086 + }, + "checkSeconds": { + "samples": 2, + "median": 1.6694268080004804, + "minimum": 1.6326302810002744, + "maximum": 1.7062233350006863 + }, + "copySeconds": { + "samples": 2, + "median": 2.1372344224996596, + "minimum": 1.4865193229998113, + "maximum": 2.787949521999508 + }, + "verifySeconds": { + "samples": 2, + "median": 0.5077693425000689, + "minimum": 0.5032371529996453, + "maximum": 0.5123015320004924 + } + }, + { + "activityFamily": "mbackground", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 8, + "median": 19.92376553, + "minimum": 19.626282426, + "maximum": 138.21304698 + }, + "checkSeconds": { + "samples": 8, + "median": 5.024046718499903, + "minimum": 4.7745690230003675, + "maximum": 53.48430508499996 + }, + "copySeconds": { + "samples": 8, + "median": 10.670739221000076, + "minimum": 10.32359564699982, + "maximum": 40.23239413300007 + }, + "verifySeconds": { + "samples": 8, + "median": 3.8808005935002257, + "minimum": 3.5650197659997502, + "maximum": 44.77779699300004 + } + }, + { + "activityFamily": "mbackground", + "resourceClass": "hpc", + "mergeSeconds": { + "samples": 2, + "median": 22.751472198000002, + "minimum": 21.729738135, + "maximum": 23.773206261 + }, + "checkSeconds": { + "samples": 2, + "median": 9.362968396861106, + "minimum": 8.317466377280653, + "maximum": 10.40847041644156 + }, + "copySeconds": { + "samples": 2, + "median": 5.138671131338924, + "minimum": 4.891942439600825, + "maximum": 5.385399823077023 + }, + "verifySeconds": { + "samples": 2, + "median": 7.284067759755999, + "minimum": 7.006542180664837, + "maximum": 7.56159333884716 + } + }, + { + "activityFamily": "mbgmodel", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 2, + "median": 16.8738145285, + "minimum": 16.747609883, + "maximum": 17.000019174 + }, + "checkSeconds": { + "samples": 2, + "median": 3.261755618000052, + "minimum": 2.698455793999983, + "maximum": 3.825055442000121 + }, + "copySeconds": { + "samples": 2, + "median": 9.35986830599984, + "minimum": 9.022188792999941, + "maximum": 9.697547818999738 + }, + "verifySeconds": { + "samples": 2, + "median": 3.6908348800000113, + "minimum": 3.6229606209999474, + "maximum": 3.758709139000075 + } + }, + { + "activityFamily": "mbgmodel", + "resourceClass": "e2-medium", + "mergeSeconds": { + "samples": 1, + "median": 3.603590835, + "minimum": 3.603590835, + "maximum": 3.603590835 + }, + "checkSeconds": { + "samples": 1, + "median": 0.5051906540002165, + "minimum": 0.5051906540002165, + "maximum": 0.5051906540002165 + }, + "copySeconds": { + "samples": 1, + "median": 2.142047676000402, + "minimum": 2.142047676000402, + "maximum": 2.142047676000402 + }, + "verifySeconds": { + "samples": 1, + "median": 0.4960838049992162, + "minimum": 0.4960838049992162, + "maximum": 0.4960838049992162 + } + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "hpc", + "mergeSeconds": { + "samples": 1, + "median": 51.168297815, + "minimum": 51.168297815, + "maximum": 51.168297815 + }, + "checkSeconds": { + "samples": 1, + "median": 29.774229486472905, + "minimum": 29.774229486472905, + "maximum": 29.774229486472905 + }, + "copySeconds": { + "samples": 1, + "median": 10.907653647474945, + "minimum": 10.907653647474945, + "maximum": 10.907653647474945 + }, + "verifySeconds": { + "samples": 1, + "median": 9.319769462570548, + "minimum": 9.319769462570548, + "maximum": 9.319769462570548 + } + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 2, + "median": 129.158736309, + "minimum": 114.001598677, + "maximum": 144.315873941 + }, + "checkSeconds": { + "samples": 2, + "median": 65.35185085299986, + "minimum": 64.67154081999979, + "maximum": 66.03216088599993 + }, + "copySeconds": { + "samples": 2, + "median": 33.53280334749991, + "minimum": 26.555731558999923, + "maximum": 40.50987513599989 + }, + "verifySeconds": { + "samples": 2, + "median": 29.360318618000065, + "minimum": 22.22765749600012, + "maximum": 36.49297974000001 + } + }, + { + "activityFamily": "mdifffit", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 14, + "median": 8.3427407115, + "minimum": 5.830948001, + "maximum": 38.347655976 + }, + "checkSeconds": { + "samples": 14, + "median": 2.511359688500079, + "minimum": 1.1038579400000117, + "maximum": 15.438960124000005 + }, + "copySeconds": { + "samples": 14, + "median": 4.124673134000005, + "minimum": 2.378768721999677, + "maximum": 13.515580894999857 + }, + "verifySeconds": { + "samples": 14, + "median": 1.6997197335000465, + "minimum": 1.0764374750000059, + "maximum": 13.448123979999764 + } + }, + { + "activityFamily": "mdifffit", + "resourceClass": "hpc", + "mergeSeconds": { + "samples": 3, + "median": 7.47857792, + "minimum": 6.511636503, + "maximum": 10.95796063 + }, + "checkSeconds": { + "samples": 3, + "median": 1.7308137826621532, + "minimum": 1.710929460823536, + "maximum": 3.6601696284487844 + }, + "copySeconds": { + "samples": 3, + "median": 2.0938274255022407, + "minimum": 1.2903437670320272, + "maximum": 2.483007412403822 + }, + "verifySeconds": { + "samples": 3, + "median": 2.644647597335279, + "minimum": 2.5150117529556155, + "maximum": 2.732031609863043 + } + }, + { + "activityFamily": "mdifffit", + "resourceClass": "e2-medium", + "mergeSeconds": { + "samples": 1, + "median": 3.536212793, + "minimum": 3.536212793, + "maximum": 3.536212793 + }, + "checkSeconds": { + "samples": 1, + "median": 0.7207421489999888, + "minimum": 0.7207421489999888, + "maximum": 0.7207421489999888 + }, + "copySeconds": { + "samples": 1, + "median": 1.7899585740000248, + "minimum": 1.7899585740000248, + "maximum": 1.7899585740000248 + }, + "verifySeconds": { + "samples": 1, + "median": 0.5052870060000032, + "minimum": 0.5052870060000032, + "maximum": 0.5052870060000032 + } + }, + { + "activityFamily": "mimgtbl", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 3, + "median": 53.028995149, + "minimum": 52.357549732, + "maximum": 131.204556309 + }, + "checkSeconds": { + "samples": 3, + "median": 18.724648188000174, + "minimum": 18.695452503000524, + "maximum": 55.52439420599967 + }, + "copySeconds": { + "samples": 3, + "median": 27.288482144000227, + "minimum": 27.0868461150003, + "maximum": 57.700238978000016 + }, + "verifySeconds": { + "samples": 3, + "median": 6.4975390220006375, + "minimum": 6.045547994999652, + "maximum": 16.76125427399984 + } + }, + { + "activityFamily": "mviewer", + "resourceClass": "e2-medium", + "mergeSeconds": { + "samples": 2, + "median": 30.185459264000002, + "minimum": 7.251629086, + "maximum": 53.119289442 + }, + "checkSeconds": { + "samples": 2, + "median": 4.712208465500225, + "minimum": 0.8757597130006616, + "maximum": 8.54865721799979 + }, + "copySeconds": { + "samples": 2, + "median": 17.59759918700047, + "minimum": 5.020064412000465, + "maximum": 30.175133962000473 + }, + "verifySeconds": { + "samples": 2, + "median": 7.402032238000174, + "minimum": 0.8792642919997888, + "maximum": 13.92480018400056 + } + }, + { + "activityFamily": "mviewer", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 2, + "median": 42.396445477, + "minimum": 30.327887263, + "maximum": 54.465003691 + }, + "checkSeconds": { + "samples": 2, + "median": 7.072915739999644, + "minimum": 6.825928993999696, + "maximum": 7.319902485999592 + }, + "copySeconds": { + "samples": 2, + "median": 16.937191215999974, + "minimum": 16.770350947000225, + "maximum": 17.104031484999723 + }, + "verifySeconds": { + "samples": 2, + "median": 17.893067211000016, + "minimum": 6.250859729000695, + "maximum": 29.535274692999337 + } + } + ], + "mergeRateObservations": [ + { + "targetClass": "e2-micro", + "source": "baseline:montage-docker-image-real-20261010-v2", + "observations": { + "checkSecondsPerInputByte": { + "samples": 34, + "median": 6.35527429607893e-09, + "minimum": 2.8968348057689955e-09, + "maximum": 7.236672806194324e-08 + }, + "copySecondsPerUniqueByte": { + "samples": 34, + "median": 1.699528122472209e-08, + "minimum": 7.844377436225745e-09, + "maximum": 6.452023672950758e-08 + }, + "verifySecondsPerUniqueByte": { + "samples": 34, + "median": 6.138832298139974e-09, + "minimum": 3.5569571742792556e-09, + "maximum": 7.073291666499625e-08 + } + } + }, + { + "targetClass": "e2-medium", + "source": "baseline:montage-docker-image-real-20261010-v2", + "observations": { + "checkSecondsPerInputByte": { + "samples": 6, + "median": 1.912526350228284e-09, + "minimum": 7.876752557337657e-10, + "maximum": 2.5742072137899076e-09 + }, + "copySecondsPerUniqueByte": { + "samples": 6, + "median": 4.454356662161575e-09, + "minimum": 2.3675786107344793e-09, + "maximum": 9.610568101437293e-09 + }, + "verifySecondsPerUniqueByte": { + "samples": 6, + "median": 8.04424495753747e-10, + "minimum": 7.83635464283875e-10, + "maximum": 4.434950997591924e-09 + } + } + }, + { + "targetClass": "hpc", + "source": "baseline:montage-docker-image-real-20261010-v2", + "observations": { + "checkSecondsPerInputByte": { + "samples": 6, + "median": 9.649068875587286e-09, + "minimum": 5.653559042397685e-09, + "maximum": 1.5362090273490482e-08 + }, + "copySecondsPerUniqueByte": { + "samples": 6, + "median": 7.94870229763006e-09, + "minimum": 6.085461673088312e-09, + "maximum": 1.718957664580683e-08 + }, + "verifySecondsPerUniqueByte": { + "samples": 6, + "median": 1.1464509411470224e-08, + "minimum": 8.721156347585568e-09, + "maximum": 1.4687200077643863e-08 + } + } + } + ], + "limitations": [ + "Single baseline; ranges are observations, not confidence intervals.", + "Do not replay per-activity durations or total makespan as a holdout prediction.", + "No destination cache hit or shared-storage route may be assumed without evidence.", + "Workspace manifests and topology-specific contention are required before applying these observations.", + "Compute samples are contention-conditioned; do not apply interference slowdown a second time." + ] +} diff --git a/experiments/montage-micro-hpc-e2/fixed-hpc-stage-holdout-20261011.json b/experiments/montage-micro-hpc-e2/fixed-hpc-stage-holdout-20261011.json new file mode 100644 index 00000000..ebbcf523 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/fixed-hpc-stage-holdout-20261011.json @@ -0,0 +1,27 @@ +{ + "issue":"AKO-165", + "scope":"fixed-58-activity-HPC-placement; shared model supports class-specific stages across runtimes", + "trainingRun":"montage-prism-all-hpc-real-20261011-v1", + "validationRun":"montage-prism-all-hpc-real-20261011-v2", + "holdoutUsedForCalibration":false, + "newRealSubmitted":false, + "computeChanged":false, + "placementChanged":false, + "previousSimulatedMakespanSeconds":731.4037441142812, + "stageCalibratedSimulatedMakespanSeconds":1021.6136364533885, + "observedHoldoutMakespanSeconds":1296.0180189609528, + "observedHoldoutRunWindowSeconds":1338, + "previousAbsolutePredictionErrorPercent":43.565359, + "stageCalibratedAbsolutePredictionErrorPercent":21.172941, + "activitiesWithin20PercentBefore":12, + "activitiesWithin20PercentAfter":26, + "medianActivityWallErrorPercentAfter":22.575879938992422, + "gateAccepted":false, + "limitations":[ + "Effective preparation residual and publication measurements still contain unidentified waiting/poll delays; not queue-free service calibration.", + "Gateway fallbacks and scheduler waits are not injected into baseline internal-route forecast.", + "Per-runtime/class calibration parameters must not be borrowed across unmeasured routes or runtimes.", + "V1 recovered/multi-handle activities excluded; 28 merge and preparation observations and 34 publication observations used.", + "Shared network model remains unchanged; no global makespan multiplier applied." + ] +} diff --git a/experiments/montage-micro-hpc-e2/input-compute-work-v1.json b/experiments/montage-micro-hpc-e2/input-compute-work-v1.json new file mode 100644 index 00000000..c265042a --- /dev/null +++ b/experiments/montage-micro-hpc-e2/input-compute-work-v1.json @@ -0,0 +1,142 @@ +{ + "schemaVersion": 1, + "sourceImageSHA256": "8f3fc1a46239f5f83a03a0e2e99c06647a85c6e896c944884d27086c518e071d", + "basis": "immutable-input-FITS-header", + "inputs": [ + { + "path": "poss2ukstu_blue_001_001.fits", + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14", + "compressedBytes": 9655401, + "band": "blue", + "unit": "input-megapixels", + "computeWorkUnits": 6.418616, + "naxis1": 2531, + "naxis2": 2536, + "bitpix": 16 + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883", + "compressedBytes": 8587351, + "band": "blue", + "unit": "input-megapixels", + "computeWorkUnits": 6.418616, + "naxis1": 2531, + "naxis2": 2536, + "bitpix": 16 + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0", + "compressedBytes": 8814262, + "band": "blue", + "unit": "input-megapixels", + "computeWorkUnits": 6.418616, + "naxis1": 2531, + "naxis2": 2536, + "bitpix": 16 + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247", + "compressedBytes": 8238776, + "band": "blue", + "unit": "input-megapixels", + "computeWorkUnits": 6.418616, + "naxis1": 2531, + "naxis2": 2536, + "bitpix": 16 + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21", + "compressedBytes": 7327980, + "band": "ir", + "unit": "input-megapixels", + "computeWorkUnits": 6.418616, + "naxis1": 2531, + "naxis2": 2536, + "bitpix": 16 + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f", + "compressedBytes": 7167256, + "band": "ir", + "unit": "input-megapixels", + "computeWorkUnits": 6.418616, + "naxis1": 2531, + "naxis2": 2536, + "bitpix": 16 + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97", + "compressedBytes": 7217490, + "band": "ir", + "unit": "input-megapixels", + "computeWorkUnits": 6.418616, + "naxis1": 2531, + "naxis2": 2536, + "bitpix": 16 + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f", + "compressedBytes": 7120717, + "band": "ir", + "unit": "input-megapixels", + "computeWorkUnits": 6.418616, + "naxis1": 2531, + "naxis2": 2536, + "bitpix": 16 + }, + { + "path": "poss2ukstu_red_001_001.fits", + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d", + "compressedBytes": 6900714, + "band": "red", + "unit": "input-megapixels", + "computeWorkUnits": 6.418616, + "naxis1": 2531, + "naxis2": 2536, + "bitpix": 16 + }, + { + "path": "poss2ukstu_red_001_002.fits", + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944", + "compressedBytes": 6494409, + "band": "red", + "unit": "input-megapixels", + "computeWorkUnits": 6.418616, + "naxis1": 2531, + "naxis2": 2536, + "bitpix": 16 + }, + { + "path": "poss2ukstu_red_002_001.fits", + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1", + "compressedBytes": 6570387, + "band": "red", + "unit": "input-megapixels", + "computeWorkUnits": 6.418616, + "naxis1": 2531, + "naxis2": 2536, + "bitpix": 16 + }, + { + "path": "poss2ukstu_red_002_002.fits", + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e", + "compressedBytes": 6376872, + "band": "red", + "unit": "input-megapixels", + "computeWorkUnits": 6.418616, + "naxis1": 2531, + "naxis2": 2536, + "bitpix": 16 + } + ], + "limitations": [ + "Pixel count is input work, not measured CPU service demand; band and hardware may need separate calibrated coefficients." + ] +} diff --git a/experiments/montage-micro-hpc-e2/input-observation-reference-20261011.json b/experiments/montage-micro-hpc-e2/input-observation-reference-20261011.json new file mode 100644 index 00000000..5a0afca2 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/input-observation-reference-20261011.json @@ -0,0 +1,19 @@ +{ + "issue": "AKO-158", + "referenceRunId": "montage-prism-transfer-real-20261011-v2", + "role": "recovered-reference-not-independent-validation", + "measurement": "remoteWorkspaceSnapshotSeconds / artifactObservationBefore unique file bytes", + "completedCloudSamples": 40, + "weightedEffectiveSecondsPerByte": 2.6495730268133445e-8, + "minimumEffectiveSecondsPerByte": 2.377949579117742e-9, + "maximumEffectiveSecondsPerByte": 7.50483825275962e-8, + "summedWorkSecondsNotMakespan": 707.4212270280001, + "serviceCalibrationAccepted": false, + "limitations": [ + "Snapshot wall time may include SSH admission and fixed command overhead; this is not yet a queue-free service coefficient.", + "The approximately 32-fold range requires examination by operation, file count, cache and resource before selecting a service profile.", + "Parallel sample durations are work totals, not elapsed workflow time.", + "Terminal artifact observation and inherited-input pruning remain separate from this pre-command snapshot measurement.", + "A fresh independent run is required after calibration and forecast freezing." + ] +} diff --git a/experiments/montage-micro-hpc-e2/input-observation-sensitivity-forecast-20261011.json b/experiments/montage-micro-hpc-e2/input-observation-sensitivity-forecast-20261011.json new file mode 100644 index 00000000..39694b53 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/input-observation-sensitivity-forecast-20261011.json @@ -0,0 +1,18 @@ +{ + "issue":"AKO-158", + "role":"exploratory-wall-time-sensitivity-not-service-calibration", + "baselinePrismSeconds":3127.9222412902786, + "baselineSimGridSeconds":3163.497678841343, + "inputObservationPrismSeconds":3310.94542475303, + "inputObservationSimGridSeconds":3376.197861747659, + "fixedOriginalPlacementInputObservationSimGridSeconds":3376.197861747659, + "fixedPlacementDeltaSeconds":212.700182906316, + "simulatedActivityCount":58, + "independentRealValidation":false, + "limitations":[ + "Input scan coefficient includes unknown SSH wait; this is sensitivity, not an accepted service fit.", + "PRISM was allowed to choose a new placement, so delta is not solely a fixed-position cost comparison.", + "Terminal observation, pruning and residual transfer preparation must still be modeled and calibrated.", + "No real run was launched from this sensitivity forecast." + ] +} diff --git a/experiments/montage-micro-hpc-e2/materialization-model-v2.json b/experiments/montage-micro-hpc-e2/materialization-model-v2.json new file mode 100644 index 00000000..a5ab3050 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/materialization-model-v2.json @@ -0,0 +1,182 @@ +{ + "schemaVersion": 1, + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "relayResourceId": "montage-relay-controller", + "resources": { + "montage-relay-controller": { + "class": "relay", + "domain": "relay" + }, + "montage-gcp-e2-medium-1": { + "class": "e2-medium", + "domain": "cloud" + }, + "montage-gcp-e2-medium-2": { + "class": "e2-medium", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-01": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-02": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-03": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-04": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-05": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-06": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-07": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-08": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-09": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-10": { + "class": "e2-micro", + "domain": "cloud" + }, + "plafrim-execution-ssh-node-bora001": { + "class": "hpc", + "domain": "hpc" + }, + "plafrim-execution-ssh-node-diablo03": { + "class": "hpc", + "domain": "hpc" + } + }, + "profiles": [ + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-micro", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "samples": 20, + "archiveToWorkspaceRatio": 0.7256941966584066, + "uploadArchiveBytesPerSecond": 39840579.06897718, + "probeSeconds": 2.4162210225000003, + "snapshotSecondsPerByte": 3.2776015672924614e-08, + "receiverSecondsPerByte": 2.18017188670382e-08 + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "samples": 45, + "archiveToWorkspaceRatio": 0.578327721631775, + "uploadArchiveBytesPerSecond": 17623724.802639503, + "probeSeconds": 2.444578959, + "snapshotSecondsPerByte": 7.357538694468153e-08, + "receiverSecondsPerByte": 2.221306294384161e-08 + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "samples": 11, + "archiveToWorkspaceRatio": 0.5736604862310096, + "uploadArchiveBytesPerSecond": 8139313.38762742, + "ingressArchiveBytesPerSecond": 4565984.029781632, + "ingressIncludesSourcePreparation": true, + "validationSecondsPerByte": 5.167299861100892e-09, + "receiverSecondsPerByte": 2.1368381960668018e-08 + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-medium", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "samples": 7, + "archiveToWorkspaceRatio": 0.5616915015784198, + "uploadArchiveBytesPerSecond": 22607894.084331468, + "probeSeconds": 3.238551043, + "snapshotSecondsPerByte": 3.255837026888948e-07, + "receiverSecondsPerByte": 1.0530734853051159e-08 + }, + { + "strategy": "gateway", + "sourceClass": "e2-micro", + "targetClass": "hpc", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "samples": 10, + "archiveToWorkspaceRatio": 0.6522022259028002, + "uploadArchiveBytesPerSecond": 3704139.690952878, + "ingressArchiveBytesPerSecond": 7741110.27791067, + "ingressIncludesSourcePreparation": true, + "validationSecondsPerByte": 5.561600398112477e-09, + "receiverSecondsPerByte": 2.166145456945102e-08 + }, + { + "strategy": "gateway", + "sourceClass": "e2-medium", + "targetClass": "hpc", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "samples": 1, + "archiveToWorkspaceRatio": 0.5811239069429878, + "uploadArchiveBytesPerSecond": 3368447.5741421315, + "ingressArchiveBytesPerSecond": 12094092.529896256, + "ingressIncludesSourcePreparation": true, + "validationSecondsPerByte": 5.291803775302269e-09, + "receiverSecondsPerByte": 1.631557426433729e-08 + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-medium", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "samples": 1, + "archiveToWorkspaceRatio": 0.7333948473912417, + "uploadArchiveBytesPerSecond": 24657022.312403273, + "probeSeconds": 4.352586419, + "snapshotSecondsPerByte": 7.94229994679085e-08, + "receiverSecondsPerByte": 3.0313912528566057e-08 + } + ], + "mergeProfiles": [ + { + "targetClass": "e2-micro", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "samples": 34, + "checkSecondsPerInputByte": 6.35527429607893e-09, + "copySecondsPerUniqueByte": 1.699528122472209e-08, + "verifySecondsPerUniqueByte": 6.138832298139974e-09 + }, + { + "targetClass": "e2-medium", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "samples": 6, + "checkSecondsPerInputByte": 1.912526350228284e-09, + "copySecondsPerUniqueByte": 4.454356662161575e-09, + "verifySecondsPerUniqueByte": 8.04424495753747e-10 + }, + { + "targetClass": "hpc", + "source": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "samples": 6, + "checkSecondsPerInputByte": 9.649068875587286e-09, + "copySecondsPerUniqueByte": 7.94870229763006e-09, + "verifySecondsPerUniqueByte": 1.1464509411470224e-08 + } + ] +} diff --git a/experiments/montage-micro-hpc-e2/materialization-resource-model.json b/experiments/montage-micro-hpc-e2/materialization-resource-model.json new file mode 100644 index 00000000..88e421b1 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/materialization-resource-model.json @@ -0,0 +1,17 @@ +{ + "montage-relay-controller": {"class": "relay", "domain": "relay"}, + "montage-gcp-e2-medium-1": {"class": "e2-medium", "domain": "cloud"}, + "montage-gcp-e2-medium-2": {"class": "e2-medium", "domain": "cloud"}, + "montage-gcp-fog-e2-micro-01": {"class": "e2-micro", "domain": "cloud"}, + "montage-gcp-fog-e2-micro-02": {"class": "e2-micro", "domain": "cloud"}, + "montage-gcp-fog-e2-micro-03": {"class": "e2-micro", "domain": "cloud"}, + "montage-gcp-fog-e2-micro-04": {"class": "e2-micro", "domain": "cloud"}, + "montage-gcp-fog-e2-micro-05": {"class": "e2-micro", "domain": "cloud"}, + "montage-gcp-fog-e2-micro-06": {"class": "e2-micro", "domain": "cloud"}, + "montage-gcp-fog-e2-micro-07": {"class": "e2-micro", "domain": "cloud"}, + "montage-gcp-fog-e2-micro-08": {"class": "e2-micro", "domain": "cloud"}, + "montage-gcp-fog-e2-micro-09": {"class": "e2-micro", "domain": "cloud"}, + "montage-gcp-fog-e2-micro-10": {"class": "e2-micro", "domain": "cloud"}, + "plafrim-execution-ssh-node-bora001": {"class": "hpc", "domain": "hpc"}, + "plafrim-execution-ssh-node-diablo03": {"class": "hpc", "domain": "hpc"} +} diff --git a/experiments/montage-micro-hpc-e2/medium-holdout-cost-review-20261011.json b/experiments/montage-micro-hpc-e2/medium-holdout-cost-review-20261011.json new file mode 100644 index 00000000..98fb6281 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/medium-holdout-cost-review-20261011.json @@ -0,0 +1,25 @@ +{ + "issue":"AKO-168", + "status":"prepared-only-awaiting-human-authorization", + "realAuthorized":false, + "realSubmitted":false, + "providerMachineType":"e2-medium", + "vmCount":1, + "executionLanes":1, + "vcpu":2, + "memoryMiB":4096, + "diskType":"pd-balanced", + "diskSizeGiB":30, + "plannedAndSimulatedSeconds":10869.913425723089, + "registeredPriceObservation":"2026-10-10T21:57:24.147279919Z", + "registeredPriceSource":"gcp-public-catalog", + "registeredPricePerHourUSD":0.03350571, + "registeredDiskPricePerGiBMonthUSD":0.1, + "historicalComputeEstimateUSD":0.10116782415760676, + "historicalDiskEstimateUSDApprox":0.0124086, + "historicalSubtotalUSDApprox":0.1135764, + "exclusions":["Provisioning, SSH/configuration/validation, stop and export idle time not included in this subtotal.","Actual price and network/public-IP billing must be revalidated before real.","Physical disk headroom must be verified after provisioning; nominal disk size is not free space."], + "readOnlyPreflight":{"workflowRunsActive":0,"nonDestroyedCloudInstances":0,"targetEnabledInCatalog":true}, + "goalComplete":false, + "remainingBlocker":"Independent medium validation requires a new real reference; explicit authorization has been requested but not received. Automatic goal continuation is not authorization." +} diff --git a/experiments/montage-micro-hpc-e2/native-stage-readiness-20261011.json b/experiments/montage-micro-hpc-e2/native-stage-readiness-20261011.json new file mode 100644 index 00000000..fcee75ad --- /dev/null +++ b/experiments/montage-micro-hpc-e2/native-stage-readiness-20261011.json @@ -0,0 +1,22 @@ +{ + "issue":"AKO-168", + "newRealSubmitted":false, + "daemonRestarted":false, + "schemaContract":{"legacyMaterializationVersion":1,"calibratedStageVersion":2,"requiredAdmissionVersion":1}, + "productionExecutorVerification":{ + "hpc":{"activities":58,"plannedSeconds":1164.2299096115516,"verifiedNativeSeconds":1164.2299096115516,"capabilityAndTraceAcknowledgementPassed":true}, + "mediumPrepared":{"activities":58,"vmCount":1,"executionLanes":1,"plannedSeconds":10869.913425723089,"verifiedNativeSeconds":10869.913425723089,"historicalComputeOnlyCostUSD":0.10116782415760676,"realAuthorized":false,"realSubmitted":false,"remaining":"User approval, current pricing, lifecycle/disk cost and physical preflight required."} + }, + "legacyRunnerProbe":{"workspaceMaterializationModelVersion":1,"bulkServicePoolModelVersion":1,"runtimeAdmissionModelVersion":null}, + "legacyNativeExecutionRejectedBeforeSimulation":true, + "currentlyDeployedRunnerStageUpgradeApplied":false, + "coverageAudit":{ + "cleanWholeRunReferences":["montage-docker-image-real-20261010-v2","montage-clean-real-20261011-v1","montage-prism-all-hpc-real-20261011-v2"], + "e2MediumCleanReferenceCount":1, + "e2MediumCleanReferenceAlreadyUsedForTraining":true, + "e2MediumIndependentCleanHoldoutAvailable":false, + "recoveredCloudReferencesExcludedFromCleanMakespanValidation":true + }, + "goalComplete":false, + "remaining":["Independent medium validation requires explicit authorization for a new real run.","New runner and Go model must be applied together at a safe point before API-based stage-calibrated simulation.","Dominant preparation/publication residuals remain despite aggregate HPC/mixed cases meeting tolerance."] +} diff --git a/experiments/montage-micro-hpc-e2/peer-phase-concurrency-reference-20261011.json b/experiments/montage-micro-hpc-e2/peer-phase-concurrency-reference-20261011.json new file mode 100644 index 00000000..58289f76 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/peer-phase-concurrency-reference-20261011.json @@ -0,0 +1,16 @@ +{ + "issue":"AKO-158", + "referenceRunId":"montage-prism-transfer-real-20261011-v2", + "sourceResource":"montage-gcp-fog-e2-micro-07", + "sourceSnapshotAcquirePhases":5, + "maximumOverlappingSourceSnapshotAcquirePhases":4, + "targetResource":"montage-gcp-fog-e2-micro-08", + "targetReceiverPhases":7, + "maximumOverlappingTargetReceiverPhases":4, + "limitations":[ + "Overlap is based on observed phase intervals, not proof of simultaneous CPU execution.", + "Snapshot acquire includes cache waits, SSH admission and command execution; existing wall time cannot identify these separately.", + "Snapshot build now records SSH admission and command wall time separately for prospective measurements.", + "Command wall time still includes SSH connection, remote CPU and I/O; it is not a CPU service measurement." + ] +} diff --git a/experiments/montage-micro-hpc-e2/peer-preparation-independent-residuals-20261011.json b/experiments/montage-micro-hpc-e2/peer-preparation-independent-residuals-20261011.json new file mode 100644 index 00000000..fe7f1e71 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/peer-preparation-independent-residuals-20261011.json @@ -0,0 +1,47 @@ +{ + "issue": "AKO-156", + "runId": "montage-prism-transfer-real-20261011-v2", + "forecastFrozenBeforeRun": true, + "independentRunCompletedWithoutRecovery": false, + "predictionAccepted": false, + "observations": [ + { + "consumer": "mViewer38", + "attemptRole": "original", + "predictedDependencySeconds": 299.64458030969763, + "observedPreparationBarrierSeconds": 250.79265328, + "snapshotAcquireSeconds": 180.206241124, + "uploadSeconds": 41.190142059, + "receiverSeconds": 25.143159386 + }, + { + "consumer": "mViewer57", + "attemptRole": "original-interrupted-during-merge", + "predictedDependencySeconds": 302.08458283221216, + "observedPreparationBarrierSeconds": 518.800038942, + "snapshotAcquireSeconds": 439.081100283, + "uploadSeconds": 56.271185525, + "receiverSeconds": 19.305561591 + }, + { + "consumer": "mViewer57", + "attemptRole": "recovery-after-AKO-155", + "observedPreparationBarrierSeconds": 423.142513732, + "snapshotAcquireSeconds": 215.496415805, + "sourceManifestCheckSeconds": 70.065198658, + "snapshotReused": true, + "uploadSeconds": 44.300248394, + "receiverSeconds": 87.093432582, + "receiverSSHChannelWaitSeconds": 0.000046458, + "mergeSeconds": 49.722276564 + } + ], + "limitations": [ + "Preparation barrier is elapsed time, not a sum of parallel phases.", + "Snapshot acquisition includes build or waiting for a shared build; it is not exclusive service.", + "Receiver wall time may contain resource contention, even when SSH admission wait is negligible.", + "Recovery observations cannot validate independent full-run makespan.", + "No runtime speedup claim is justified by the extraction code change alone.", + "Original frozen forecasts must not be overwritten or refitted as validation results." + ] +} diff --git a/experiments/montage-micro-hpc-e2/pricing-20261010.json b/experiments/montage-micro-hpc-e2/pricing-20261010.json new file mode 100644 index 00000000..df31a889 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/pricing-20261010.json @@ -0,0 +1,25 @@ +{ + "issue": "AKO-140", + "currency": "USD", + "region": "us-central1", + "observedAt": "2026-10-10T21:57:24.147279919Z", + "source": "GCP Cloud Billing public SKU catalog, on-demand", + "machines": { + "e2-micro": {"pricePerHour": 0.0083764275, "billableCpuFraction": 0.25}, + "e2-medium": {"pricePerHour": 0.03350571, "billableCpuFraction": 1} + }, + "bootDisk": { + "type": "pd-balanced", + "sizeGiB": 30, + "pricePerGiBMonth": 0.1, + "skuId": "6AE1-525F-8B80", + "monthlyConversionHours": 730 + }, + "accounting": { + "real": "One concrete VM generation is counted once using its billable compute clock and disk lifetime. Stop pauses compute, not disk charges. Missing tariffs produce incomplete status and a null total.", + "basis": "Lifecycle estimate, not a provider invoice. Marginal list rates; no automatic free-tier or discount credits.", + "excluded": ["network-egress", "public-ipv4", "image-storage", "discounts-and-tax"], + "hpc": "No PlaFRIM monetary tariff was supplied; the cloud estimate does not assign a monetary cost to HPC allocation.", + "simulation": "The simulation task/window cost and the plan lifecycle cost currently use different retention windows; report both without treating them as equivalent to a provider invoice." + } +} diff --git a/experiments/montage-micro-hpc-e2/prism-independent-recovered-audit-20261011.json b/experiments/montage-micro-hpc-e2/prism-independent-recovered-audit-20261011.json new file mode 100644 index 00000000..37c1fa43 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/prism-independent-recovered-audit-20261011.json @@ -0,0 +1,30 @@ +{ + "runId": "montage-prism-transfer-real-20261011-v2", + "status": "completed", + "distinctLatestActivities": 58, + "currentFailedActivities": 0, + "activeHandles": 0, + "transferFinishedAfterLatestConsumerStartViolations": 0, + "product": { + "filename": "mosaic-color.png", + "bytes": 4920451, + "width": 1800, + "height": 1800, + "sha256": "63fa4d8447274bdb8a44f6be8cf5a23c0a2f0baa479bd2ae1ec3b541df4e19f2", + "verification": "read-only daemon console API stat, SHA-256 and file signature" + }, + "forecast": { + "prismMakespanSeconds": 3127.9222412902786, + "simgridMakespanSeconds": 3163.497678841343, + "frozenBeforeOriginalRun": true + }, + "predictionValidated": false, + "reason": "Original attempt failed at 56/58; completion required code change and recovery. Recovered run cannot validate independent makespan.", + "cleanupComplete": false, + "remainingWork": [ + "Export and checksum final products before destroying their VM disks.", + "Confirm all cloud operations terminal and no HPC jobs residual.", + "Measure preparation and artifact observation separately; consume evidenced services in both PRISM and SimGrid.", + "Freeze a new forecast and validate against another fresh independent real execution." + ] +} diff --git a/experiments/montage-micro-hpc-e2/prism-time-quota-aware-real-20261011.json b/experiments/montage-micro-hpc-e2/prism-time-quota-aware-real-20261011.json new file mode 100644 index 00000000..177d90ff --- /dev/null +++ b/experiments/montage-micro-hpc-e2/prism-time-quota-aware-real-20261011.json @@ -0,0 +1,92 @@ +{ + "issue": "AKO-154", + "realRunId": "montage-prism-transfer-real-20261011-v2", + "statusAtSubmission": "running", + "startedAt": "2026-10-11T17:17:35Z", + "forecastFrozenBeforeReal": true, + "previousForecastBlockedByStorageQuota": true, + "previousPlanSeconds": 2321.6725360502005, + "previousSimulationSeconds": 2368.118319561188, + "quotaAwarePlan": { + "makespanSeconds": 3127.9222412902786, + "cost": 0.03860991076813537, + "feasible": true + }, + "quotaAwareSimulation": { + "computeSeconds": 5108.837997198106, + "cost": 0.022520250649858534, + "feasible": true, + "interferenceSeconds": 0.0, + "makespanSeconds": 3163.497678841343, + "overheadSeconds": 0.0, + "queueSeconds": 4735.875838555146, + "transferSeconds": 25382.247772585157 + }, + "storageBudget": { + "storageId": "ssh-user-quota-dc75389d01af9cf7565a99bb2940c556c7ce1ac8df5c859a8ac82c59f3801d6a", + "availableBytes": 20091539456, + "observedAt": "2026-10-11T17:15:11.253167252Z", + "source": "live-posix-filesystem-and-user-quota-v1" + }, + "completedSimulationTasks": 58, + "distribution": [ + { + "resource": "montage-gcp-fog-e2-micro-01", + "activities": 1 + }, + { + "resource": "montage-gcp-fog-e2-micro-02", + "activities": 1 + }, + { + "resource": "montage-gcp-fog-e2-micro-03", + "activities": 1 + }, + { + "resource": "montage-gcp-fog-e2-micro-04", + "activities": 9 + }, + { + "resource": "montage-gcp-fog-e2-micro-05", + "activities": 2 + }, + { + "resource": "montage-gcp-fog-e2-micro-06", + "activities": 2 + }, + { + "resource": "montage-gcp-fog-e2-micro-07", + "activities": 9 + }, + { + "resource": "montage-gcp-fog-e2-micro-08", + "activities": 4 + }, + { + "resource": "montage-gcp-fog-e2-micro-09", + "activities": 2 + }, + { + "resource": "montage-gcp-fog-e2-micro-10", + "activities": 9 + }, + { + "resource": "plafrim-execution-ssh-node-bora001", + "activities": 9 + }, + { + "resource": "plafrim-execution-ssh-node-diablo03", + "activities": 9 + } + ], + "physicalNetworkBytes": 11366975738, + "limitations": [ + "Real result pending; not yet a completed comparison.", + "Whole-workspace model retains combined pack/ingress where separate measurements are unavailable.", + "A live effective HPC storage quota changes feasible placement and must accompany planning.", + "Previous calibration fit is not validation of this independent execution.", + "Planner active-window and native task costs use different billing bases.", + "Native task queue currently includes workspace preparation; final audit must separate service, admission and execution waiting rather than adding aggregate totals.", + "Micro and HPC class-profile extrapolations and missing route-class coverage remain explicit." + ] +} diff --git a/experiments/montage-micro-hpc-e2/prism-time-transfer-experimental-20261011.json b/experiments/montage-micro-hpc-e2/prism-time-transfer-experimental-20261011.json new file mode 100644 index 00000000..d77be3cb --- /dev/null +++ b/experiments/montage-micro-hpc-e2/prism-time-transfer-experimental-20261011.json @@ -0,0 +1,373 @@ +{ + "issue": "AKO-154", + "provenanceSHA256": { + "planningInput": "c865f0d5da403d268121c4b2b77040add6f498abbe84d94b5e5b160f38824073", + "nativeSimulationInput": "fae259a4751f26ba8dc2f7258ab99caa32742d6c84ce2c8025dadde490625c55", + "nativePlatform": "fec229513ff8618bbdcca0180a4b0a7999f168508743ab352221f3bc2b23b9f3", + "nativeResult": "4615d429057e0df8ad9570e3c2beaa73e33dd91ac28d81b23dad34f58198b7f3" + }, + "missingActivityIds": [], + "duplicateTasks": 0, + "microActivityOverlaps": 0, + "experimental": true, + "realExecutionStarted": false, + "daemonRestarted": false, + "algorithm": "prism-time", + "beamWidth": 120, + "candidateCount": 25, + "scopeSchedulableResources": 14, + "planPredicted": { + "makespanSeconds": 2321.6725360502005, + "cost": 0.008595049929251326, + "feasible": true + }, + "simulated": { + "computeSeconds": 5038.7502698898315, + "cost": 0.005018118661143563, + "feasible": true, + "interferenceSeconds": 0.0, + "makespanSeconds": 2368.118319561188, + "overheadSeconds": 0.0, + "queueSeconds": 1489.6036462550921, + "transferSeconds": 6783.000817146088 + }, + "previousPlacementSimulation": 2835.6021967566785, + "observedPreviousPlacementReal": 3150.6341450214386, + "simVsPlanPercent": 2.000531202820044, + "simVsPreviousPlacementPercent": -16.48622919428515, + "activities": 58, + "completed": 58, + "distribution": [ + { + "resource": "montage-gcp-fog-e2-micro-01", + "activities": 8 + }, + { + "resource": "montage-gcp-fog-e2-micro-02", + "activities": 9 + }, + { + "resource": "montage-gcp-fog-e2-micro-03", + "activities": 8 + }, + { + "resource": "plafrim-execution-ssh-node-bora001", + "activities": 33 + } + ], + "physicalNetworkBytes": 4150785071, + "previousPlacementPhysicalNetworkBytes": 4383304054, + "routes": [ + { + "strategy": "direct-runtime", + "count": 2 + }, + { + "strategy": "gateway", + "count": 15 + } + ], + "barrierViolations": 0, + "assignments": [ + { + "activityId": "montage-independent-work-v1-mprojectid0000003", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-1" + }, + { + "activityId": "montage-independent-work-v1-mprojectid0000004", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-2" + }, + { + "activityId": "montage-independent-work-v1-mprojectid0000001", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-3" + }, + { + "activityId": "montage-independent-work-v1-mprojectid0000002", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-4" + }, + { + "activityId": "montage-independent-work-v1-mprojectid0000022", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-5" + }, + { + "activityId": "montage-independent-work-v1-mprojectid0000023", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-6" + }, + { + "activityId": "montage-independent-work-v1-mprojectid0000020", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-7" + }, + { + "activityId": "montage-independent-work-v1-mprojectid0000021", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-8" + }, + { + "activityId": "montage-independent-work-v1-mprojectid0000041", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-9" + }, + { + "activityId": "montage-independent-work-v1-mprojectid0000039", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-10" + }, + { + "activityId": "montage-independent-work-v1-mprojectid0000040", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-11" + }, + { + "activityId": "montage-independent-work-v1-mprojectid0000042", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-12" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000005", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-13" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000009", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-14" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000010", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-15" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000007", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-16" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000008", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-17" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000024", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-18" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000025", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-19" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000029", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-20" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000028", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-21" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000026", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-22" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000027", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-23" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000043", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-24" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000047", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-25" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000048", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-26" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000046", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-27" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000045", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-28" + }, + { + "activityId": "montage-independent-work-v1-mconcatfitid0000030", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-29" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000044", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-30" + }, + { + "activityId": "montage-independent-work-v1-mconcatfitid0000049", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-31" + }, + { + "activityId": "montage-independent-work-v1-mdifffitid0000006", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-32" + }, + { + "activityId": "montage-independent-work-v1-mconcatfitid0000011", + "resourceId": "plafrim-execution-ssh-node-bora001", + "coreId": "plafrim-execution-ssh-node-bora001-core-33" + }, + { + "activityId": "montage-independent-work-v1-mbgmodelid0000031", + "resourceId": "montage-gcp-fog-e2-micro-01", + "coreId": "montage-gcp-fog-e2-micro-01-core-1" + }, + { + "activityId": "montage-independent-work-v1-mbackgroundid0000033", + "resourceId": "montage-gcp-fog-e2-micro-01", + "coreId": "montage-gcp-fog-e2-micro-01-core-1" + }, + { + "activityId": "montage-independent-work-v1-mbackgroundid0000032", + "resourceId": "montage-gcp-fog-e2-micro-01", + "coreId": "montage-gcp-fog-e2-micro-01-core-1" + }, + { + "activityId": "montage-independent-work-v1-mbackgroundid0000035", + "resourceId": "montage-gcp-fog-e2-micro-01", + "coreId": "montage-gcp-fog-e2-micro-01-core-1" + }, + { + "activityId": "montage-independent-work-v1-mbackgroundid0000034", + "resourceId": "montage-gcp-fog-e2-micro-01", + "coreId": "montage-gcp-fog-e2-micro-01-core-1" + }, + { + "activityId": "montage-independent-work-v1-mimgtblid0000036", + "resourceId": "montage-gcp-fog-e2-micro-01", + "coreId": "montage-gcp-fog-e2-micro-01-core-1" + }, + { + "activityId": "montage-independent-work-v1-maddid0000037", + "resourceId": "montage-gcp-fog-e2-micro-01", + "coreId": "montage-gcp-fog-e2-micro-01-core-1" + }, + { + "activityId": "montage-independent-work-v1-mbgmodelid0000012", + "resourceId": "montage-gcp-fog-e2-micro-02", + "coreId": "montage-gcp-fog-e2-micro-02-core-1" + }, + { + "activityId": "montage-independent-work-v1-mbgmodelid0000050", + "resourceId": "montage-gcp-fog-e2-micro-03", + "coreId": "montage-gcp-fog-e2-micro-03-core-1" + }, + { + "activityId": "montage-independent-work-v1-mbackgroundid0000014", + "resourceId": "montage-gcp-fog-e2-micro-02", + "coreId": "montage-gcp-fog-e2-micro-02-core-1" + }, + { + "activityId": "montage-independent-work-v1-mbackgroundid0000015", + "resourceId": "montage-gcp-fog-e2-micro-02", + "coreId": "montage-gcp-fog-e2-micro-02-core-1" + }, + { + "activityId": "montage-independent-work-v1-mbackgroundid0000052", + "resourceId": "montage-gcp-fog-e2-micro-03", + "coreId": "montage-gcp-fog-e2-micro-03-core-1" + }, + { + "activityId": "montage-independent-work-v1-mbackgroundid0000053", + "resourceId": "montage-gcp-fog-e2-micro-03", + "coreId": "montage-gcp-fog-e2-micro-03-core-1" + }, + { + "activityId": "montage-independent-work-v1-mbackgroundid0000051", + "resourceId": "montage-gcp-fog-e2-micro-03", + "coreId": "montage-gcp-fog-e2-micro-03-core-1" + }, + { + "activityId": "montage-independent-work-v1-mbackgroundid0000054", + "resourceId": "montage-gcp-fog-e2-micro-03", + "coreId": "montage-gcp-fog-e2-micro-03-core-1" + }, + { + "activityId": "montage-independent-work-v1-mimgtblid0000055", + "resourceId": "montage-gcp-fog-e2-micro-03", + "coreId": "montage-gcp-fog-e2-micro-03-core-1" + }, + { + "activityId": "montage-independent-work-v1-maddid0000056", + "resourceId": "montage-gcp-fog-e2-micro-03", + "coreId": "montage-gcp-fog-e2-micro-03-core-1" + }, + { + "activityId": "montage-independent-work-v1-mviewerid0000038", + "resourceId": "montage-gcp-fog-e2-micro-01", + "coreId": "montage-gcp-fog-e2-micro-01-core-1" + }, + { + "activityId": "montage-independent-work-v1-mviewerid0000057", + "resourceId": "montage-gcp-fog-e2-micro-03", + "coreId": "montage-gcp-fog-e2-micro-03-core-1" + }, + { + "activityId": "montage-independent-work-v1-mbackgroundid0000016", + "resourceId": "montage-gcp-fog-e2-micro-02", + "coreId": "montage-gcp-fog-e2-micro-02-core-1" + }, + { + "activityId": "montage-independent-work-v1-mbackgroundid0000013", + "resourceId": "montage-gcp-fog-e2-micro-02", + "coreId": "montage-gcp-fog-e2-micro-02-core-1" + }, + { + "activityId": "montage-independent-work-v1-mimgtblid0000017", + "resourceId": "montage-gcp-fog-e2-micro-02", + "coreId": "montage-gcp-fog-e2-micro-02-core-1" + }, + { + "activityId": "montage-independent-work-v1-maddid0000018", + "resourceId": "montage-gcp-fog-e2-micro-02", + "coreId": "montage-gcp-fog-e2-micro-02-core-1" + }, + { + "activityId": "montage-independent-work-v1-mviewerid0000019", + "resourceId": "montage-gcp-fog-e2-micro-02", + "coreId": "montage-gcp-fog-e2-micro-02-core-1" + }, + { + "activityId": "montage-independent-work-v1-mviewerid0000058", + "resourceId": "montage-gcp-fog-e2-micro-02", + "coreId": "montage-gcp-fog-e2-micro-02-core-1" + } + ], + "limitations": [ + "Experimental calibration fit; not independent real validation.", + "Only two materialization route classes evidenced in the current calibrated profile; missing route classes are not silently filled.", + "Source packing and ingress remain combined; shared capacity is auxiliary fitted service, not measured NIC bandwidth.", + "Compute micro profiles extrapolate to equivalent micro targets; physical interference is not independently calibrated.", + "PRISM raw confidence label does not validate empirical transfer model accuracy.", + "Planner active-window compute/disk cost and simulator task cost are different billing bases; network/IP/HPC billing incomplete.", + "Transfer and queue totals sum parallel work and must not be added to makespan.", + "No new real run, VM provisioning, daemon restart or database access." + ] +} diff --git a/experiments/montage-micro-hpc-e2/publication-wall-reference-20261011.json b/experiments/montage-micro-hpc-e2/publication-wall-reference-20261011.json new file mode 100644 index 00000000..6a5d37ab --- /dev/null +++ b/experiments/montage-micro-hpc-e2/publication-wall-reference-20261011.json @@ -0,0 +1,17 @@ +{ + "issue":"AKO-158", + "referenceRunId":"montage-prism-transfer-real-20261011-v2", + "role":"publication-wall-reference-not-queue-free-service-fit", + "samples":40, + "weightedWallSecondsPerScanByte":3.845388864356909e-8, + "summedWorkSecondsNotMakespan":1133.2548651695251, + "finalViewerPublicationWallSeconds":298.7341306209564, + "finalViewerTerminalScanBytes":3144545877, + "finalViewerExcludedPollDelaySeconds":3.1423821449279785, + "limitations":[ + "Time is completion event minus terminal observation, matched to the latest completed task and handle attempt.", + "Observation transport, sealing, inherited-input pruning and scheduling are combined; admission wait has not been separated.", + "The recovered run is a reference, not independent validation.", + "Parallel publication durations must not be summed as elapsed workflow time." + ] +} diff --git a/experiments/montage-micro-hpc-e2/recovered-products-preservation-20261011.json b/experiments/montage-micro-hpc-e2/recovered-products-preservation-20261011.json new file mode 100644 index 00000000..5d800262 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/recovered-products-preservation-20261011.json @@ -0,0 +1,17 @@ +{ + "issue": "AKO-158", + "runId": "montage-prism-transfer-real-20261011-v2", + "products": [ + {"name":"1-mosaic.png","sha256":"90b53e98f8de8ba50db292fb7727c1698a8f0139ba53ca877f05b34f7e4a75cb"}, + {"name":"2-mosaic.png","sha256":"3ac1c8aee8f6f96813ffa29cddee00d9a632a5626d75feb36853671379d37082"}, + {"name":"3-mosaic.png","sha256":"407f7524a7c3644731f39743990c9debc28f03e1eda4c21545a190a7fead104f"}, + {"name":"mosaic-color.png","sha256":"63fa4d8447274bdb8a44f6be8cf5a23c0a2f0baa479bd2ae1ec3b541df4e19f2"} + ], + "preservation": "API console export; archive digest verified before product hashes were read from local archives", + "activeHandles": 0, + "activeTransfers": 0, + "unreleasedWorkspaceLeases": 0, + "allTenInstancesDestroyedInInventory": true, + "destroyOperationsTerminalConfirmed": true, + "predictionValidated": false +} diff --git a/experiments/montage-micro-hpc-e2/relay-fresh-comparison.json b/experiments/montage-micro-hpc-e2/relay-fresh-comparison.json new file mode 100644 index 00000000..6e79a2a7 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/relay-fresh-comparison.json @@ -0,0 +1,5772 @@ +{ + "schema_version": 1, + "run_id": "montage-micro-hpc-e2-real-20261008-relay-fresh-v1", + "attempt_history": "multiple_attempts", + "audit": { + "expected_count": 58, + "missing": [], + "unexpected": [], + "not_completed": [], + "incomplete_transfer_count": 0, + "barrier_violation_count": 0, + "unreleased_lease_count": 0, + "single_lane_micro_overlap_count": 0, + "missing_dependency_route_count": 0, + "cross_cloud_hpc_non_relay_count": 0 + }, + "plan": { + "makespanSeconds": 3617.9382322026677, + "cost": 0 + }, + "plan_assignment_compute_work_seconds": 27568.49476684796, + "plan_assignment_transfer_work_seconds": 30.488017932813136, + "simulation": { + "makespanSeconds": 3617.4981106317523, + "cost": 0, + "breakdown": { + "computeSeconds": 27557.975652713318, + "transferSeconds": 39.960353403544104, + "queueSeconds": 5162.8883215401875, + "interferenceSeconds": 3339.975652713315, + "overheadSeconds": 0 + }, + "transferredBytes": 880135168 + }, + "real_api_reported": { + "makespanSeconds": 7796.457294225693, + "cost": 0, + "breakdown": { + "computeSeconds": 13131.177605390549, + "transferSeconds": 1895.0516393184662, + "queueSeconds": 16026.768799304962, + "interferenceSeconds": 0, + "overheadSeconds": 35.79461359977722 + }, + "transferredBytes": 4249956208 + }, + "real_api_run_timestamp_wall_seconds": 589, + "real_network_bytes_from_routes": 33564916733, + "transfer_route_count": 122, + "dependency_route_count": 114, + "cross_cloud_hpc_relay_count": 24, + "relay_server_network_bytes": 10486984513, + "routes": [ + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 6.314751386642456, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.011451482772827148, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.006067752838134766, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.00992894172668457, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.009818792343139648, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000046", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.018524885177612305, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000039", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.007843494415283203, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000040", + "producer_activity_id": null, + "strategy": "use-existing", + "duration_seconds": 0.028724193572998047, + "logical_bytes": 340992000, + "network_bytes": null, + "relay_server_network_bytes": null, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013", + "strategy": "direct-runtime", + "duration_seconds": 614.3309240341187, + "logical_bytes": 741210108, + "network_bytes": 426472193, + "relay_server_network_bytes": 0, + "archive_bytes": 426472193, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.696555709 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 37.448481474 + }, + { + "name": "peer-private-upload", + "durationSeconds": 6.94946617 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 18.766858259 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 409.363066935 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 204.970025468 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014", + "strategy": "direct-runtime", + "duration_seconds": 614.3319420814514, + "logical_bytes": 106387200, + "network_bytes": 426365774, + "relay_server_network_bytes": 0, + "archive_bytes": 426365774, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.500666335 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 50.899191981 + }, + { + "name": "peer-private-upload", + "durationSeconds": 20.743827926 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 60.205929361 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 409.363066935 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 204.970025468 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "strategy": "gateway", + "duration_seconds": 614.331832408905, + "logical_bytes": 106346880, + "network_bytes": 853046040, + "relay_server_network_bytes": 853046040, + "archive_bytes": 426470897, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000020542, + "rsync_execution_seconds": 292.919092466, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 60.018909777 + }, + { + "name": "archive-validation", + "durationSeconds": 3.698325752 + }, + { + "name": "archive-upload", + "durationSeconds": 292.919378758 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 12.03831084 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 409.363066935 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 204.970025468 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016", + "strategy": "direct-runtime", + "duration_seconds": 614.3317084312439, + "logical_bytes": 106387200, + "network_bytes": 426441434, + "relay_server_network_bytes": 0, + "archive_bytes": 426441434, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.473411293 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 45.564087729 + }, + { + "name": "peer-private-upload", + "durationSeconds": 22.229222218 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 39.22544881 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 409.363066935 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 204.970025468 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "strategy": "direct-runtime", + "duration_seconds": 614.33243227005, + "logical_bytes": 2606, + "network_bytes": 596550583, + "relay_server_network_bytes": 0, + "archive_bytes": 596550583, + "files": 1, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.6179640429999997 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 357.677977746 + }, + { + "name": "peer-private-upload", + "durationSeconds": 25.054433636 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 21.229282217 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 409.363066935 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 204.970025468 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032", + "strategy": "direct-runtime", + "duration_seconds": 654.819329738617, + "logical_bytes": 733301626, + "network_bytes": 419527742, + "relay_server_network_bytes": 0, + "archive_bytes": 419527742, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.516659502 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 46.013634438 + }, + { + "name": "peer-private-upload", + "durationSeconds": 19.305967884 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 21.343756676 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 420.8945164 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 233.922758107 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033", + "strategy": "direct-runtime", + "duration_seconds": 654.8196921348572, + "logical_bytes": 105154560, + "network_bytes": 419115000, + "relay_server_network_bytes": 0, + "archive_bytes": 419115000, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.446660876 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 119.17582193 + }, + { + "name": "peer-private-upload", + "durationSeconds": 42.504151228 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 14.729308424 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 420.8945164 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 233.922758107 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034", + "strategy": "direct-runtime", + "duration_seconds": 654.8194923400879, + "logical_bytes": 105073920, + "network_bytes": 419370513, + "relay_server_network_bytes": 0, + "archive_bytes": 419370513, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.520697252 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 32.50288139 + }, + { + "name": "peer-private-upload", + "durationSeconds": 6.674080961 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 16.306726674 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 420.8945164 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 233.922758107 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035", + "strategy": "direct-runtime", + "duration_seconds": 654.8190784454346, + "logical_bytes": 105154560, + "network_bytes": 419166529, + "relay_server_network_bytes": 0, + "archive_bytes": 419166529, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.524931501 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 178.179390457 + }, + { + "name": "peer-private-upload", + "durationSeconds": 26.112237095 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 76.306967326 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 420.8945164 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 233.922758107 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "strategy": "direct-runtime", + "duration_seconds": 654.8197107315063, + "logical_bytes": 2600, + "network_bytes": 587421724, + "relay_server_network_bytes": 0, + "archive_bytes": 587421724, + "files": 1, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.810051793 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 323.932901273 + }, + { + "name": "peer-private-upload", + "durationSeconds": 29.651184096 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 62.718283861 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 420.8945164 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 233.922758107 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "strategy": "gateway", + "duration_seconds": 902.0535116195679, + "logical_bytes": 739148022, + "network_bytes": 848131317, + "relay_server_network_bytes": 848131317, + "archive_bytes": 424013836, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000376458, + "rsync_execution_seconds": 474.442208257, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 51.371051149 + }, + { + "name": "archive-validation", + "durationSeconds": 3.561882168 + }, + { + "name": "archive-upload", + "durationSeconds": 474.442923382 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 88.708711374 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 619.040034573 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 283.013860171 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052", + "strategy": "direct-runtime", + "duration_seconds": 902.05379986763, + "logical_bytes": 106058880, + "network_bytes": 423790974, + "relay_server_network_bytes": 0, + "archive_bytes": 423790974, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 10.223374338 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 44.109440396 + }, + { + "name": "peer-private-upload", + "durationSeconds": 21.498305218 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 202.924363633 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 619.040034573 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 283.013860171 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053", + "strategy": "direct-runtime", + "duration_seconds": 902.0536737442017, + "logical_bytes": 106058880, + "network_bytes": 424020474, + "relay_server_network_bytes": 0, + "archive_bytes": 424020474, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 9.641296587 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 58.485955777 + }, + { + "name": "peer-private-upload", + "durationSeconds": 18.600893676 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 250.215288947 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 619.040034573 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 283.013860171 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054", + "strategy": "direct-runtime", + "duration_seconds": 902.053548336029, + "logical_bytes": 106058880, + "network_bytes": 423858028, + "relay_server_network_bytes": 0, + "archive_bytes": 423858028, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 10.256114838 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 47.370554147 + }, + { + "name": "peer-private-upload", + "durationSeconds": 21.86901251 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 236.300084024 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 619.040034573 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 283.013860171 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "strategy": "direct-runtime", + "duration_seconds": 902.0540244579315, + "logical_bytes": 2594, + "network_bytes": 591910948, + "relay_server_network_bytes": 0, + "archive_bytes": 591910948, + "files": 1, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 9.887461463 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 223.604156602 + }, + { + "name": "peer-private-upload", + "durationSeconds": 105.39849838 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 182.637080417 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 619.040034573 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 283.013860171 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "strategy": "direct-runtime", + "duration_seconds": 281.8709681034088, + "logical_bytes": 634782588, + "network_bytes": 369736839, + "relay_server_network_bytes": 0, + "archive_bytes": 369736839, + "files": 73, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.5062835850000003 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 137.434823522 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 45.145640854 + }, + { + "name": "peer-private-upload", + "durationSeconds": 69.861771365 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 13.195819923 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 270.108200332 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 11.762270089 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000001", + "strategy": "runtime-local", + "duration_seconds": 282.7809317111969, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 270.108200332 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 11.762270089 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "strategy": "direct-runtime", + "duration_seconds": 280.3702085018158, + "logical_bytes": 634782588, + "network_bytes": 369736839, + "relay_server_network_bytes": 0, + "archive_bytes": 369736839, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.494062876 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 137.438620897 + }, + { + "name": "peer-private-upload", + "durationSeconds": 107.01930309 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 10.862254088 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 260.111651953 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 20.258116134 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000002", + "strategy": "runtime-local", + "duration_seconds": 281.28264021873474, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 260.111651953 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 20.258116134 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "strategy": "gateway", + "duration_seconds": 763.6487035751343, + "logical_bytes": null, + "network_bytes": 739564074, + "relay_server_network_bytes": 739564074, + "archive_bytes": 369736839, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000057458, + "rsync_execution_seconds": 527.983878574, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 195.534266548 + }, + { + "name": "archive-validation", + "durationSeconds": 3.1084487100000002 + }, + { + "name": "archive-upload", + "durationSeconds": 527.985381533 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 10.658598546 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 743.329154213 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 20.317531426 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000003", + "strategy": "gateway", + "duration_seconds": 763.6494445800781, + "logical_bytes": null, + "network_bytes": 290209799, + "relay_server_network_bytes": 290209799, + "archive_bytes": 145087125, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.00002875, + "rsync_execution_seconds": 194.006008338, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 14.903157007 + }, + { + "name": "archive-validation", + "durationSeconds": 1.030943 + }, + { + "name": "archive-upload", + "durationSeconds": 194.006319797 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 12.091310714 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 743.329154213 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 20.317531426 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "strategy": "runtime-local", + "duration_seconds": 180.0232071876526, + "logical_bytes": 634782588, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 31.269985598 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 147.835616818 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000004", + "strategy": "direct-runtime", + "duration_seconds": 179.1061429977417, + "logical_bytes": null, + "network_bytes": 144937261, + "relay_server_network_bytes": 0, + "archive_bytes": 144937261, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.443690084 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 11.954264714 + }, + { + "name": "peer-private-upload", + "durationSeconds": 5.994418211 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 8.82620892 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 31.269985598 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 147.835616818 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "strategy": "direct-runtime", + "duration_seconds": 271.62722158432007, + "logical_bytes": 628187386, + "network_bytes": 363248071, + "relay_server_network_bytes": 0, + "archive_bytes": 363248071, + "files": 73, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.545101501 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 147.840374401 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 47.780492271 + }, + { + "name": "peer-private-upload", + "durationSeconds": 40.679319519 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 11.011978505 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 251.763697781 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.862637800999998 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000020", + "strategy": "runtime-local", + "duration_seconds": 275.13195848464966, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 251.763697781 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.862637800999998 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "strategy": "direct-runtime", + "duration_seconds": 295.60314750671387, + "logical_bytes": 628187386, + "network_bytes": 363248071, + "relay_server_network_bytes": 0, + "archive_bytes": 363248071, + "files": 73, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.509992001 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 147.793062151 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 47.780213396 + }, + { + "name": "peer-private-upload", + "durationSeconds": 61.861914362 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 12.537392381 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 274.3637555 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 21.238856801 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000021", + "strategy": "runtime-local", + "duration_seconds": 299.08795189857483, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 274.3637555 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 21.238856801 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "strategy": "direct-runtime", + "duration_seconds": 270.89739298820496, + "logical_bytes": 628187386, + "network_bytes": 363248071, + "relay_server_network_bytes": 0, + "archive_bytes": 363248071, + "files": 73, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.405920127 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 150.576803527 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 47.785550646 + }, + { + "name": "peer-private-upload", + "durationSeconds": 46.422484272 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 13.256668464 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 262.366893245 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.529488046 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000022", + "strategy": "runtime-local", + "duration_seconds": 271.7986271381378, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 262.366893245 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.529488046 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "strategy": "direct-runtime", + "duration_seconds": 276.56462240219116, + "logical_bytes": 628187386, + "network_bytes": 363248071, + "relay_server_network_bytes": 0, + "archive_bytes": 363248071, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.431771626 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 150.865991319 + }, + { + "name": "peer-private-upload", + "durationSeconds": 88.959894791 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 12.587019047 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 256.668754117 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.895121884 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000023", + "strategy": "direct-runtime", + "duration_seconds": 276.564759016037, + "logical_bytes": null, + "network_bytes": 143228071, + "relay_server_network_bytes": 0, + "archive_bytes": 143228071, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.434955335 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 11.160557672 + }, + { + "name": "peer-private-upload", + "durationSeconds": 4.112578418 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 3.936012127 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 256.668754117 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.895121884 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "strategy": "gateway", + "duration_seconds": 414.72493290901184, + "logical_bytes": null, + "network_bytes": 735942728, + "relay_server_network_bytes": 735942728, + "archive_bytes": 367926386, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000044917, + "rsync_execution_seconds": 345.63816745, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 30.36157214 + }, + { + "name": "archive-validation", + "durationSeconds": 2.827200667 + }, + { + "name": "archive-upload", + "durationSeconds": 345.638546908 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 10.649182378999999 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 396.479874348 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.244609717 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000039", + "strategy": "direct-runtime", + "duration_seconds": 581.4172084331512, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 396.479874348 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 18.244609717 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "strategy": "direct-runtime", + "duration_seconds": 260.8665964603424, + "logical_bytes": 633129462, + "network_bytes": 367926386, + "relay_server_network_bytes": 0, + "archive_bytes": 367926386, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.353854793 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 31.299148931 + }, + { + "name": "peer-private-upload", + "durationSeconds": 5.727737628 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 10.765665588 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 241.653465194 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.212322008 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000040", + "strategy": "gateway", + "duration_seconds": 260.866628408432, + "logical_bytes": null, + "network_bytes": 106265672, + "relay_server_network_bytes": 106265672, + "archive_bytes": 53126285, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000042416, + "rsync_execution_seconds": 26.534619095, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 212.08486393 + }, + { + "name": "archive-validation", + "durationSeconds": 0.477033 + }, + { + "name": "archive-upload", + "durationSeconds": 26.535173387 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.6406673760000001 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 241.653465194 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.212322008 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "strategy": "direct-runtime", + "duration_seconds": 270.9440813064575, + "logical_bytes": 633129462, + "network_bytes": 367926386, + "relay_server_network_bytes": 0, + "archive_bytes": 367926386, + "files": 73, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.547835418 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 28.500759888 + }, + { + "name": "peer-private-upload", + "durationSeconds": 6.590586378 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 87.578910415 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 127.217878267 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 143.72586194 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000041", + "strategy": "runtime-local", + "duration_seconds": 271.9007866382599, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 127.217878267 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 143.72586194 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "strategy": "direct-runtime", + "duration_seconds": 75.03178119659424, + "logical_bytes": 633129462, + "network_bytes": 367926386, + "relay_server_network_bytes": 0, + "archive_bytes": 367926386, + "files": 73, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.619575251 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 29.854826721 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 2.8302050850000002 + }, + { + "name": "peer-private-upload", + "durationSeconds": 5.970946169 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 12.265419547 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 55.324760233 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.706502426 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000042", + "strategy": "direct-runtime", + "duration_seconds": 75.0317952632904, + "logical_bytes": null, + "network_bytes": 144466623, + "relay_server_network_bytes": 0, + "archive_bytes": 144466623, + "files": null, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.620175126 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 14.725870298 + }, + { + "name": "peer-private-upload", + "durationSeconds": 4.08022071 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 3.448457293 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 55.324760233 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 19.706502426 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "strategy": "gateway", + "duration_seconds": 607.9305422306061, + "logical_bytes": 634782312, + "network_bytes": 739568420, + "relay_server_network_bytes": 739568420, + "archive_bytes": 369739011, + "files": 72, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000017833, + "rsync_execution_seconds": 323.392784192, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 251.858159615 + }, + { + "name": "archive-validation", + "durationSeconds": 2.896204918 + }, + { + "name": "archive-upload", + "durationSeconds": 323.392935567 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 9.082637671 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 590.19212673 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.727275007 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "strategy": "direct-runtime", + "duration_seconds": 265.28645491600037, + "logical_bytes": 628187110, + "network_bytes": 363247666, + "relay_server_network_bytes": 0, + "archive_bytes": 363247666, + "files": 72, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.388611168 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 218.511741725 + }, + { + "name": "peer-private-upload", + "durationSeconds": 13.554433756 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 10.899791713 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 248.196333155 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.0898088 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "strategy": "direct-runtime", + "duration_seconds": 63.65419936180115, + "logical_bytes": 633129186, + "network_bytes": 367926467, + "relay_server_network_bytes": 0, + "archive_bytes": 367926467, + "files": 72, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.506786876 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 30.149714014 + }, + { + "name": "peer-private-upload", + "durationSeconds": 12.088991463 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 10.528675172 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 57.083123277 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 6.572613712 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000005", + "strategy": "gateway", + "duration_seconds": 652.6848080158234, + "logical_bytes": null, + "network_bytes": 426557260, + "relay_server_network_bytes": 426557260, + "archive_bytes": 213252535, + "files": 52, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000008834, + "rsync_execution_seconds": 264.14142033, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 20.572842218 + }, + { + "name": "archive-validation", + "durationSeconds": 1.6265003340000002 + }, + { + "name": "archive-upload", + "durationSeconds": 264.141550914 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 9.367313213 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 616.375398324 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 36.308577057 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000006", + "strategy": "gateway", + "duration_seconds": 652.6848516464233, + "logical_bytes": null, + "network_bytes": 429294254, + "relay_server_network_bytes": 429294254, + "archive_bytes": 214620864, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000034042, + "rsync_execution_seconds": 277.566261002, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 29.942551139 + }, + { + "name": "archive-validation", + "durationSeconds": 1.620963709 + }, + { + "name": "archive-upload", + "durationSeconds": 277.56659746 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 7.650653212 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 616.375398324 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 36.308577057 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000007", + "strategy": "gateway", + "duration_seconds": 652.6844909191132, + "logical_bytes": null, + "network_bytes": 405546200, + "relay_server_network_bytes": 405546200, + "archive_bytes": 202748285, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000018083, + "rsync_execution_seconds": 196.576118173, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 16.749759466 + }, + { + "name": "archive-validation", + "durationSeconds": 1.609920126 + }, + { + "name": "archive-upload", + "durationSeconds": 196.576329132 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 45.573716979 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 616.375398324 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 36.308577057 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000008", + "strategy": "gateway", + "duration_seconds": 652.6846537590027, + "logical_bytes": null, + "network_bytes": 407475944, + "relay_server_network_bytes": 407475944, + "archive_bytes": 203713041, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000010292, + "rsync_execution_seconds": 229.566732314, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 19.614414426 + }, + { + "name": "archive-validation", + "durationSeconds": 1.5673055420000002 + }, + { + "name": "archive-upload", + "durationSeconds": 229.566848481 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 15.611233216 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 616.375398324 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 36.308577057 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009", + "strategy": "direct-runtime", + "duration_seconds": 656.0008978843689, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 616.375398324 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 36.308577057 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000010", + "strategy": "gateway", + "duration_seconds": 652.6853134632111, + "logical_bytes": null, + "network_bytes": 419956542, + "relay_server_network_bytes": 419956542, + "archive_bytes": 209952576, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 2.454666543, + "rsync_execution_seconds": 385.457718426, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 67.47242174 + }, + { + "name": "archive-validation", + "durationSeconds": 1.629782542 + }, + { + "name": "archive-upload", + "durationSeconds": 387.912780802 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 6.578137294 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 616.375398324 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 36.308577057 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000024", + "strategy": "runtime-local", + "duration_seconds": 185.99872636795044, + "logical_bytes": 331918633, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 52, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 149.304178902 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 35.7947211 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000025", + "strategy": "direct-runtime", + "duration_seconds": 185.0986704826355, + "logical_bytes": 136224289, + "network_bytes": 211411738, + "relay_server_network_bytes": 0, + "archive_bytes": 211411738, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 4.435207252 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 17.009688424 + }, + { + "name": "peer-private-upload", + "durationSeconds": 7.854852628 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 16.729584383 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 149.304178902 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 35.7947211 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000026", + "strategy": "direct-runtime", + "duration_seconds": 185.09843158721924, + "logical_bytes": 114307483, + "network_bytes": 199485248, + "relay_server_network_bytes": 0, + "archive_bytes": 199485248, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 4.518207252 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 16.956187217 + }, + { + "name": "peer-private-upload", + "durationSeconds": 7.09634467 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 12.655428547 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 149.304178902 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 35.7947211 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000027", + "strategy": "direct-runtime", + "duration_seconds": 185.09855437278748, + "logical_bytes": 9187481, + "network_bytes": 200618813, + "relay_server_network_bytes": 0, + "archive_bytes": 200618813, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 4.485541419 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 14.122754631 + }, + { + "name": "peer-private-upload", + "durationSeconds": 4.9466358360000005 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 13.123740756 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 149.304178902 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 35.7947211 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000028", + "strategy": "direct-runtime", + "duration_seconds": 185.09829258918762, + "logical_bytes": 18288283, + "network_bytes": 205320505, + "relay_server_network_bytes": 0, + "archive_bytes": 205320505, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 4.198965294 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 126.463010976 + }, + { + "name": "peer-private-upload", + "durationSeconds": 10.03860888 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 6.52149171 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 149.304178902 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 35.7947211 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000029", + "strategy": "direct-runtime", + "duration_seconds": 185.09922909736633, + "logical_bytes": 18259484, + "network_bytes": 206564380, + "relay_server_network_bytes": 0, + "archive_bytes": 206564380, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 4.75086746 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 15.706082966 + }, + { + "name": "peer-private-upload", + "durationSeconds": 5.086773127 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 14.25090909 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 149.304178902 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 35.7947211 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000043", + "strategy": "direct-runtime", + "duration_seconds": 204.2971155643463, + "logical_bytes": 334124710, + "network_bytes": 212132039, + "relay_server_network_bytes": 0, + "archive_bytes": 212132039, + "files": 52, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.490275001 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 17.092995882 + }, + { + "name": "peer-private-upload", + "durationSeconds": 6.81153317 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 15.706527631 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 170.594113411 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 33.70487864 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000044", + "strategy": "runtime-local", + "duration_seconds": 205.23656487464905, + "logical_bytes": 137623968, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 170.594113411 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 33.70487864 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045", + "strategy": "gateway", + "duration_seconds": 204.2967972755432, + "logical_bytes": 115303963, + "network_bytes": 403025892, + "relay_server_network_bytes": 403025892, + "archive_bytes": 201488287, + "files": 5, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000006791, + "rsync_execution_seconds": 105.87002484, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 54.969837733 + }, + { + "name": "archive-validation", + "durationSeconds": 1.5398483760000001 + }, + { + "name": "archive-upload", + "durationSeconds": 105.870139965 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 7.276137629 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 170.594113411 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 33.70487864 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000046", + "strategy": "gateway", + "duration_seconds": 204.29700016975403, + "logical_bytes": 9256599, + "network_bytes": 405335430, + "relay_server_network_bytes": 405335430, + "archive_bytes": 202642912, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.0000065, + "rsync_execution_seconds": 105.792789132, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 54.545440774 + }, + { + "name": "archive-validation", + "durationSeconds": 1.5712100420000001 + }, + { + "name": "archive-upload", + "durationSeconds": 105.792921506 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 7.358542878 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 170.594113411 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 33.70487864 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000047", + "strategy": "direct-runtime", + "duration_seconds": 204.29672813415527, + "logical_bytes": 18403484, + "network_bytes": 207598133, + "relay_server_network_bytes": 0, + "archive_bytes": 207598133, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.494544043 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 15.630427632 + }, + { + "name": "peer-private-upload", + "durationSeconds": 6.473384461 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 16.003756132 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 170.594113411 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 33.70487864 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000048", + "strategy": "direct-runtime", + "duration_seconds": 204.2974681854248, + "logical_bytes": 18415005, + "network_bytes": 208932402, + "relay_server_network_bytes": 0, + "archive_bytes": 208932402, + "files": 3, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.313721168 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 16.817563299 + }, + { + "name": "peer-private-upload", + "durationSeconds": 7.050127379 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 16.135577382 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 170.594113411 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 33.70487864 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000005", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000001", + "strategy": "direct-runtime", + "duration_seconds": 66.46953964233398, + "logical_bytes": 196978822, + "network_bytes": 142938003, + "relay_server_network_bytes": 0, + "archive_bytes": 142938003, + "files": 47, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.478756876 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 0.000057917 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 1.098315792 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.476758044 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 3.563936794 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 57.534868151 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.935036004 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000005", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000002", + "strategy": "direct-runtime", + "duration_seconds": 66.46959853172302, + "logical_bytes": 106387200, + "network_bytes": 143717245, + "relay_server_network_bytes": 0, + "archive_bytes": 143717245, + "files": 2, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 10.243664213 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 0.000197667 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 12.98712063 + }, + { + "name": "peer-private-upload", + "durationSeconds": 19.996901009 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 4.055138418 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 57.534868151 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.935036004 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000006", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000001", + "strategy": "direct-runtime", + "duration_seconds": 62.52905774116516, + "logical_bytes": 196978822, + "network_bytes": 142938003, + "relay_server_network_bytes": 0, + "archive_bytes": 142938003, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.442025668 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 10.219660338 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.91963696 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 6.087234836 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 27.384603138 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 35.14418639 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000006", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000003", + "strategy": "direct-runtime", + "duration_seconds": 62.52943420410156, + "logical_bytes": 106346880, + "network_bytes": 145087125, + "relay_server_network_bytes": 0, + "archive_bytes": 145087125, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.449628918 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 10.493874089 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.857341085 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 6.519990336 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 27.384603138 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 35.14418639 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000007", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000001", + "strategy": "direct-runtime", + "duration_seconds": 187.4970350265503, + "logical_bytes": 196978822, + "network_bytes": 142938003, + "relay_server_network_bytes": 0, + "archive_bytes": 142938003, + "files": 47, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.62780175 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 0.00005 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 1.82465575 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.724145585 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 3.630290044 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 178.471809789 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 9.025138212 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000007", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000004", + "strategy": "direct-runtime", + "duration_seconds": 187.49725008010864, + "logical_bytes": 106387200, + "network_bytes": 144937261, + "relay_server_network_bytes": 0, + "archive_bytes": 144937261, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 6.252224127 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 146.270671565 + }, + { + "name": "peer-private-upload", + "durationSeconds": 19.499906801 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 4.06608696 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 178.471809789 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 9.025138212 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000008", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000002", + "strategy": "direct-runtime", + "duration_seconds": 116.37553095817566, + "logical_bytes": 196938502, + "network_bytes": 143717245, + "relay_server_network_bytes": 0, + "archive_bytes": 143717245, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.995029627 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 71.209413491 + }, + { + "name": "peer-private-upload", + "durationSeconds": 23.997694719 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 6.046395295 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 106.084877298 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 10.290322213 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000008", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000003", + "strategy": "runtime-local", + "duration_seconds": 117.272714138031, + "logical_bytes": 106346880, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 106.084877298 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 10.290322213 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000002", + "strategy": "gateway", + "duration_seconds": 504.94350481033325, + "logical_bytes": null, + "network_bytes": 287469704, + "relay_server_network_bytes": 287469704, + "archive_bytes": 143717245, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000034375, + "rsync_execution_seconds": 365.254483834, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 108.49965109 + }, + { + "name": "archive-validation", + "durationSeconds": 1.197743334 + }, + { + "name": "archive-upload", + "durationSeconds": 365.254826834 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 5.329628627 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 492.625308808 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 12.317526589 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000004", + "strategy": "gateway", + "duration_seconds": 504.94351077079773, + "logical_bytes": null, + "network_bytes": 289910040, + "relay_server_network_bytes": 289910040, + "archive_bytes": 144937261, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 3.832110626, + "rsync_execution_seconds": 115.047871345, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 158.986707946 + }, + { + "name": "archive-validation", + "durationSeconds": 1.030272084 + }, + { + "name": "archive-upload", + "durationSeconds": 118.880372471 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 4.736107752 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 492.625308808 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 12.317526589 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000010", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000003", + "strategy": "direct-runtime", + "duration_seconds": 145.28475213050842, + "logical_bytes": 196898182, + "network_bytes": 145087125, + "relay_server_network_bytes": 0, + "archive_bytes": 145087125, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.4933886259999998 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 13.373688048 + }, + { + "name": "peer-private-upload", + "durationSeconds": 6.486026753 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 3.573951667 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 136.770193771 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.514610588 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000010", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000004", + "strategy": "direct-runtime", + "duration_seconds": 145.2849087715149, + "logical_bytes": 106387200, + "network_bytes": 144937261, + "relay_server_network_bytes": 0, + "archive_bytes": 144937261, + "files": 2, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 7.491883629 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 0.000164083 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 14.997653799 + }, + { + "name": "peer-private-upload", + "durationSeconds": 18.483972508 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 4.598530169 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 136.770193771 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.514610588 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000024", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000020", + "strategy": "direct-runtime", + "duration_seconds": 133.22930908203125, + "logical_bytes": 195665542, + "network_bytes": 141396213, + "relay_server_network_bytes": 0, + "archive_bytes": 141396213, + "files": 47, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.313380252 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 0.000138625 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 1.02851725 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.862005626 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 4.102986002 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 126.530978517 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 6.69817192 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000024", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000021", + "strategy": "direct-runtime", + "duration_seconds": 133.229474067688, + "logical_bytes": 105154560, + "network_bytes": 142297171, + "relay_server_network_bytes": 0, + "archive_bytes": 142297171, + "files": 2, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 6.7468419189999995 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 80.758853245 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 14.244988632 + }, + { + "name": "peer-private-upload", + "durationSeconds": 18.235347342 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 4.013915044 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 126.530978517 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 6.69817192 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000025", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000020", + "strategy": "runtime-local", + "duration_seconds": 35.70151400566101, + "logical_bytes": 195665542, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 23.791440219 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 11.014713296 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000025", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000022", + "strategy": "direct-runtime", + "duration_seconds": 34.805907249450684, + "logical_bytes": 105073920, + "network_bytes": 143458908, + "relay_server_network_bytes": 0, + "archive_bytes": 143458908, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.398061667 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 9.89596313 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.6887702940000002 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 4.035141335 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 23.791440219 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 11.014713296 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000026", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000020", + "strategy": "direct-runtime", + "duration_seconds": 84.30387496948242, + "logical_bytes": 195665542, + "network_bytes": 141396213, + "relay_server_network_bytes": 0, + "archive_bytes": 141396213, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.625618794 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 10.052187712 + }, + { + "name": "peer-private-upload", + "durationSeconds": 4.074194877 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 29.065164388 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 50.655918857 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 33.648178473 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000026", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000023", + "strategy": "direct-runtime", + "duration_seconds": 84.30418920516968, + "logical_bytes": 105154560, + "network_bytes": 143228071, + "relay_server_network_bytes": 0, + "archive_bytes": 143228071, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.62268521 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 10.396417754 + }, + { + "name": "peer-private-upload", + "durationSeconds": 4.498511627 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 30.318646014 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 50.655918857 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 33.648178473 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000027", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000021", + "strategy": "direct-runtime", + "duration_seconds": 143.87353658676147, + "logical_bytes": 195705862, + "network_bytes": 142297171, + "relay_server_network_bytes": 0, + "archive_bytes": 142297171, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.6050648770000002 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 107.41510084 + }, + { + "name": "peer-private-upload", + "durationSeconds": 19.735992301 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 4.094739502 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 136.929768104 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 6.943490004 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000027", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000022", + "strategy": "direct-runtime", + "duration_seconds": 143.87355995178223, + "logical_bytes": 105073920, + "network_bytes": 143458908, + "relay_server_network_bytes": 0, + "archive_bytes": 143458908, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.605970585 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 9.965538712 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.901776836 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 3.746274419 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 136.929768104 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 6.943490004 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000028", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000021", + "strategy": "runtime-local", + "duration_seconds": 140.04533123970032, + "logical_bytes": 195705862, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 66.412851905 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 72.725267992 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000028", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000023", + "strategy": "direct-runtime", + "duration_seconds": 139.13855051994324, + "logical_bytes": 105154560, + "network_bytes": 143228071, + "relay_server_network_bytes": 0, + "archive_bytes": 143228071, + "files": 2, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.423773084 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 0.000123208 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 1.001709667 + }, + { + "name": "peer-private-upload", + "durationSeconds": 13.590817505 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 47.200062897 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 66.412851905 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 72.725267992 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000029", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000022", + "strategy": "direct-runtime", + "duration_seconds": 41.417306900024414, + "logical_bytes": 195625222, + "network_bytes": 143458908, + "relay_server_network_bytes": 0, + "archive_bytes": 143458908, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 4.92303071 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 11.760180881 + }, + { + "name": "peer-private-upload", + "durationSeconds": 5.286655794 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 8.700003503 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 32.434681389 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.983081629 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000029", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000023", + "strategy": "runtime-local", + "duration_seconds": 42.36921787261963, + "logical_bytes": 105154560, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 32.434681389 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.983081629 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000043", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000039", + "strategy": "gateway", + "duration_seconds": 75.43630909919739, + "logical_bytes": 106018560, + "network_bytes": 103997681, + "relay_server_network_bytes": 103997681, + "archive_bytes": 51992430, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000007875, + "rsync_execution_seconds": 48.899691022, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 18.246992883 + }, + { + "name": "archive-validation", + "durationSeconds": 0.507384292 + }, + { + "name": "archive-upload", + "durationSeconds": 48.899877147 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.838642917 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 71.865525409 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 3.575245252 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000043", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000040", + "strategy": "gateway", + "duration_seconds": 75.4400897026062, + "logical_bytes": 106058880, + "network_bytes": 106265672, + "relay_server_network_bytes": 106265672, + "archive_bytes": 53126285, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000021375, + "rsync_execution_seconds": 49.835328232, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 18.632541717 + }, + { + "name": "archive-validation", + "durationSeconds": 0.491476625 + }, + { + "name": "archive-upload", + "durationSeconds": 49.835570481 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.9972432100000002 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 71.865525409 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 3.575245252 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000044", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000039", + "strategy": "gateway", + "duration_seconds": 173.40080380439758, + "logical_bytes": 106018560, + "network_bytes": 103997681, + "relay_server_network_bytes": 103997681, + "archive_bytes": 51992430, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.00135175, + "rsync_execution_seconds": 10.743058006, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 153.051702361 + }, + { + "name": "archive-validation", + "durationSeconds": 0.480600958 + }, + { + "name": "archive-upload", + "durationSeconds": 10.744751547 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 1.6235946669999999 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 166.81424291 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 6.586628378 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000044", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000041", + "strategy": "direct-runtime", + "duration_seconds": 173.40082049369812, + "logical_bytes": 196610182, + "network_bytes": 144488542, + "relay_server_network_bytes": 0, + "archive_bytes": 144488542, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 3.373361751 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 9.953354088 + }, + { + "name": "peer-private-upload", + "durationSeconds": 3.9217513349999997 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 4.008268418 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 166.81424291 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 6.586628378 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000039", + "strategy": "direct-runtime", + "duration_seconds": 103.09657859802246, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 92.808933792 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.27773367 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000042", + "strategy": "gateway", + "duration_seconds": 101.08658576011658, + "logical_bytes": null, + "network_bytes": 288968644, + "relay_server_network_bytes": 288968644, + "archive_bytes": 144466623, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000033959, + "rsync_execution_seconds": 73.086553783, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 12.459985255 + }, + { + "name": "archive-validation", + "durationSeconds": 1.3071645840000001 + }, + { + "name": "archive-upload", + "durationSeconds": 73.086863575 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 3.8476616679999998 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 92.808933792 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 8.27773367 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000046", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000040", + "strategy": "direct-runtime", + "duration_seconds": 403.96275639533997, + "logical_bytes": null, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 391.325458929 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 10.55416438 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000046", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000041", + "strategy": "gateway", + "duration_seconds": 401.8796820640564, + "logical_bytes": null, + "network_bytes": 289012490, + "relay_server_network_bytes": 289012490, + "archive_bytes": 144488542, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000020291, + "rsync_execution_seconds": 79.466218078, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 80.713093746 + }, + { + "name": "archive-validation", + "durationSeconds": 1.190307292 + }, + { + "name": "archive-upload", + "durationSeconds": 79.466464203 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 5.151668711 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 391.325458929 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 10.55416438 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000047", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000040", + "strategy": "gateway", + "duration_seconds": 202.5039165019989, + "logical_bytes": 106058880, + "network_bytes": 106265672, + "relay_server_network_bytes": 106265672, + "archive_bytes": 53126285, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000017459, + "rsync_execution_seconds": 28.753494637, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 12.364945089 + }, + { + "name": "archive-validation", + "durationSeconds": 0.499485209 + }, + { + "name": "archive-upload", + "durationSeconds": 28.753710596 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 35.494144433 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 161.79732924 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 40.70646006 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000047", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000042", + "strategy": "direct-runtime", + "duration_seconds": 202.50408005714417, + "logical_bytes": 196610182, + "network_bytes": 144466623, + "relay_server_network_bytes": 0, + "archive_bytes": 144466623, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 5.00336296 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 9.647498963 + }, + { + "name": "peer-private-upload", + "durationSeconds": 20.311715717 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 81.177289579 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 161.79732924 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 40.70646006 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000048", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000041", + "strategy": "runtime-local", + "duration_seconds": 111.15899515151978, + "logical_bytes": 196610182, + "network_bytes": null, + "relay_server_network_bytes": 0, + "archive_bytes": null, + "files": 47, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "shared-preparation-barrier", + "durationSeconds": 74.835036493 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 35.222976348 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000048", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000042", + "strategy": "direct-runtime", + "duration_seconds": 110.05774259567261, + "logical_bytes": 106058880, + "network_bytes": 144466623, + "relay_server_network_bytes": 0, + "archive_bytes": 144466623, + "files": 2, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 4.7559595439999995 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 0.000165792 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 1.037282126 + }, + { + "name": "peer-private-upload", + "durationSeconds": 18.9088933 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 47.941839564 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 74.835036493 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 35.222976348 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013", + "strategy": "direct-runtime", + "duration_seconds": 449.87063813209534, + "logical_bytes": 741210108, + "network_bytes": 426472193, + "relay_server_network_bytes": 0, + "archive_bytes": 426472193, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.3781523350000002 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 33.650547057 + }, + { + "name": "peer-private-upload", + "durationSeconds": 7.208194753 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 20.133394884 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 395.404969888 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 54.476385108 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014", + "strategy": "direct-runtime", + "duration_seconds": 449.8704171180725, + "logical_bytes": 106387200, + "network_bytes": 426365774, + "relay_server_network_bytes": 0, + "archive_bytes": 426365774, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.38744196 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 49.507386231 + }, + { + "name": "peer-private-upload", + "durationSeconds": 13.321296298 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 67.807544613 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 395.404969888 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 54.476385108 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "strategy": "gateway", + "duration_seconds": 449.87037992477417, + "logical_bytes": 106346880, + "network_bytes": 853046040, + "relay_server_network_bytes": 853046040, + "archive_bytes": 426470897, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000061666, + "rsync_execution_seconds": 314.184302975, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 66.654863947 + }, + { + "name": "archive-validation", + "durationSeconds": 3.369466502 + }, + { + "name": "archive-upload", + "durationSeconds": 314.184784976 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 10.215388921 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 395.404969888 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 54.476385108 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016", + "strategy": "direct-runtime", + "duration_seconds": 449.87099862098694, + "logical_bytes": 106387200, + "network_bytes": 426441434, + "relay_server_network_bytes": 0, + "archive_bytes": 426441434, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.372991293 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 50.36258394 + }, + { + "name": "peer-private-upload", + "durationSeconds": 14.69739659 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 68.218247489 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 395.404969888 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 54.476385108 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032", + "strategy": "direct-runtime", + "duration_seconds": 573.1349940299988, + "logical_bytes": 733301626, + "network_bytes": 419527742, + "relay_server_network_bytes": 0, + "archive_bytes": 419527742, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 4.002438336 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 140.357959274 + }, + { + "name": "peer-private-upload", + "durationSeconds": 21.684435051 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 14.060131382 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 336.609416195 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 236.51497886 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033", + "strategy": "direct-runtime", + "duration_seconds": 573.1349549293518, + "logical_bytes": 105154560, + "network_bytes": 419115000, + "relay_server_network_bytes": 0, + "archive_bytes": 419115000, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 4.608344211 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 291.091687759 + }, + { + "name": "peer-private-upload", + "durationSeconds": 24.891700427 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 14.203071464 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 336.609416195 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 236.51497886 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034", + "strategy": "direct-runtime", + "duration_seconds": 573.1349215507507, + "logical_bytes": 105073920, + "network_bytes": 419370513, + "relay_server_network_bytes": 0, + "archive_bytes": 419370513, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 4.310183836 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 39.163348185 + }, + { + "name": "peer-private-upload", + "durationSeconds": 6.414093087 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 12.904189506 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 336.609416195 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 236.51497886 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035", + "strategy": "direct-runtime", + "duration_seconds": 573.1351981163025, + "logical_bytes": 105154560, + "network_bytes": 419166529, + "relay_server_network_bytes": 0, + "archive_bytes": 419166529, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 4.319766835 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 230.391706272 + }, + { + "name": "peer-private-upload", + "durationSeconds": 37.482240226 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 14.475958923 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 336.609416195 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 236.51497886 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "strategy": "gateway", + "duration_seconds": 975.9899892807007, + "logical_bytes": 739148022, + "network_bytes": 848131317, + "relay_server_network_bytes": 848131317, + "archive_bytes": 424013836, + "files": 75, + "snapshot_reused": null, + "ssh_channel_wait_seconds": 0.000046625, + "rsync_execution_seconds": 491.515547849, + "phases": [ + { + "name": "pack-and-receive", + "durationSeconds": 378.095518257 + }, + { + "name": "archive-validation", + "durationSeconds": 3.405897751 + }, + { + "name": "archive-upload", + "durationSeconds": 491.515959849 + }, + { + "name": "receiver-verify-and-extract", + "durationSeconds": 12.142094631 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 886.529509405 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 89.456599624 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052", + "strategy": "direct-runtime", + "duration_seconds": 975.98965716362, + "logical_bytes": 106058880, + "network_bytes": 423790974, + "relay_server_network_bytes": 0, + "archive_bytes": 423790974, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.495530709 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 43.728164937 + }, + { + "name": "peer-private-upload", + "durationSeconds": 40.80043531 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 148.20256036 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 886.529509405 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 89.456599624 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053", + "strategy": "direct-runtime", + "duration_seconds": 975.9895095825195, + "logical_bytes": 106058880, + "network_bytes": 424020474, + "relay_server_network_bytes": 0, + "archive_bytes": 424020474, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.499964127 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 179.482691833 + }, + { + "name": "peer-private-upload", + "durationSeconds": 55.021037859 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 161.990992491 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 886.529509405 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 89.456599624 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054", + "strategy": "direct-runtime", + "duration_seconds": 975.9902129173279, + "logical_bytes": 106058880, + "network_bytes": 423858028, + "relay_server_network_bytes": 0, + "archive_bytes": 423858028, + "files": 2, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.795197876 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 221.685793893 + }, + { + "name": "peer-private-upload", + "durationSeconds": 49.487928023 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 156.254224862 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 886.529509405 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 89.456599624 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000019", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "strategy": "direct-runtime", + "duration_seconds": 489.5976278781891, + "logical_bytes": 1112179754, + "network_bytes": 624696001, + "relay_server_network_bytes": 0, + "archive_bytes": 624696001, + "files": 84, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.392070375 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 381.551461925 + }, + { + "name": "peer-private-upload", + "durationSeconds": 63.248959779 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 23.551734302 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 472.592960508 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 17.00389159 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000038", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "strategy": "direct-runtime", + "duration_seconds": 496.11766481399536, + "logical_bytes": 1100533026, + "network_bytes": 615384081, + "relay_server_network_bytes": 0, + "archive_bytes": 615384081, + "files": 84, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.535010085 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 369.179075709 + }, + { + "name": "peer-private-upload", + "durationSeconds": 28.475871305 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 28.424556805 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 430.455847529 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 65.67194503 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000057", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "strategy": "direct-runtime", + "duration_seconds": 356.57969522476196, + "logical_bytes": 1109173016, + "network_bytes": 619789160, + "relay_server_network_bytes": 0, + "archive_bytes": 619789160, + "files": 84, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.411485834 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 242.299948401 + }, + { + "name": "peer-private-upload", + "durationSeconds": 56.499335151 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 21.550417926 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 324.924833689 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 31.655269014 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000058", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "strategy": "direct-runtime", + "duration_seconds": 512.8238842487335, + "logical_bytes": 1112179754, + "network_bytes": 624696001, + "relay_server_network_bytes": 0, + "archive_bytes": 624696001, + "files": 84, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.635526543 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 372.048713043 + }, + { + "name": "peer-private-upload", + "durationSeconds": 49.851714648 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 26.797397262 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 453.112882288 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 59.711382235 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000058", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "strategy": "direct-runtime", + "duration_seconds": 512.8238472938538, + "logical_bytes": 1009981724, + "network_bytes": 615384081, + "relay_server_network_bytes": 0, + "archive_bytes": 615384081, + "files": 39, + "snapshot_reused": null, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.626107168 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 363.566953039 + }, + { + "name": "peer-private-upload", + "durationSeconds": 51.233791774 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 28.567688346 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 453.112882288 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 59.711382235 + } + ] + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000058", + "producer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "strategy": "direct-runtime", + "duration_seconds": 512.824212551117, + "logical_bytes": 1018621714, + "network_bytes": 619789160, + "relay_server_network_bytes": 0, + "archive_bytes": 619789160, + "files": 39, + "snapshot_reused": true, + "ssh_channel_wait_seconds": null, + "rsync_execution_seconds": null, + "phases": [ + { + "name": "peer-authenticated-connectivity-probe", + "durationSeconds": 2.4875807930000002 + }, + { + "name": "peer-snapshot-acquire", + "durationSeconds": 241.861008609 + }, + { + "name": "peer-reused-source-manifest-check", + "durationSeconds": 69.006215656 + }, + { + "name": "peer-private-upload", + "durationSeconds": 32.397370556 + }, + { + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 17.812518174 + }, + { + "name": "shared-preparation-barrier", + "durationSeconds": 453.112882288 + }, + { + "name": "shared-conflict-check-copy-and-verify", + "durationSeconds": 59.711382235 + } + ] + } + ], + "shared_preparations": [ + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "barrier_seconds": 409.363066935, + "merge_seconds": 204.970025468, + "merge_check_seconds": 104.40285946900076, + "merge_copy_seconds": 63.35295432800012, + "merge_verify_seconds": 36.6612788880002 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "barrier_seconds": 420.8945164, + "merge_seconds": 233.922758107, + "merge_check_seconds": 136.61670997599958, + "merge_copy_seconds": 55.04701536900029, + "merge_verify_seconds": 41.70437811000011 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "barrier_seconds": 619.040034573, + "merge_seconds": 283.013860171, + "merge_check_seconds": 180.43556642900057, + "merge_copy_seconds": 55.05433872699996, + "merge_verify_seconds": 46.694435960999726 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013", + "barrier_seconds": 270.108200332, + "merge_seconds": 11.762270089, + "merge_check_seconds": 5.486503027999788, + "merge_copy_seconds": 3.552520055000059, + "merge_verify_seconds": 2.1664215890004925 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014", + "barrier_seconds": 260.111651953, + "merge_seconds": 20.258116134, + "merge_check_seconds": 5.184462961000463, + "merge_copy_seconds": 10.599890837000203, + "merge_verify_seconds": 3.962181805 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "barrier_seconds": 743.329154213, + "merge_seconds": 20.317531426, + "merge_check_seconds": 7.821334353648126, + "merge_copy_seconds": 5.807759420014918, + "merge_verify_seconds": 5.693534042686224 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016", + "barrier_seconds": 31.269985598, + "merge_seconds": 147.835616818, + "merge_check_seconds": 61.76992534500005, + "merge_copy_seconds": 43.9681134829998, + "merge_verify_seconds": 37.762985756000035 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032", + "barrier_seconds": 251.763697781, + "merge_seconds": 19.862637800999998, + "merge_check_seconds": 5.1073565540000345, + "merge_copy_seconds": 10.127387417000136, + "merge_verify_seconds": 4.0069172550001895 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033", + "barrier_seconds": 274.3637555, + "merge_seconds": 21.238856801, + "merge_check_seconds": 6.001633186000163, + "merge_copy_seconds": 10.238027299999885, + "merge_verify_seconds": 4.398234144000071 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034", + "barrier_seconds": 262.366893245, + "merge_seconds": 8.529488046, + "merge_check_seconds": 3.3208714380002675, + "merge_copy_seconds": 2.373866808999992, + "merge_verify_seconds": 2.3390101660002074 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035", + "barrier_seconds": 256.668754117, + "merge_seconds": 19.895121884, + "merge_check_seconds": 5.111687399999937, + "merge_copy_seconds": 10.590446376999807, + "merge_verify_seconds": 3.6762618950001524 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "barrier_seconds": 396.479874348, + "merge_seconds": 18.244609717, + "merge_check_seconds": 6.96128227096051, + "merge_copy_seconds": 4.972433067858219, + "merge_verify_seconds": 5.125589379109442 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052", + "barrier_seconds": 241.653465194, + "merge_seconds": 19.212322008, + "merge_check_seconds": 4.720261811, + "merge_copy_seconds": 10.241356709999991, + "merge_verify_seconds": 3.6985617479999746 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053", + "barrier_seconds": 127.217878267, + "merge_seconds": 143.72586194, + "merge_check_seconds": 63.98408541500021, + "merge_copy_seconds": 41.50971212700006, + "merge_verify_seconds": 36.26938559099972 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054", + "barrier_seconds": 55.324760233, + "merge_seconds": 19.706502426, + "merge_check_seconds": 5.005632080999931, + "merge_copy_seconds": 10.394144000999859, + "merge_verify_seconds": 3.7874770620001073 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "barrier_seconds": 590.19212673, + "merge_seconds": 17.727275007, + "merge_check_seconds": 3.7636981350001406, + "merge_copy_seconds": 9.591068649000135, + "merge_verify_seconds": 3.8570948420001514 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "barrier_seconds": 248.196333155, + "merge_seconds": 17.0898088, + "merge_check_seconds": 3.7757816530001946, + "merge_copy_seconds": 9.017431609999676, + "merge_verify_seconds": 3.681647571999747 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "barrier_seconds": 57.083123277, + "merge_seconds": 6.572613712, + "merge_check_seconds": 2.1952739789999214, + "merge_copy_seconds": 1.654390784000043, + "merge_verify_seconds": 2.235217118000037 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "barrier_seconds": 616.375398324, + "merge_seconds": 36.308577057, + "merge_check_seconds": 19.713207182474434, + "merge_copy_seconds": 8.11883544549346, + "merge_verify_seconds": 6.6765945460647345 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "barrier_seconds": 149.304178902, + "merge_seconds": 35.7947211, + "merge_check_seconds": 13.769190284000047, + "merge_copy_seconds": 17.622240934000047, + "merge_verify_seconds": 3.8217569079999976 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "barrier_seconds": 170.594113411, + "merge_seconds": 33.70487864, + "merge_check_seconds": 12.323312722999958, + "merge_copy_seconds": 17.040445356000077, + "merge_verify_seconds": 3.7904755990000467 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000005", + "barrier_seconds": 57.534868151, + "merge_seconds": 8.935036004, + "merge_check_seconds": 2.5486268370000005, + "merge_copy_seconds": 4.054626322999994, + "merge_verify_seconds": 1.7719136200000065 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000006", + "barrier_seconds": 27.384603138, + "merge_seconds": 35.14418639, + "merge_check_seconds": 13.304883715000003, + "merge_copy_seconds": 8.263048659000006, + "merge_verify_seconds": 12.961258418 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000007", + "barrier_seconds": 178.471809789, + "merge_seconds": 9.025138212, + "merge_check_seconds": 2.433638842999926, + "merge_copy_seconds": 4.427479200000107, + "merge_verify_seconds": 1.6703503870000986 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000008", + "barrier_seconds": 106.084877298, + "merge_seconds": 10.290322213, + "merge_check_seconds": 2.4767447499998525, + "merge_copy_seconds": 4.589167368999824, + "merge_verify_seconds": 2.686420623999993 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009", + "barrier_seconds": 492.625308808, + "merge_seconds": 12.317526589, + "merge_check_seconds": 4.003067781217396, + "merge_copy_seconds": 2.5864740572869778, + "merge_verify_seconds": 4.037082724273205 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000010", + "barrier_seconds": 136.770193771, + "merge_seconds": 8.514610588, + "merge_check_seconds": 2.3380270459999792, + "merge_copy_seconds": 4.057291421000002, + "merge_verify_seconds": 1.607678925000016 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000024", + "barrier_seconds": 126.530978517, + "merge_seconds": 6.69817192, + "merge_check_seconds": 1.788572815000009, + "merge_copy_seconds": 2.941448582000021, + "merge_verify_seconds": 1.4724969700000088 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000025", + "barrier_seconds": 23.791440219, + "merge_seconds": 11.014713296, + "merge_check_seconds": 2.3786062209999272, + "merge_copy_seconds": 3.5944330539998646, + "merge_verify_seconds": 4.525038675000133 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000026", + "barrier_seconds": 50.655918857, + "merge_seconds": 33.648178473, + "merge_check_seconds": 13.589566594000019, + "merge_copy_seconds": 7.7564536109999835, + "merge_verify_seconds": 11.505252552999991 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000027", + "barrier_seconds": 136.929768104, + "merge_seconds": 6.943490004, + "merge_check_seconds": 1.5983100620001096, + "merge_copy_seconds": 3.012553146999835, + "merge_verify_seconds": 1.3373702659998798 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000028", + "barrier_seconds": 66.412851905, + "merge_seconds": 72.725267992, + "merge_check_seconds": 31.009929966000072, + "merge_copy_seconds": 15.483342659000073, + "merge_verify_seconds": 24.76224385100022 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000029", + "barrier_seconds": 32.434681389, + "merge_seconds": 8.983081629, + "merge_check_seconds": 3.8248352260000047, + "merge_copy_seconds": 1.7281099329999279, + "merge_verify_seconds": 2.820531891000087 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000043", + "barrier_seconds": 71.865525409, + "merge_seconds": 3.575245252, + "merge_check_seconds": 0.693428566999998, + "merge_copy_seconds": 1.7207255590000017, + "merge_verify_seconds": 0.6838429230000003 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000044", + "barrier_seconds": 166.81424291, + "merge_seconds": 6.586628378, + "merge_check_seconds": 1.7346229369999833, + "merge_copy_seconds": 2.7410267939999926, + "merge_verify_seconds": 1.5872935680000069 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045", + "barrier_seconds": 92.808933792, + "merge_seconds": 8.27773367, + "merge_check_seconds": 2.257804854772985, + "merge_copy_seconds": 2.1391556449234486, + "merge_verify_seconds": 2.9420743314549327 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000046", + "barrier_seconds": 391.325458929, + "merge_seconds": 10.55416438, + "merge_check_seconds": 1.8772381376475096, + "merge_copy_seconds": 2.296949239447713, + "merge_verify_seconds": 3.096043976955116 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000047", + "barrier_seconds": 161.79732924, + "merge_seconds": 40.70646006, + "merge_check_seconds": 16.786035868, + "merge_copy_seconds": 12.482960667000043, + "merge_verify_seconds": 9.981046372000037 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000048", + "barrier_seconds": 74.835036493, + "merge_seconds": 35.222976348, + "merge_check_seconds": 14.506336318999956, + "merge_copy_seconds": 9.039638048999905, + "merge_verify_seconds": 10.219984688000068 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "barrier_seconds": 395.404969888, + "merge_seconds": 54.476385108, + "merge_check_seconds": 18.960385158999998, + "merge_copy_seconds": 28.01598616299998, + "merge_verify_seconds": 6.9480845240000235 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "barrier_seconds": 336.609416195, + "merge_seconds": 236.51497886, + "merge_check_seconds": 59.98474574200009, + "merge_copy_seconds": 99.47461752399977, + "merge_verify_seconds": 76.27340309400006 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "barrier_seconds": 886.529509405, + "merge_seconds": 89.456599624, + "merge_check_seconds": 22.531136283000706, + "merge_copy_seconds": 27.73135644500053, + "merge_verify_seconds": 38.61186166399966 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000019", + "barrier_seconds": 472.592960508, + "merge_seconds": 17.00389159, + "merge_check_seconds": 6.863229503000184, + "merge_copy_seconds": 5.7328069579998555, + "merge_verify_seconds": 3.802742183999726 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000038", + "barrier_seconds": 430.455847529, + "merge_seconds": 65.67194503, + "merge_check_seconds": 9.1708560510001, + "merge_copy_seconds": 16.714276274999975, + "merge_verify_seconds": 39.025672160999875 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000057", + "barrier_seconds": 324.924833689, + "merge_seconds": 31.655269014, + "merge_check_seconds": 7.015984723999281, + "merge_copy_seconds": 17.19763509999939, + "merge_verify_seconds": 6.89889297100035 + }, + { + "consumer_activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000058", + "barrier_seconds": 453.112882288, + "merge_seconds": 59.711382235, + "merge_check_seconds": 15.396912143999543, + "merge_copy_seconds": 25.130958781000118, + "merge_verify_seconds": 18.612263287000133 + } + ], + "activities": [ + { + "activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "resource_id": "montage-gcp-fog-e2-micro-10", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 3.1248689280014332, + "observed_runtime_seconds": 7.952651023864746, + "validated_handle_runtime_seconds": 7.952651023864746, + "terminal_observation_delay_seconds": 1.5457541942596436, + "preparation_elapsed_seconds": 614.33243227005, + "overlapping_transfer_work_seconds": 3071.658839225769, + "launch_wait_seconds": 33.0821852684021, + "queue_seconds": 652.3926000595093, + "network_bytes": 2728876024 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "resource_id": "montage-gcp-fog-e2-micro-10", + "attempt": 2, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 1.2850327680016562, + "observed_runtime_seconds": 7.94272518157959, + "validated_handle_runtime_seconds": 7.94272518157959, + "terminal_observation_delay_seconds": 1.3128626346588135, + "preparation_elapsed_seconds": 654.8197107315063, + "overlapping_transfer_work_seconds": 3274.097303390503, + "launch_wait_seconds": 30.340030431747437, + "queue_seconds": 685.8661358356476, + "network_bytes": 2264601508 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "resource_id": "montage-gcp-fog-e2-micro-06", + "attempt": 1, + "planned_compute_seconds": 8, + "planned_transfer_seconds": 2.964032768000834, + "observed_runtime_seconds": 8.43309736251831, + "validated_handle_runtime_seconds": 8.43309736251831, + "terminal_observation_delay_seconds": 1.6200916767120361, + "preparation_elapsed_seconds": 902.0540244579315, + "overlapping_transfer_work_seconds": 4510.26855802536, + "launch_wait_seconds": 40.454829692840576, + "queue_seconds": 1221.6100583076477, + "network_bytes": 2711711741 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013", + "resource_id": "montage-gcp-e2-medium-2", + "attempt": 2, + "planned_compute_seconds": 1, + "planned_transfer_seconds": 0.0010655360001692316, + "observed_runtime_seconds": 1.0969316959381104, + "validated_handle_runtime_seconds": 1.0969316959381104, + "terminal_observation_delay_seconds": 5.2375288009643555, + "preparation_elapsed_seconds": 282.7809317111969, + "overlapping_transfer_work_seconds": 564.6518998146057, + "launch_wait_seconds": 9.430667638778687, + "queue_seconds": 292.29327869415283, + "network_bytes": 369736839 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014", + "resource_id": "montage-gcp-fog-e2-micro-04", + "attempt": 2, + "planned_compute_seconds": 4, + "planned_transfer_seconds": 0.0010655360001692316, + "observed_runtime_seconds": 2.4231278896331787, + "validated_handle_runtime_seconds": 2.4231278896331787, + "terminal_observation_delay_seconds": 1.1674253940582275, + "preparation_elapsed_seconds": 281.28264021873474, + "overlapping_transfer_work_seconds": 561.6528487205505, + "launch_wait_seconds": 8.141331195831299, + "queue_seconds": 289.4915339946747, + "network_bytes": 369736839 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 2, + "planned_compute_seconds": 1, + "planned_transfer_seconds": 0.6411638399999902, + "observed_runtime_seconds": 3.896073818206787, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 771.440446138382, + "overlapping_transfer_work_seconds": 1533.6128995418549, + "launch_wait_seconds": 9.15446424484253, + "queue_seconds": 3.746631622314453, + "network_bytes": 1029773873 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016", + "resource_id": "montage-gcp-fog-e2-micro-05", + "attempt": 2, + "planned_compute_seconds": 4, + "planned_transfer_seconds": 0.29466666666712626, + "observed_runtime_seconds": 10.693037271499634, + "validated_handle_runtime_seconds": 10.693037271499634, + "terminal_observation_delay_seconds": 1.557462215423584, + "preparation_elapsed_seconds": 180.0232071876526, + "overlapping_transfer_work_seconds": 359.1293501853943, + "launch_wait_seconds": 69.75866556167603, + "queue_seconds": 249.86078023910522, + "network_bytes": 144937261 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032", + "resource_id": "montage-gcp-fog-e2-micro-08", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.0011310720001347363, + "observed_runtime_seconds": 1.186690092086792, + "validated_handle_runtime_seconds": 1.186690092086792, + "terminal_observation_delay_seconds": 1.2275409698486328, + "preparation_elapsed_seconds": 275.13195848464966, + "overlapping_transfer_work_seconds": 546.7591800689697, + "launch_wait_seconds": 8.144700765609741, + "queue_seconds": 283.3310601711273, + "network_bytes": 363248071 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 100, + "planned_transfer_seconds": 0.0011310720001347363, + "observed_runtime_seconds": 2.5079245567321777, + "validated_handle_runtime_seconds": 2.5079245567321777, + "terminal_observation_delay_seconds": 1.138282060623169, + "preparation_elapsed_seconds": 299.08795189857483, + "overlapping_transfer_work_seconds": 594.6910994052887, + "launch_wait_seconds": 10.515694379806519, + "queue_seconds": 309.67597794532776, + "network_bytes": 363248071 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034", + "resource_id": "montage-gcp-e2-medium-1", + "attempt": 1, + "planned_compute_seconds": 23, + "planned_transfer_seconds": 0.0011310720001347363, + "observed_runtime_seconds": 0.7834560871124268, + "validated_handle_runtime_seconds": 0.7834560871124268, + "terminal_observation_delay_seconds": 1.13071870803833, + "preparation_elapsed_seconds": 271.7986271381378, + "overlapping_transfer_work_seconds": 542.6960201263428, + "launch_wait_seconds": 6.934953212738037, + "queue_seconds": 278.78053975105286, + "network_bytes": 363248071 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035", + "resource_id": "montage-gcp-fog-e2-micro-09", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.24213107200012018, + "observed_runtime_seconds": 1.0729665756225586, + "validated_handle_runtime_seconds": 1.0729665756225586, + "terminal_observation_delay_seconds": 2.045548915863037, + "preparation_elapsed_seconds": 276.564759016037, + "overlapping_transfer_work_seconds": 553.1293814182281, + "launch_wait_seconds": 11.012932062149048, + "queue_seconds": 288.2181205749512, + "network_bytes": 506476142 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "resource_id": "plafrim-execution-ssh-node-diablo03", + "attempt": 1, + "planned_compute_seconds": 16, + "planned_transfer_seconds": 0.080163839999841, + "observed_runtime_seconds": 2.615283489227295, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 581.4387257099152, + "overlapping_transfer_work_seconds": 996.1535928249359, + "launch_wait_seconds": 6.941134214401245, + "queue_seconds": 0.8256244659423828, + "network_bytes": 735942728 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052", + "resource_id": "montage-gcp-fog-e2-micro-01", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 1.2810983040001247, + "observed_runtime_seconds": 2.353093385696411, + "validated_handle_runtime_seconds": 2.353093385696411, + "terminal_observation_delay_seconds": 1.2755682468414307, + "preparation_elapsed_seconds": 260.866628408432, + "overlapping_transfer_work_seconds": 521.7332248687744, + "launch_wait_seconds": 14.253580808639526, + "queue_seconds": 275.75964641571045, + "network_bytes": 474192058 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053", + "resource_id": "montage-gcp-fog-e2-micro-09", + "attempt": 1, + "planned_compute_seconds": 48, + "planned_transfer_seconds": 0.001098304000151984, + "observed_runtime_seconds": 10.19669795036316, + "validated_handle_runtime_seconds": 10.19669795036316, + "terminal_observation_delay_seconds": 1.819373607635498, + "preparation_elapsed_seconds": 271.9007866382599, + "overlapping_transfer_work_seconds": 542.8448679447174, + "launch_wait_seconds": 44.01043939590454, + "queue_seconds": 551.1394593715668, + "network_bytes": 367926386 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054", + "resource_id": "montage-gcp-fog-e2-micro-03", + "attempt": 1, + "planned_compute_seconds": 48, + "planned_transfer_seconds": 0.29576497066727825, + "observed_runtime_seconds": 0.9560744762420654, + "validated_handle_runtime_seconds": 0.9560744762420654, + "terminal_observation_delay_seconds": 1.2141711711883545, + "preparation_elapsed_seconds": 75.0317952632904, + "overlapping_transfer_work_seconds": 150.06357645988464, + "launch_wait_seconds": 8.148884773254395, + "queue_seconds": 85.98558163642883, + "network_bytes": 512393009 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "resource_id": "montage-gcp-fog-e2-micro-05", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.080163839999841, + "observed_runtime_seconds": 0.6130115985870361, + "validated_handle_runtime_seconds": 0.6130115985870361, + "terminal_observation_delay_seconds": 22.618241548538208, + "preparation_elapsed_seconds": 607.9305422306061, + "overlapping_transfer_work_seconds": 607.9305422306061, + "launch_wait_seconds": 9.507911682128906, + "queue_seconds": 618.0661466121674, + "network_bytes": 739568420 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "resource_id": "montage-gcp-fog-e2-micro-04", + "attempt": 1, + "planned_compute_seconds": 28, + "planned_transfer_seconds": 0.0010327680001864792, + "observed_runtime_seconds": 0.5788273811340332, + "validated_handle_runtime_seconds": 0.5788273811340332, + "terminal_observation_delay_seconds": 1.125159502029419, + "preparation_elapsed_seconds": 265.28645491600037, + "overlapping_transfer_work_seconds": 265.28645491600037, + "launch_wait_seconds": 6.942858934402466, + "queue_seconds": 272.8460326194763, + "network_bytes": 363247666 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "resource_id": "montage-gcp-e2-medium-1", + "attempt": 1, + "planned_compute_seconds": 20, + "planned_transfer_seconds": 0.0010327680001864792, + "observed_runtime_seconds": 0.6775872707366943, + "validated_handle_runtime_seconds": 0.6775872707366943, + "terminal_observation_delay_seconds": 1.002912998199463, + "preparation_elapsed_seconds": 63.65419936180115, + "overlapping_transfer_work_seconds": 63.65419936180115, + "launch_wait_seconds": 9.041228294372559, + "queue_seconds": 76.65943002700806, + "network_bytes": 367926467 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 17, + "planned_transfer_seconds": 0.40215015679905264, + "observed_runtime_seconds": 2.7550485134124756, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 656.0198519229889, + "overlapping_transfer_work_seconds": 3919.4310834407806, + "launch_wait_seconds": 46.325281858444214, + "queue_seconds": 0.22789573669433594, + "network_bytes": 2088830200 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "resource_id": "montage-gcp-fog-e2-micro-01", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.005163840000705022, + "observed_runtime_seconds": 0.22596335411071777, + "validated_handle_runtime_seconds": 0.22596335411071777, + "terminal_observation_delay_seconds": 0.5108873844146729, + "preparation_elapsed_seconds": 185.99872636795044, + "overlapping_transfer_work_seconds": 1111.4919044971466, + "launch_wait_seconds": 12.075294494628906, + "queue_seconds": 198.1125295162201, + "network_bytes": 1023400684 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 20, + "planned_transfer_seconds": 0.16342598400024144, + "observed_runtime_seconds": 0.21471381187438965, + "validated_handle_runtime_seconds": 0.21471381187438965, + "terminal_observation_delay_seconds": 0.55808424949646, + "preparation_elapsed_seconds": 205.23656487464905, + "overlapping_transfer_work_seconds": 1226.7216742038727, + "launch_wait_seconds": 148.63358426094055, + "queue_seconds": 353.89586639404297, + "network_bytes": 1437023896 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000005", + "resource_id": "montage-gcp-fog-e2-micro-01", + "attempt": 1, + "planned_compute_seconds": 120, + "planned_transfer_seconds": 0.44249999999988177, + "observed_runtime_seconds": 2.447896718978882, + "validated_handle_runtime_seconds": 2.447896718978882, + "terminal_observation_delay_seconds": 2.098862648010254, + "preparation_elapsed_seconds": 66.46959853172302, + "overlapping_transfer_work_seconds": 132.939138174057, + "launch_wait_seconds": 4.859830379486084, + "queue_seconds": 143.45088624954224, + "network_bytes": 286655248 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000006", + "resource_id": "montage-gcp-fog-e2-micro-06", + "attempt": 1, + "planned_compute_seconds": 112, + "planned_transfer_seconds": 0.44250000000010914, + "observed_runtime_seconds": 27.200522899627686, + "validated_handle_runtime_seconds": 27.200522899627686, + "terminal_observation_delay_seconds": 3.40519642829895, + "preparation_elapsed_seconds": 62.52943420410156, + "overlapping_transfer_work_seconds": 125.05849194526672, + "launch_wait_seconds": 20.623772859573364, + "queue_seconds": 145.49950337409973, + "network_bytes": 288025128 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000007", + "resource_id": "montage-gcp-fog-e2-micro-05", + "attempt": 1, + "planned_compute_seconds": 96, + "planned_transfer_seconds": 0.5356666666671117, + "observed_runtime_seconds": 0.982825756072998, + "validated_handle_runtime_seconds": 0.982825756072998, + "terminal_observation_delay_seconds": 0.7109932899475098, + "preparation_elapsed_seconds": 187.49725008010864, + "overlapping_transfer_work_seconds": 374.99428510665894, + "launch_wait_seconds": 5.051562547683716, + "queue_seconds": 198.82776308059692, + "network_bytes": 287875264 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000008", + "resource_id": "montage-gcp-fog-e2-micro-03", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.20149999999989632, + "observed_runtime_seconds": 0.8271548748016357, + "validated_handle_runtime_seconds": 0.8271548748016357, + "terminal_observation_delay_seconds": 9.954599380493164, + "preparation_elapsed_seconds": 117.272714138031, + "overlapping_transfer_work_seconds": 233.64824509620667, + "launch_wait_seconds": 5.0572474002838135, + "queue_seconds": 125.50041651725769, + "network_bytes": 143717245 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009", + "resource_id": "plafrim-execution-ssh-node-diablo03", + "attempt": 1, + "planned_compute_seconds": 11, + "planned_transfer_seconds": 1.202000000000453, + "observed_runtime_seconds": 2.1502537727355957, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 506.4503540992737, + "overlapping_transfer_work_seconds": 1009.8969445228577, + "launch_wait_seconds": 7.573828220367432, + "queue_seconds": 0.45148396492004395, + "network_bytes": 577379744 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000010", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 96, + "planned_transfer_seconds": 0.49616666666724996, + "observed_runtime_seconds": 1.3688268661499023, + "validated_handle_runtime_seconds": 1.3688268661499023, + "terminal_observation_delay_seconds": 3.930633306503296, + "preparation_elapsed_seconds": 145.2849087715149, + "overlapping_transfer_work_seconds": 290.5696609020233, + "launch_wait_seconds": 6.074968099594116, + "queue_seconds": 222.5801088809967, + "network_bytes": 290024386 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000024", + "resource_id": "montage-gcp-fog-e2-micro-01", + "attempt": 1, + "planned_compute_seconds": 64, + "planned_transfer_seconds": 0.3220000000001164, + "observed_runtime_seconds": 1.9339656829833984, + "validated_handle_runtime_seconds": 1.9339656829833984, + "terminal_observation_delay_seconds": 2.4974417686462402, + "preparation_elapsed_seconds": 133.229474067688, + "overlapping_transfer_work_seconds": 266.45878314971924, + "launch_wait_seconds": 5.215738296508789, + "queue_seconds": 164.30715894699097, + "network_bytes": 283693384 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000025", + "resource_id": "montage-gcp-fog-e2-micro-08", + "attempt": 1, + "planned_compute_seconds": 80, + "planned_transfer_seconds": 0.24099999999998545, + "observed_runtime_seconds": 17.59670615196228, + "validated_handle_runtime_seconds": 17.59670615196228, + "terminal_observation_delay_seconds": 5.580717086791992, + "preparation_elapsed_seconds": 35.70151400566101, + "overlapping_transfer_work_seconds": 70.5074212551117, + "launch_wait_seconds": 11.353219747543335, + "queue_seconds": 57.561424016952515, + "network_bytes": 143458908 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000026", + "resource_id": "montage-gcp-fog-e2-micro-06", + "attempt": 1, + "planned_compute_seconds": 64, + "planned_transfer_seconds": 0.40200000000004366, + "observed_runtime_seconds": 12.29628849029541, + "validated_handle_runtime_seconds": 12.29628849029541, + "terminal_observation_delay_seconds": 2.450336456298828, + "preparation_elapsed_seconds": 84.30418920516968, + "overlapping_transfer_work_seconds": 168.6080641746521, + "launch_wait_seconds": 13.897461414337158, + "queue_seconds": 249.9475975036621, + "network_bytes": 284624284 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000027", + "resource_id": "montage-gcp-fog-e2-micro-04", + "attempt": 1, + "planned_compute_seconds": 60, + "planned_transfer_seconds": 0.40200000000004366, + "observed_runtime_seconds": 0.8316574096679688, + "validated_handle_runtime_seconds": 0.8316574096679688, + "terminal_observation_delay_seconds": 0.8443078994750977, + "preparation_elapsed_seconds": 143.87355995178223, + "overlapping_transfer_work_seconds": 287.7470965385437, + "launch_wait_seconds": 4.75709867477417, + "queue_seconds": 151.59686160087585, + "network_bytes": 285756079 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000028", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 79.99999999999977, + "planned_transfer_seconds": 0.24099999999998545, + "observed_runtime_seconds": 15.397364139556885, + "validated_handle_runtime_seconds": 15.397364139556885, + "terminal_observation_delay_seconds": 1.623500108718872, + "preparation_elapsed_seconds": 140.04533123970032, + "overlapping_transfer_work_seconds": 279.18388175964355, + "launch_wait_seconds": 19.72241735458374, + "queue_seconds": 162.13104033470154, + "network_bytes": 143228071 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000029", + "resource_id": "montage-gcp-e2-medium-2", + "attempt": 1, + "planned_compute_seconds": 13, + "planned_transfer_seconds": 0.24099999999998545, + "observed_runtime_seconds": 8.676194667816162, + "validated_handle_runtime_seconds": 8.676194667816162, + "terminal_observation_delay_seconds": 3.5752487182617188, + "preparation_elapsed_seconds": 42.36921787261963, + "overlapping_transfer_work_seconds": 83.78652477264404, + "launch_wait_seconds": 10.466334342956543, + "queue_seconds": 58.178486824035645, + "network_bytes": 143458908 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000043", + "resource_id": "montage-gcp-fog-e2-micro-05", + "attempt": 1, + "planned_compute_seconds": 44, + "planned_transfer_seconds": 2.1599999999998545, + "observed_runtime_seconds": 4.034695863723755, + "validated_handle_runtime_seconds": 4.034695863723755, + "terminal_observation_delay_seconds": 0.8480274677276611, + "preparation_elapsed_seconds": 75.4400897026062, + "overlapping_transfer_work_seconds": 150.8763988018036, + "launch_wait_seconds": 5.511667728424072, + "queue_seconds": 140.9136393070221, + "network_bytes": 210263353 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000044", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 44, + "planned_transfer_seconds": 0.9620000000002165, + "observed_runtime_seconds": 2.248190402984619, + "validated_handle_runtime_seconds": 2.248190402984619, + "terminal_observation_delay_seconds": 1.1198883056640625, + "preparation_elapsed_seconds": 173.40082049369812, + "overlapping_transfer_work_seconds": 346.8016242980957, + "launch_wait_seconds": 5.880494832992554, + "queue_seconds": 380.23929047584534, + "network_bytes": 248486223 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045", + "resource_id": "plafrim-execution-ssh-node-diablo03", + "attempt": 1, + "planned_compute_seconds": 16, + "planned_transfer_seconds": 0.6410000000005311, + "observed_runtime_seconds": 1.7191390991210938, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 103.11808204650879, + "overlapping_transfer_work_seconds": 204.19298315048218, + "launch_wait_seconds": 4.590351581573486, + "queue_seconds": 0.3003077507019043, + "network_bytes": 288968644 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000046", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 14, + "planned_transfer_seconds": 0.4810000000002219, + "observed_runtime_seconds": 2.307098865509033, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 403.99042654037476, + "overlapping_transfer_work_seconds": 805.860963344574, + "launch_wait_seconds": 15.983623027801514, + "queue_seconds": 0.7625946998596191, + "network_bytes": 289012490 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000047", + "resource_id": "montage-gcp-fog-e2-micro-10", + "attempt": 1, + "planned_compute_seconds": 56, + "planned_transfer_seconds": 1.574666666667099, + "observed_runtime_seconds": 12.452471733093262, + "validated_handle_runtime_seconds": 12.452471733093262, + "terminal_observation_delay_seconds": 9.807345628738403, + "preparation_elapsed_seconds": 202.50408005714417, + "overlapping_transfer_work_seconds": 405.00799655914307, + "launch_wait_seconds": 11.684876441955566, + "queue_seconds": 1324.1596858501434, + "network_bytes": 250732295 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000048", + "resource_id": "montage-gcp-fog-e2-micro-09", + "attempt": 1, + "planned_compute_seconds": 76, + "planned_transfer_seconds": 0.29466666666712626, + "observed_runtime_seconds": 12.960306882858276, + "validated_handle_runtime_seconds": 12.960306882858276, + "terminal_observation_delay_seconds": 1.5392072200775146, + "preparation_elapsed_seconds": 111.15899515151978, + "overlapping_transfer_work_seconds": 221.21673774719238, + "launch_wait_seconds": 12.4910147190094, + "queue_seconds": 127.10087728500366, + "network_bytes": 144466623 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 88, + "planned_transfer_seconds": 3.1238361600012468, + "observed_runtime_seconds": 4.462711572647095, + "validated_handle_runtime_seconds": 4.462711572647095, + "terminal_observation_delay_seconds": 1.361527681350708, + "preparation_elapsed_seconds": 449.87099862098694, + "overlapping_transfer_work_seconds": 1799.482433795929, + "launch_wait_seconds": 13.905594825744629, + "queue_seconds": 466.47415947914124, + "network_bytes": 2132325441 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "resource_id": "montage-gcp-fog-e2-micro-03", + "attempt": 1, + "planned_compute_seconds": 100, + "planned_transfer_seconds": 1.2840000000014697, + "observed_runtime_seconds": 3.456937313079834, + "validated_handle_runtime_seconds": 3.456937313079834, + "terminal_observation_delay_seconds": 1.3075461387634277, + "preparation_elapsed_seconds": 573.1351981163025, + "overlapping_transfer_work_seconds": 2292.540068626404, + "launch_wait_seconds": 81.0121717453003, + "queue_seconds": 654.7839007377625, + "network_bytes": 1677179784 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 2, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 2.9630000000006476, + "observed_runtime_seconds": 2.939439296722412, + "validated_handle_runtime_seconds": 2.939439296722412, + "terminal_observation_delay_seconds": 1.8218133449554443, + "preparation_elapsed_seconds": 975.9902129173279, + "overlapping_transfer_work_seconds": 3903.959368944168, + "launch_wait_seconds": 60.21223521232605, + "queue_seconds": 1036.8853514194489, + "network_bytes": 2119800793 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000001", + "resource_id": "montage-gcp-e2-medium-2", + "attempt": 1, + "planned_compute_seconds": 2509.0870506376314, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 626.2200429439545, + "validated_handle_runtime_seconds": 626.2200429439545, + "terminal_observation_delay_seconds": 2.493950366973877, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 136.95378637313843, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000002", + "resource_id": "montage-gcp-fog-e2-micro-04", + "attempt": 1, + "planned_compute_seconds": 1656, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1598.1546354293823, + "validated_handle_runtime_seconds": 1598.1546354293823, + "terminal_observation_delay_seconds": 4.127002000808716, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 127.45114874839783, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000003", + "resource_id": "montage-gcp-fog-e2-micro-03", + "attempt": 1, + "planned_compute_seconds": 2200, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1567.5464174747467, + "validated_handle_runtime_seconds": 1567.5464174747467, + "terminal_observation_delay_seconds": 3.6765007972717285, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 103.76896691322327, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000004", + "resource_id": "montage-gcp-fog-e2-micro-10", + "attempt": 1, + "planned_compute_seconds": 3296, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1629.2610213756561, + "validated_handle_runtime_seconds": 1629.2610213756561, + "terminal_observation_delay_seconds": 2.117905855178833, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 121.09256625175476, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000020", + "resource_id": "montage-gcp-fog-e2-micro-08", + "attempt": 1, + "planned_compute_seconds": 2112, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1591.4156937599182, + "validated_handle_runtime_seconds": 1591.4156937599182, + "terminal_observation_delay_seconds": 1.9995102882385254, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 122.06359219551086, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000021", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 1444, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1808.3638439178467, + "validated_handle_runtime_seconds": 1808.3638439178467, + "terminal_observation_delay_seconds": 3.2913191318511963, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 116.37402749061584, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000022", + "resource_id": "montage-gcp-e2-medium-1", + "attempt": 1, + "planned_compute_seconds": 1953.1603327863493, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 527.7171688079834, + "validated_handle_runtime_seconds": 527.7171688079834, + "terminal_observation_delay_seconds": 0.7628450393676758, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 122.17907047271729, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000023", + "resource_id": "montage-gcp-e2-medium-2", + "attempt": 1, + "planned_compute_seconds": 1974.0870506376316, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 469.87716364860535, + "validated_handle_runtime_seconds": 469.87716364860535, + "terminal_observation_delay_seconds": 1.5030717849731445, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 136.99560546875, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000039", + "resource_id": "plafrim-execution-ssh-node-diablo03", + "attempt": 1, + "planned_compute_seconds": 1540, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 435.39326882362366, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 0.007843494415283203, + "overlapping_transfer_work_seconds": 0.007843494415283203, + "launch_wait_seconds": 5.185002326965332, + "queue_seconds": 2.2521579265594482, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000040", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 1537, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 428.0586898326874, + "validated_handle_runtime_seconds": null, + "terminal_observation_delay_seconds": null, + "preparation_elapsed_seconds": 0.028724193572998047, + "overlapping_transfer_work_seconds": 0.028724193572998047, + "launch_wait_seconds": 5.3885817527771, + "queue_seconds": 1.9199984073638916, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000041", + "resource_id": "montage-gcp-fog-e2-micro-09", + "attempt": 1, + "planned_compute_seconds": 3012, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1655.5500555038452, + "validated_handle_runtime_seconds": 1655.5500555038452, + "terminal_observation_delay_seconds": 2.816364049911499, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 136.28686261177063, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000042", + "resource_id": "montage-gcp-e2-medium-1", + "attempt": 1, + "planned_compute_seconds": 2379.1603327863495, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 530.1431140899658, + "validated_handle_runtime_seconds": 530.1431140899658, + "terminal_observation_delay_seconds": 4.37717342376709, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 122.03553557395935, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000019", + "resource_id": "montage-gcp-e2-medium-1", + "attempt": 1, + "planned_compute_seconds": 17, + "planned_transfer_seconds": 0.1610000000000582, + "observed_runtime_seconds": 2.4861814975738525, + "validated_handle_runtime_seconds": 2.4861814975738525, + "terminal_observation_delay_seconds": 0.9068527221679688, + "preparation_elapsed_seconds": 489.5976278781891, + "overlapping_transfer_work_seconds": 489.5976278781891, + "launch_wait_seconds": 8.551166534423828, + "queue_seconds": 501.0140976905823, + "network_bytes": 624696001 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000038", + "resource_id": "montage-gcp-fog-e2-micro-06", + "attempt": 1, + "planned_compute_seconds": 32, + "planned_transfer_seconds": 0.1610000000000582, + "observed_runtime_seconds": 26.953650951385498, + "validated_handle_runtime_seconds": 26.953650951385498, + "terminal_observation_delay_seconds": 1.5477750301361084, + "preparation_elapsed_seconds": 496.11766481399536, + "overlapping_transfer_work_seconds": 496.11766481399536, + "launch_wait_seconds": 55.2406165599823, + "queue_seconds": 554.2020151615143, + "network_bytes": 615384081 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000057", + "resource_id": "montage-gcp-fog-e2-micro-08", + "attempt": 2, + "planned_compute_seconds": 4, + "planned_transfer_seconds": 0.1610000000000582, + "observed_runtime_seconds": 21.950023651123047, + "validated_handle_runtime_seconds": 21.950023651123047, + "terminal_observation_delay_seconds": 3.141009569168091, + "preparation_elapsed_seconds": 356.57969522476196, + "overlapping_transfer_work_seconds": 356.57969522476196, + "launch_wait_seconds": 21.604023694992065, + "queue_seconds": 380.9324095249176, + "network_bytes": 619789160 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000058", + "resource_id": "montage-gcp-e2-medium-2", + "attempt": 2, + "planned_compute_seconds": 7, + "planned_transfer_seconds": 0.4830000000001746, + "observed_runtime_seconds": 4.621002435684204, + "validated_handle_runtime_seconds": 4.621002435684204, + "terminal_observation_delay_seconds": 0.4313650131225586, + "preparation_elapsed_seconds": 512.824212551117, + "overlapping_transfer_work_seconds": 1538.4719440937042, + "launch_wait_seconds": 24.893104076385498, + "queue_seconds": 540.8094902038574, + "network_bytes": 1859869242 + } + ], + "limitations": [ + "Multiple attempts: performance can be contaminated by recovery.", + "Runtime finish semantics depend on the driver; observation delay is exported only for exact-attempt handles with validated terminal timestamps.", + "API aggregate counters can contain older attempts; audit uses latest distinct IDs.", + "Transfer work is summed overlapping route time, not wall-clock preparation.", + "Plan assignment work sums are not makespan; API timestamp wall and recovered makespan can differ.", + "Cost zero is uncalibrated, not free cloud execution.", + "e2-medium interference extrapolated to HPC/micro; micro speedup 0.25 is approximate.", + "Whole-workspace real traffic differs from estimated plan edge volumes.", + "Zero reported interference does not prove absence of physical contention." + ], + "product": { + "filename": "mosaic-color.png", + "bytes": 4920343, + "sha256": "0f75c2e811e89ac56ae44784fa22e4e01368abf19a10cbe59a77bd19446fdf4a" + }, + "cleanup": { + "cloud_instances_destroyed": 12, + "non_destroyed_inventory_count": 0, + "active_cloud_operations": 0, + "hpc_squeue_jobs": 0, + "final_product_export_verified": true, + "gcp_managed_instances_residual_count": 0, + "gcp_managed_disks_residual_count": 0, + "gcp_this_run_firewalls_residual_count": 0, + "unrelated_older_managed_firewall_count": 3 + } +} diff --git a/experiments/montage-micro-hpc-e2/remote-service-instrumentation-20261011.json b/experiments/montage-micro-hpc-e2/remote-service-instrumentation-20261011.json new file mode 100644 index 00000000..a64ca546 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/remote-service-instrumentation-20261011.json @@ -0,0 +1,31 @@ +{ + "issue": "AKO-158", + "implementationCommit": "c633b8f", + "role": "instrumentation-readiness-not-model-validation", + "deploymentSafePoint": { + "activeExecutionRuns": 0, + "activeCloudOperations": 0 + }, + "sourceSnapshotDiagnostics": [ + "snapshotBuildSshWaitSeconds", + "snapshotBuildCommandSeconds", + "snapshotBuildServiceSeconds", + "snapshotBuildCpuSeconds" + ], + "receiverDiagnostics": [ + "peerReceiverServiceSeconds", + "peerReceiverCpuSeconds" + ], + "nativePublicationFixture": { + "taskCount": 3, + "makespanSeconds": 15, + "appliedDaemonBinaryVerified": true + }, + "hpcConnectionHealth": "online", + "limitations": [ + "Remote wall time excludes local SSH admission and handshake but includes source scheduling and IO contention.", + "CPU time measures child user plus system consumption, not exclusive CPU occupancy or queue-free wall service.", + "Cached snapshot measurements refer to the original shared build and must not be charged again per consumer.", + "These measurements have not yet been collected in a new real run. The dominant transfer residual gate remains unsatisfied." + ] +} diff --git a/experiments/montage-micro-hpc-e2/rolling-storage-bound-20261011.json b/experiments/montage-micro-hpc-e2/rolling-storage-bound-20261011.json new file mode 100644 index 00000000..db2ffc05 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/rolling-storage-bound-20261011.json @@ -0,0 +1,13 @@ +{ + "issue": "AKO-161", + "plan": "montage-prism-all-hpc-speedup-20261011-v1", + "status": "offline-diagnostic-not-deployed", + "runtimeContract": "Producer leases released after successful consumer preparation; workspace reclaimed only after every downstream lease released. Leaves retained/pruned to final outputs.", + "scratchContract": "WorkspaceRsync preparation scratch cleanup runs before preparation returns; forecast conservatively retains two input copies until consumer command barrier.", + "oldReservedUpperBoundBytesApprox": 100432511613, + "rollingReservedUpperBoundBytes": 41020996139, + "referenceAvailableBytes": 20091539456, + "fitsQuota": false, + "realSubmitted": false, + "limitation": "Predicted lifetime bound, not observed physical disk peak. Parallelism, producer fanout and retained leaves still consume simultaneous space. No hardlinks or unverified cleanup credit." +} diff --git a/experiments/montage-micro-hpc-e2/runtime-admission-holdout-20261011.json b/experiments/montage-micro-hpc-e2/runtime-admission-holdout-20261011.json new file mode 100644 index 00000000..1dc4f39f --- /dev/null +++ b/experiments/montage-micro-hpc-e2/runtime-admission-holdout-20261011.json @@ -0,0 +1,28 @@ +{ + "issue":"AKO-167", + "mode":"offline-fixed-placement-native-SimGrid-holdout", + "trainingRun":"montage-prism-all-hpc-real-20261011-v1", + "holdoutRun":"montage-prism-all-hpc-real-20261011-v2", + "trainingSamples":34, + "admissionSecondsMedian":17.318344116210938, + "admissionSource":"native handle submittedAt to startedAt; single-attempt completed samples only; noncausal timestamp samples excluded", + "plannedMakespanSeconds":1164.2299096115516, + "simulatedMakespanSeconds":1164.2299096115516, + "holdoutMakespanSeconds":1296.0180189609528, + "realAbovePlannedPercentApprox":11.32, + "activityWallWithin20Percent":31, + "activityCount":58, + "medianActivityWallErrorPercent":14.901861081330308, + "maximumActivityWallErrorPercent":59.05136490848852, + "nativeFixture":{"preparationSeconds":4,"admissionSeconds":7,"runtimeOverheadSeconds":2,"computeSeconds":3,"publicationSeconds":5,"makespanSeconds":21}, + "newRealSubmitted":false, + "daemonRestarted":false, + "goalComplete":false, + "limitations":[ + "Current mixed cloud/HPC holdout already meets aggregate tolerance, but stage residuals remain and e2-medium lacks clean independent makespan holdout.", + "Admission profile represents measured external wall latency, not CPU work and not a guarantee of future Slurm queue time.", + "Runtime identity is matched explicitly; absent native submission telemetry does not inherit the Slurm delay.", + "Three gateway fallback dependencies in observed HPC holdout are not represented by the internal-route baseline.", + "Preparation/publication fits still include unidentified polling/admission effects; remaining service isolation needs stronger telemetry." + ] +} diff --git a/experiments/montage-micro-hpc-e2/runtime-stage-holdout-20261011.json b/experiments/montage-micro-hpc-e2/runtime-stage-holdout-20261011.json new file mode 100644 index 00000000..13400216 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/runtime-stage-holdout-20261011.json @@ -0,0 +1,33 @@ +{ + "issue":"AKO-166", + "mode":"offline-fixed-placement-evaluation-and-native-SimGrid", + "newRealSubmitted":false, + "daemonRestarted":false, + "globalMakespanMultiplierUsed":false, + "stageRules":[ + "Merge component service fitted once per consumer, including peer/relay routes without folding network into merge.", + "Initial workspace scan is moved from runtime overhead to preparation, exactly once.", + "Network-route residual cannot be interpreted as input scan; direct scan telemetry is required for cloud.", + "Different runtime drivers cannot be mixed in a single class calibration.", + "Training workspace projection uses actual training resource types, not holdout placement.", + "Canonical activity names align historical IDs exactly; no family-only joins.", + "Fixed placement evaluator updates PRISM forecast without changing resource/core/order." + ], + "cases":[ + {"scope":"HPC Slurm","training":"montage-prism-all-hpc-real-20261011-v1, single-attempt subset","holdout":"montage-prism-all-hpc-real-20261011-v2","plannedSeconds":1021.6136364533885,"simulatedSeconds":1021.6136364533885,"observedMakespanSeconds":1296.0180189609528,"absoluteMakespanErrorPercentApprox":21.17,"activityWallWithin20Percent":26,"activityCount":58,"aggregateWithin25Percent":true,"fullyValidated":false}, + {"scope":"mixed cloud e2-micro / HPC Slurm; cloud stage calibration","training":"montage-docker-image-real-20261010-v2","holdout":"montage-clean-real-20261011-v1","plannedSeconds":2765.959075445268,"simulatedSeconds":2773.638558133343,"observedMakespanSeconds":3150.6341450214386,"absolutePlannedMakespanErrorPercentApprox":12.21,"absoluteSimulatedMakespanErrorPercentApprox":11.9657,"activityWallWithin20Percent":20,"activityCount":58,"medianActivityWallErrorPercent":48.42921507396264,"aggregateWithin25Percent":true,"fullyValidated":false}, + {"scope":"cloud e2-medium","training":"montage-docker-image-real-20261010-v2","mergeSamples":6,"publicationSamples":10,"independentHoldoutAvailable":false,"fullyValidated":false} + ], + "goalComplete":false, + "toleranceDefinition":"For the user target, check abs(real - predicted) / predicted <= 0.25. Report reference-denominator accuracy separately; do not interchange them.", + "realExcessRelativePlannedPercent":{"hpcApprox":26.86,"mixedCloudHpcApprox":13.91}, + "realExcessRelativeSimulatedPercent":{"hpcApprox":26.86,"mixedCloudHpcApprox":13.59}, + "userTolerancePassed":{"hpc":false,"mixedCloudHpcAggregate":true,"e2Medium":"not-verified"}, + "remaining":[ + "Dominant activity residuals still too large; aggregate agreement alone does not establish validation.", + "Need an independent holdout with e2-medium before claiming that class meets tolerance.", + "Cloud training and holdout use different runtime-era overheads; profiles must carry provenance and confidence.", + "Remaining effective-wall publication/preparation coefficients contain unidentified waiting; distinguish CPU/IO service from channel queue before accepting contention model.", + "Recovered cloud/HPC run must not be used as clean makespan holdout." + ] +} diff --git a/experiments/montage-micro-hpc-e2/scan-cleanup-sensitivity-20261011.json b/experiments/montage-micro-hpc-e2/scan-cleanup-sensitivity-20261011.json new file mode 100644 index 00000000..d9619d83 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/scan-cleanup-sensitivity-20261011.json @@ -0,0 +1,16 @@ +{ + "issue":"AKO-158", + "role":"two-feature-wall-fit-not-independent-validation", + "features":["pre-prune terminal snapshot bytes","final workspace inherited-input cleanup bytes"], + "scanSecondsPerWorkspaceByte":2.710558377450728e-8, + "cleanupSecondsPerInputByte":6.068931912359514e-8, + "samples":40, + "fixedPlacementSimGridSeconds":3825.5614337389516, + "simulatedActivities":58, + "limitations":[ + "Reference wall times still include SSH transport and possible admission wait.", + "Final publication cleanup is distinct from later reclamation of intermediate workspaces after their consumers.", + "This fit uses recovered reference observations and is not independent validation.", + "Transfer-service residuals still need validation separately from publication." + ] +} diff --git a/experiments/montage-micro-hpc-e2/single-lane-first-micro-compute-evidence.json b/experiments/montage-micro-hpc-e2/single-lane-first-micro-compute-evidence.json new file mode 100644 index 00000000..9b1116e8 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/single-lane-first-micro-compute-evidence.json @@ -0,0 +1,22 @@ +{ + "runId": "montage-micro-hpc-e2-real-20261008-single-lane-v1", + "status": "partial_running_measurement", + "activityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000002", + "resourceId": "montage-gcp-fog-e2-micro-04", + "executionAttempt": 2, + "activityStatus": "completed", + "handleStatus": "completed", + "source": "Matching execution-attempt task and native handle timestamps from daemon API; selected plan assignment", + "startedAtUnixSeconds": 1791445903.2506673, + "finishedAtUnixSeconds": 1791447535.248157, + "validatedHandleRuntimeSeconds": 1631.997490, + "plannedRuntimeSeconds": 1656, + "differenceSeconds": -24.002510, + "differencePercent": -1.449427, + "limitations": [ + "One completed micro activity; not proof that all 58 activities match the plan.", + "This compares scientific execution duration, excluding provisioning and workspace preparation.", + "The overall run includes recovery and is not an uninterrupted makespan benchmark.", + "The micro speed factor remains approximate and the cloud price remains uncalibrated." + ] +} diff --git a/experiments/montage-micro-hpc-e2/single-lane-gateway-rsync-real-evidence.json b/experiments/montage-micro-hpc-e2/single-lane-gateway-rsync-real-evidence.json new file mode 100644 index 00000000..c530132a --- /dev/null +++ b/experiments/montage-micro-hpc-e2/single-lane-gateway-rsync-real-evidence.json @@ -0,0 +1,21 @@ +{ + "issue": "AKO-130", + "executionRunId": "montage-micro-hpc-e2-real-20261008-single-lane-v1", + "consumer": "mconcatfitid0000049", + "dependencies": [ + {"producer": "mdifffitid0000045", "archiveBytes": 201488310, "packAndReceiveSeconds": 122.883186014, "sshChannelWaitSeconds": 0.0007745, "rsyncExecutionSeconds": 235.721634609, "uploadSeconds": 235.722517859}, + {"producer": "mdifffitid0000046", "archiveBytes": 202642917, "packAndReceiveSeconds": 123.280823721, "sshChannelWaitSeconds": 0.000172042, "rsyncExecutionSeconds": 238.480644068, "uploadSeconds": 238.48131236} + ], + "sharedPreparationSeconds": 371.602997795, + "sharedMergeSeconds": 33.281667057, + "mergeCheckSeconds": 12.408034978000046, + "mergeCopySeconds": 16.593500748999986, + "mergeVerifySeconds": 3.7013017250000075, + "reusableActivitiesAfterSafeCancellation": 37, + "conclusions": [ + "SSH channel acquisition does not explain the measured upload delay.", + "Rsync invocation dominates upload; transport, subprocess execution and execution retries require diagnosis.", + "Shared preparation and merge timings must be counted once, not summed per dependency.", + "Different workspace from earlier mConcatFit30: no controlled merge speedup claim." + ] +} diff --git a/experiments/montage-micro-hpc-e2/single-lane-hpc-upload-diagnostic.json b/experiments/montage-micro-hpc-e2/single-lane-hpc-upload-diagnostic.json new file mode 100644 index 00000000..9bd1f688 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/single-lane-hpc-upload-diagnostic.json @@ -0,0 +1,26 @@ +{ + "issue": "AKO-130", + "runId": "montage-micro-hpc-e2-real-20261008-single-lane-v1", + "consumerActivity": "mdifffitid0000009", + "strategy": "gateway", + "destinationResource": "plafrim-execution-ssh-node-diablo03", + "dependencies": [ + { + "producerActivity": "mprojectid0000002", + "packAndReceiveSeconds": 12.142290673, + "archiveUploadSeconds": 161.00987145, + "receiverVerifyExtractSeconds": 5.710891961 + }, + { + "producerActivity": "mprojectid0000004", + "packAndReceiveSeconds": 19.536244426, + "archiveUploadSeconds": 168.933765369, + "receiverVerifyExtractSeconds": 4.677060627 + } + ], + "jointPreparationSeconds": 198.513253549, + "jointMergeSeconds": 11.517878255, + "interpretation": "Upload phase is dominant but includes SSH channel acquisition, rsync execution and retry waits. Current evidence does not isolate network throughput or prove SSH contention caused the delay.", + "action": "Run safely cancelled with 30 reusable activities. Introduced separate channel-wait and rsync-execution measurements without changing transport security, concurrency limits, whole-workspace contract or execution barriers.", + "limitation": "Joint preparation and merge values are shared across dependencies; count each only once. Instrumentation requires deployment and real measurement before selecting a performance fix." +} diff --git a/experiments/montage-micro-hpc-e2/single-lane-initial-stage-evidence.json b/experiments/montage-micro-hpc-e2/single-lane-initial-stage-evidence.json new file mode 100644 index 00000000..0a56bede --- /dev/null +++ b/experiments/montage-micro-hpc-e2/single-lane-initial-stage-evidence.json @@ -0,0 +1,18 @@ +{ + "runId": "montage-micro-hpc-e2-real-20261008-single-lane-v1", + "status": "partial_running_measurement", + "source": "Daemon API cloud operation timestamps and allowlisted provisioning log milestones", + "resourceId": "montage-gcp-fog-e2-micro-02", + "provisionElapsedSeconds": 349.332897, + "sshReadinessSeconds": 3.466, + "configurationPlaybookSeconds": 269.335, + "dockerValidationSeconds": 2.774, + "workspaceValidationSeconds": 1.640, + "inference": "Configuration playbook is the largest measured component of this instance's initial provisioning delay; this is not dependency-transfer elapsed time.", + "limitations": [ + "No per-task playbook timing available in these milestones; package installation is not individually attributed.", + "Unattributed remainder includes infrastructure creation, trust verification and orchestration; do not label it entirely Terraform.", + "Single instance observation; do not extrapolate to all machines or final makespan.", + "Workflow not yet completed; no final performance conclusion." + ] +} diff --git a/experiments/montage-micro-hpc-e2/single-lane-merge-real-evidence.json b/experiments/montage-micro-hpc-e2/single-lane-merge-real-evidence.json new file mode 100644 index 00000000..7a6e9490 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/single-lane-merge-real-evidence.json @@ -0,0 +1,24 @@ +{ + "issue": "AKO-131", + "implementationCommit": "d891da1", + "executionRunId": "montage-micro-hpc-e2-real-20261008-single-lane-v1", + "activity": "mbgmodelid0000031", + "attempt": 2, + "status": "completed", + "workspaceStrategy": "direct-runtime", + "wholeWorkspace": true, + "mergeCheckSeconds": 3.9360992620000275, + "mergeCopySeconds": 9.166654937000033, + "mergeVerifySeconds": 3.7432944340000063, + "transferDurationSeconds": 118.86065554618835, + "transferFinishedAt": 1791458100.0528774, + "consumerStartedAt": 1791458107.9347284, + "consumerFinishedAt": 1791458108.5117002, + "transferBeforeConsumer": true, + "limitations": [ + "One completed activity proves real execution and the transfer barrier, not full workflow completion.", + "This workspace differs from mConcatFit30; comparison with its prior 159.8218-second merge is not a controlled speedup measurement.", + "Transfer duration includes preparation beyond the three merge stages.", + "Recovery-contaminated run; not an uninterrupted scientific benchmark." + ] +} diff --git a/experiments/montage-micro-hpc-e2/single-lane-micro-peer-fanout-evidence.json b/experiments/montage-micro-hpc-e2/single-lane-micro-peer-fanout-evidence.json new file mode 100644 index 00000000..96926213 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/single-lane-micro-peer-fanout-evidence.json @@ -0,0 +1,29 @@ +{ + "issue": "AKO-129", + "runId": "montage-micro-hpc-e2-real-20261008-single-lane-v1", + "source": "Daemon API transfer phases and latest activity attempt records, 2026-10-08", + "producer": "montage-58-fog-hpc-e2-20261006-mprojectid0000004", + "producerResource": "montage-gcp-fog-e2-micro-10", + "transfers": [ + {"consumerSuffix": "mdifffitid0000007", "archiveBytes": 144937223, "packHashSeconds": 101.026189296, "privateUploadSeconds": 32.782604931, "receiverVerifyExtractSeconds": 4.201669461, "serverPayloadBytes": 0}, + {"consumerSuffix": "mdifffitid0000010", "archiveBytes": 144937223, "packHashSeconds": 100.912997962, "privateUploadSeconds": 35.194665975, "receiverVerifyExtractSeconds": 3.991413586, "serverPayloadBytes": 0} + ], + "comparison": { + "mediumProducerArchiveBytes": 142937977, + "mediumProducerPackHashSeconds": 7.97850542, + "mediumProducerPrivateUploadSeconds": 3.827686085 + }, + "confirmed": "Current peer implementation independently packs and hashes a whole-workspace archive for each consumer; the same micro producer repeated this work for two similarly sized archive transfers.", + "notYetProven": "The relative contributions of micro shared CPU, concurrent compression, disk contention and network throughput require separate validation. Size equality alone does not prove immutable content identity.", + "safePoint": { + "runStatus": "cancelled", + "activeHandleCount": 0, + "activeCloudOperationCount": 0, + "recoverable": true, + "reusableActivityCount": 25, + "cleanupPendingActivityCount": 0 + }, + "preservation": "Required producer workspaces remain retained; no required output was intentionally destroyed. Runtime response is preserved in ignored local storage.", + "requiredFix": "Coordinate sealed immutable snapshots by exact cloud generation and workspace identity, preserving full-workspace content validation, independent dependency routing and the joint consumer barrier. Do not reuse mutable content or remove an archive while a reader owns it.", + "limitations": ["Recovered experiment, not an uninterrupted benchmark.", "Public-cloud pricing and micro speedup remain uncalibrated.", "Goal remains incomplete until 58 activities, physical final product, comparative metrics and paid-resource cleanup are verified."] +} diff --git a/experiments/montage-micro-hpc-e2/single-lane-mixed-route-evidence.json b/experiments/montage-micro-hpc-e2/single-lane-mixed-route-evidence.json new file mode 100644 index 00000000..66bb843a --- /dev/null +++ b/experiments/montage-micro-hpc-e2/single-lane-mixed-route-evidence.json @@ -0,0 +1,38 @@ +{ + "runId": "montage-micro-hpc-e2-real-20261008-single-lane-v1", + "status": "partial_running_measurement", + "consumerActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000025", + "source": "Daemon API completed transfer routes and consumer execution timestamps", + "sameInstanceDependency": { + "producerActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000020", + "strategy": "runtime-local", + "exactConcreteInstanceIdsMatch": true, + "verifiedFilesystem": "ext4", + "serverNetworkBytes": 0, + "archivePackagingUsed": false, + "evidence": "Independently trusted same-instance filesystem proof, runtime-local strategy, absent pack/upload phases; zero-valued network field is omitted in API JSON" + }, + "otherInstanceDependency": { + "producerActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000022", + "strategy": "gateway", + "archiveBytes": 143458892, + "serverNetworkBytes": 286952942, + "packAndReceiveSeconds": 18.017085467, + "archiveValidationSeconds": 1.003890333, + "archiveUploadSeconds": 69.197414698, + "receiverVerificationSeconds": 3.095867335 + }, + "jointPreparationBarrierSeconds": 92.230612876, + "jointConflictCheckCopyVerifySeconds": 8.024644670, + "allRoutesFinishedAtUnixSeconds": 1791447708.7832112, + "consumerStartedAtUnixSeconds": 1791447712.9259932, + "consumerFinishedAtUnixSeconds": 1791447714.8598275, + "barrierSatisfied": true, + "plannedTransferSeconds": 0.241, + "limitations": [ + "Shared barrier and merge phases are repeated in route records; do not sum them across dependencies.", + "The runtime-local route duration includes waiting for the gateway dependency, not only internal copying.", + "Whole-workspace bytes and uncalibrated network predictions differ from the small planned transfer estimate.", + "The overall run includes recovery and is not an uninterrupted makespan benchmark." + ] +} diff --git a/experiments/montage-micro-hpc-e2/single-lane-peer-probe-fallback-evidence.json b/experiments/montage-micro-hpc-e2/single-lane-peer-probe-fallback-evidence.json new file mode 100644 index 00000000..87429476 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/single-lane-peer-probe-fallback-evidence.json @@ -0,0 +1,36 @@ +{ + "schemaVersion": 1, + "runId": "montage-micro-hpc-e2-real-20261008-single-lane-v1", + "scope": "recovery after AKO-125 and AKO-126; before AKO-127", + "wholeWorkspaceContract": true, + "privatePeerRealValidated": false, + "consumers": [ + { + "activity": "mDiffFit6", + "privateProbeFailedSeconds": [2.287067501, 2.290368959], + "strategies": ["gateway", "gateway"], + "serverNetworkBytes": [285910984, 290209742], + "archiveUploadSeconds": [139.627520438, 139.690869231], + "sharedPreparationBarrierSeconds": 162.541874742, + "sharedMergeAndVerificationSeconds": 8.674259296 + }, + { + "activity": "mDiffFit44", + "privateProbeFailedSeconds": [2.279054627], + "strategies": ["gateway", "gateway"], + "serverNetworkBytes": [103997679, 289012486], + "archiveUploadSeconds": [56.485266442, 102.646916214], + "sharedPreparationBarrierSeconds": 125.626070224, + "sharedMergeAndVerificationSeconds": 7.132608837 + } + ], + "interpretation": { + "confirmedBottleneck": "gateway archive upload remains dominant after private authentication probes fail", + "barrierAccounting": "each shared barrier and merge is one joint operation, not a sum across predecessor records", + "credentialDefect": "OpenSSH expiry-time rejects the T-separated timestamp; isolated real sshd accepts the corrected continuous UTC timestamp, enforces the forced command and rejects expired authorization", + "identityCompatibilityDefect": "the gateway container client reads PKCS8 Ed25519 but the host OpenSSH client rejects it; canonical OpenSSH private-key serialization is used instead and verified against the matching public key", + "productionCausalityLimitation": "production probe stderr was not persisted; the credential defect is confirmed but cannot yet be asserted as the sole cause of those production probes", + "response": "cancel recovery, preserve reusable completed workspaces, stop idle VMs and correct/test before recovery", + "comparisonLimitation": "recovery and simultaneous transfers contaminate makespan and gateway throughput; this is not an uninterrupted benchmark" + } +} diff --git a/experiments/montage-micro-hpc-e2/single-lane-post-peer-stage-evidence.json b/experiments/montage-micro-hpc-e2/single-lane-post-peer-stage-evidence.json new file mode 100644 index 00000000..fc9dce7f --- /dev/null +++ b/experiments/montage-micro-hpc-e2/single-lane-post-peer-stage-evidence.json @@ -0,0 +1,20 @@ +{ + "issue": "AKO-128", + "runId": "montage-micro-hpc-e2-real-20261008-single-lane-v1", + "observedAt": "2026-10-08T09:37:45Z", + "source": "Execution-runs API latest activity attempts and original real request plan assignments", + "activities": [ + {"activitySuffix": "mdifffitid0000005", "attempt": 3, "status": "completed", "plannedComputeSeconds": 120, "apiRuntimeSeconds": 31.22002100944519, "plannedTransferSeconds": 0.4425, "transferElapsedSeconds": 86.53249669075012, "overlappingTransferWorkSeconds": 173.064523935318, "postTransferLaunchWaitSeconds": 196.33128261566162}, + {"activitySuffix": "mdifffitid0000006", "attempt": 3, "status": "completed", "plannedComputeSeconds": 112, "apiRuntimeSeconds": 22.749738931655884, "plannedTransferSeconds": 0.4425, "transferElapsedSeconds": 32.084054946899414, "overlappingTransferWorkSeconds": 64.16807198524475, "postTransferLaunchWaitSeconds": 13.554718732833862}, + {"activitySuffix": "mdifffitid0000026", "attempt": 3, "status": "completed", "plannedComputeSeconds": 64, "apiRuntimeSeconds": 7.965739727020264, "plannedTransferSeconds": 0.402, "transferElapsedSeconds": 92.44837427139282, "overlappingTransferWorkSeconds": 184.89670491218567, "postTransferLaunchWaitSeconds": 9.323382139205933} + ], + "finding": "Private uploads are approximately four seconds; whole-workspace receiver verification/merge and the variable post-transfer launch interval remain substantially larger than estimated edge transfer time. The 196-second launch interval is not network upload time.", + "nextMeasurement": "Separate remote directory preparation, full workspace snapshot, Docker launch (which can include image pull), and initial telemetry. Instrumentation is committed locally but not deployed into the healthy running daemon.", + "limitations": [ + "API runtime is an observed task interval; these entries do not independently prove pure application compute duration.", + "Transfer work sums overlapping dependency records and must not be interpreted as elapsed preparation time.", + "Reported queue time includes waits before launch and is not evidence of Slurm queue duration for these cloud tasks.", + "Recovered execution, potentially warm workspaces/images, approximate micro speedup and uncalibrated prices preclude claiming an uninterrupted calibrated benchmark.", + "Estimated plan edge volume and real whole-workspace bytes are different contracts. No scope or integrity reduction was made." + ] +} diff --git a/experiments/montage-micro-hpc-e2/single-lane-private-peer-real-evidence.json b/experiments/montage-micro-hpc-e2/single-lane-private-peer-real-evidence.json new file mode 100644 index 00000000..89b0e486 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/single-lane-private-peer-real-evidence.json @@ -0,0 +1,32 @@ +{ + "issue": "AKO-127", + "runId": "montage-micro-hpc-e2-real-20261008-single-lane-v1", + "consumerActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000006", + "consumerAttempt": 3, + "consumerStatus": "completed", + "consumerExitCode": 0, + "source": "Daemon execution-runs API, observed 2026-10-08", + "contract": "Whole-workspace synchronization with authenticated private peer archive transfer, archive integrity/member validation and joint content-conflict merge barrier", + "dependencies": [ + {"index": 0, "status": "completed", "serverPayloadBytes": 0, "compressedArchiveBytes": 142937977, "privateUploadSeconds": 3.824271002}, + {"index": 1, "status": "completed", "serverPayloadBytes": 0, "compressedArchiveBytes": 145087096, "privateUploadSeconds": 4.433083919} + ], + "jointPreparationSeconds": 24.184386094, + "jointMergeVerificationSeconds": 7.900701004, + "transferFinishedAtUnixSeconds": 1791451595.735822, + "consumerStartedAtUnixSeconds": 1791451609.2905407, + "consumerFinishedAtUnixSeconds": 1791451632.0402796, + "executionBarrierVerified": true, + "previousGatewayAttempt": { + "uploadSeconds": [139.627520438, 139.690869231], + "jointPreparationSeconds": 162.541874742, + "jointMergeVerificationSeconds": 8.674259296 + }, + "limitations": [ + "Recovered execution; destination files may reuse valid materialization from an earlier preparation. This is not a controlled cold-workspace benchmark.", + "Joint preparation and merge durations are shared barriers and must each be counted once, not once per dependency.", + "Server payload bytes exclude control-plane SSH traffic. Archive bytes still traverse the cloud private network.", + "Consumer timestamps measure the observed task interval, not independently verified native Docker compute duration.", + "This validates a transfer path and consumer barrier, not completion of the entire 58-activity campaign." + ] +} diff --git a/experiments/montage-micro-hpc-e2/single-lane-recovery-provision-evidence.json b/experiments/montage-micro-hpc-e2/single-lane-recovery-provision-evidence.json new file mode 100644 index 00000000..04c4d752 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/single-lane-recovery-provision-evidence.json @@ -0,0 +1,21 @@ +{ + "runId": "montage-micro-hpc-e2-real-20261008-single-lane-v1", + "status": "partial_running_measurement", + "source": "Daemon API operation timestamps and allowlisted provisioning log milestones", + "resourceId": "montage-gcp-fog-e2-micro-02", + "generationContext": "Replacement generation during recovery; not the initial provisioning observation", + "provisionElapsedSeconds": 1135.695667, + "sshReadinessSeconds": 7.869, + "configurationPlaybookSeconds": 1076.721, + "dockerValidationSeconds": 2.750, + "workspaceValidationSeconds": 1.614, + "configurationValidated": true, + "inference": "The configuration playbook accounts for approximately 95 percent of this provisioning interval. This delay is infrastructure preparation, not workspace transfer or scientific compute.", + "diagnosticIssue": "AKO-124", + "limitations": [ + "Per-task playbook durations are unavailable; do not attribute all configuration time to Docker installation.", + "Live-output telemetry is committed locally but was not applied to this operation.", + "A recovered run cannot be presented as an uninterrupted makespan benchmark.", + "This is one generation observation, not a final workflow performance result." + ] +} diff --git a/experiments/montage-micro-hpc-e2/single-lane-snapshot-reuse-real-evidence.json b/experiments/montage-micro-hpc-e2/single-lane-snapshot-reuse-real-evidence.json new file mode 100644 index 00000000..e9966a34 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/single-lane-snapshot-reuse-real-evidence.json @@ -0,0 +1,35 @@ +{ + "issues": ["AKO-128", "AKO-129"], + "runId": "montage-micro-hpc-e2-real-20261008-single-lane-v1", + "consumerActivity": "mdifffitid0000024", + "consumerAttempt": 2, + "consumerStatus": "completed", + "transferStatus": "completed", + "snapshotReused": true, + "serverNetworkBytes": 0, + "transferFinishedAt": 1791455261.1301634, + "consumerStartedAt": 1791455265.9831588, + "consumerFinishedAt": 1791455268.3057332, + "transferPhasesSeconds": { + "peer-authenticated-connectivity-probe": 2.463686126, + "peer-snapshot-acquire": 9.799630963, + "peer-reused-source-manifest-check": 1.221616334, + "peer-private-upload": 6.49630642, + "peer-receiver-verify-and-extract": 5.173608419, + "shared-preparation-barrier": 26.942102221, + "shared-conflict-check-copy-and-verify": 9.005693212 + }, + "consumerRemoteStagesSeconds": { + "remoteWorkspacePrepareSeconds": 1.674723126, + "remoteWorkspaceSnapshotSeconds": 1.463208042, + "remoteContainerLaunchSeconds": 1.080457, + "remoteInitialTelemetrySeconds": 1.349214667 + }, + "limitations": [ + "Evidence covers one completed reused snapshot transfer and its consumer, not the full workflow.", + "Snapshot acquisition includes waiting for another reader's build; it is not exclusive compression time.", + "Shared barrier and merge are joint preparation metrics; do not sum duplicated values across dependencies.", + "The run includes recoveries and preserved workspaces; this is not a controlled cold-start benchmark.", + "Server network bytes exclude control traffic and describe payload routing only." + ] +} diff --git a/experiments/montage-micro-hpc-e2/single-lane-validation.json b/experiments/montage-micro-hpc-e2/single-lane-validation.json new file mode 100644 index 00000000..38c53bcf --- /dev/null +++ b/experiments/montage-micro-hpc-e2/single-lane-validation.json @@ -0,0 +1,21 @@ +{ + "issue": "AKO-120", + "implementationCommit": "a2ac846", + "planningSessionId": "montage-micro-hpc-e2-planning-20261008-single-lane-v1", + "planId": "montage-micro-hpc-e2-planning-20261008-single-lane-v1-dd4cb4119a0d", + "activityCount": 58, + "assignmentCounts": {"e2Micro": 40, "e2Medium": 10, "hpc": 8}, + "microSchedulingLanesPerVm": 1, + "plannedMakespanSeconds": 3617.9382322026677, + "simulationRunId": "montage-micro-hpc-e2-simulation-20261008-single-lane-v2", + "simulationStatus": "completed", + "simulationCompletedActivities": 58, + "simulationMakespanSeconds": 3617.4981106317523, + "realValidationStatus": "pending", + "limitations": [ + "Price is not calibrated; predicted zero cost does not mean free cloud execution.", + "Micro computeSpeedup 0.25 is an uncalibrated approximation.", + "Measured e2-medium interference is extrapolated to micro and HPC resources.", + "Earlier two-lane micro plans are not valid baselines for the single-activity-per-micro contract." + ] +} diff --git a/experiments/montage-micro-hpc-e2/speedup-fallback-20261011.json b/experiments/montage-micro-hpc-e2/speedup-fallback-20261011.json new file mode 100644 index 00000000..844563df --- /dev/null +++ b/experiments/montage-micro-hpc-e2/speedup-fallback-20261011.json @@ -0,0 +1,26 @@ +{ + "issue": "AKO-160", + "mode": "offline-simulation-only", + "realExecutionSubmitted": false, + "computeFallback": { + "formula": "donorSecondsPerWorkUnit * sourceSpeedup / targetSpeedup", + "donorSelection": "measured work-unit profile with greatest sample count; stable profile ID tie-break", + "measuredProfilesPreserved": true, + "derivedProfilesAdded": 16, + "derivedSampleSize": 0, + "status": "extrapolated-not-measured" + }, + "hpcStorageCap": "explicitly-removed-by-user-AKO-159", + "selectedPlacement": {"hpcActivities": 58, "cloudActivities": 0, "hpcNodesUsed": 1}, + "predictedMakespanSeconds": 731.4037441142812, + "simulatedMakespanSeconds": 731.4037441142812, + "simulatedActivities": 58, + "interpretation": "The same-node HPC candidate is now eligible. PRISM and SimGrid consume identical completed compute profiles and the unchanged transfer/preparation model. Agreement is not independent validation.", + "limitations": [ + "Speedups are approximate; estimated profiles are not HPC measurements.", + "Same-node placement avoids inter-machine network traffic, not workspace copy/merge.", + "HPC merge profile exists; explicit HPC input-observation and publication profiles are absent in this input.", + "Future Slurm queue waits are not established by this result.", + "Runtime compatibility, CPU/memory feasibility and route evidence remain required. No firewall/storage trust is relaxed." + ] +} diff --git a/experiments/montage-micro-hpc-e2/storage-quota-preflight-20261011.json b/experiments/montage-micro-hpc-e2/storage-quota-preflight-20261011.json new file mode 100644 index 00000000..8e4024a3 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/storage-quota-preflight-20261011.json @@ -0,0 +1,21 @@ +{ + "schemaVersion": 1, + "issue": "AKO-144", + "observedDateUTC": "2026-10-11", + "observationSource": "live-posix-filesystem-and-user-quota-v1", + "effectiveAvailableBytes": 184320, + "reservationModel": "conservative-retain-workspaces-and-input-staging-until-run-end", + "reservedUpperBoundBytes": 19710502133, + "reservationIsExactPeakPrediction": false, + "preflightResponseHTTPStatus": 422, + "preflightCreatedRun": false, + "activeScientificRunsAfterPreflight": 0, + "completedIndependentHoldoutActivitiesBeforeQuotaFailure": 25, + "independentHoldoutCompleted": false, + "preservedWorkspacesRemoved": false, + "limitations": [ + "The observer covers the mounted filesystem and current user quota; additional group/project quota policies are not yet observed.", + "Reservation is a conservative bound, not a modeled time-dependent occupancy peak with verified early releases.", + "Legacy resources without an explicit budget remain compatible and must not be presented as quota-validated." + ] +} diff --git a/experiments/montage-micro-hpc-e2/transfer-model-simulation-validation-20261011.json b/experiments/montage-micro-hpc-e2/transfer-model-simulation-validation-20261011.json new file mode 100644 index 00000000..27f4f640 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/transfer-model-simulation-validation-20261011.json @@ -0,0 +1,3245 @@ +{ + "schemaVersion": 1, + "issue": "AKO-151", + "realExecutionStarted": false, + "daemonRestarted": false, + "validationMode": "native SimGrid offline; exact production export; fixed frozen observed placement and compute", + "previousForecastFilesSHA256": { + "montage-clean-simulation-20261011-v1-request.json": "f094195c7d3ba6f5a7fddde0c5e6a94097603c2b390bd3f48d5f773fc434001c", + "selected-plan.json": "88202b434c5b2ab7da139de1a2ae97d1faee3e6e9e0ac871832fd99f98bc8c0a", + "simulation-results.json": "3d69d698523bc6de4ee98152449a47cf57fc667ccaa9d1f7e472a7aec58456a3" + }, + "cases": [ + { + "schemaVersion": 1, + "comparisonRole": "original-frozen-forecast", + "observedRunId": "montage-clean-real-20261011-v1", + "modelSource": "montage-docker-image-real-20261010-v2:sha256:833f3e2da6cbbe87baf8e2e5b6db3a1e875604c3fe1612e2bf75cd49ae6f8318", + "simulatedMakespanSeconds": 1203.8392300266385, + "observedMakespanSeconds": 3150.6341450214386, + "makespanErrorPercent": -61.79057375071878, + "withinInitialMakespanTolerance": false, + "predictedNetworkBytes": 4925802756, + "observedNetworkBytes": 4399426793, + "networkBytesErrorPercent": 11.964648754640614, + "routeMismatchCount": 0, + "dominantTransfers": [ + { + "producer": "montage-independent-work-v1-mdifffitid0000025", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 242510991, + "predictedNetworkBytes": 276780410, + "observedDependencyPhaseSeconds": 458.625032877, + "predictedDependencySeconds": 75.46213107697815, + "durationErrorPercent": -83.54600694087787, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000028", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 230333020, + "predictedNetworkBytes": 262115992, + "observedDependencyPhaseSeconds": 457.641658918, + "predictedDependencySeconds": 73.10218777470078, + "durationErrorPercent": -84.02632576161534, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000027", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 220934982, + "predictedNetworkBytes": 251628190, + "observedDependencyPhaseSeconds": 439.837226243, + "predictedDependencySeconds": 70.32254016207133, + "durationErrorPercent": -84.0116897874352, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mconcatfitid0000049", + "consumer": "montage-independent-work-v1-mbgmodelid0000050", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 555568726, + "predictedNetworkBytes": 622425076, + "observedDependencyPhaseSeconds": 314.390931602, + "predictedDependencySeconds": 122.0913303430417, + "durationErrorPercent": -61.16575954627026, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-maddid0000056", + "consumer": "montage-independent-work-v1-mviewerid0000058", + "observedStrategy": "direct-runtime", + "executorStrategy": "direct-runtime", + "predictedStrategy": "direct-runtime", + "observedNetworkBytes": 619673320, + "predictedNetworkBytes": 641398880, + "observedDependencyPhaseSeconds": 302.049842513, + "predictedDependencySeconds": 145.0865769752395, + "durationErrorPercent": -51.96601469209669, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mconcatfitid0000011", + "consumer": "montage-independent-work-v1-mbgmodelid0000012", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 559032552, + "predictedNetworkBytes": 624143308, + "observedDependencyPhaseSeconds": 296.56120751, + "predictedDependencySeconds": 122.42477741296159, + "durationErrorPercent": -58.71854635308853, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000024", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 240230880, + "predictedNetworkBytes": 276787022, + "observedDependencyPhaseSeconds": 172.51386337099999, + "predictedDependencySeconds": 75.46308819650238, + "durationErrorPercent": -56.256797730965474, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000029", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 232804610, + "predictedNetworkBytes": 261977210, + "observedDependencyPhaseSeconds": 170.45803487, + "predictedDependencySeconds": 73.07476693457124, + "durationErrorPercent": -57.130347659878986, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000026", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 218669206, + "predictedNetworkBytes": 251595146, + "observedDependencyPhaseSeconds": 169.27667491, + "predictedDependencySeconds": 70.3117093349369, + "durationErrorPercent": -58.46343899871628, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000042", + "consumer": "montage-independent-work-v1-mbackgroundid0000054", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 108668676, + "predictedNetworkBytes": 121637318, + "observedDependencyPhaseSeconds": 117.85378738499999, + "predictedDependencySeconds": 27.965600578986823, + "durationErrorPercent": -76.27093604753665, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000020", + "consumer": "montage-independent-work-v1-mbackgroundid0000032", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 102529126, + "predictedNetworkBytes": 120553514, + "observedDependencyPhaseSeconds": 102.231519922, + "predictedDependencySeconds": 27.725130147655875, + "durationErrorPercent": -72.88005678795597, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000001", + "consumer": "montage-independent-work-v1-mbackgroundid0000013", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 105617907, + "predictedNetworkBytes": 122060266, + "observedDependencyPhaseSeconds": 98.08221692, + "predictedDependencySeconds": 28.057277005251763, + "durationErrorPercent": -71.3941243516789, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000023", + "consumer": "montage-independent-work-v1-mbackgroundid0000035", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 106193023, + "predictedNetworkBytes": 120553514, + "observedDependencyPhaseSeconds": 92.910540916, + "predictedDependencySeconds": 27.725130147655875, + "durationErrorPercent": -70.15932759155709, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000002", + "consumer": "montage-independent-work-v1-mbackgroundid0000014", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 107170830, + "predictedNetworkBytes": 122014006, + "observedDependencyPhaseSeconds": 89.978375582, + "predictedDependencySeconds": 28.048585295110684, + "durationErrorPercent": -68.82741534987018, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000039", + "consumer": "montage-independent-work-v1-mbackgroundid0000051", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 103997683, + "predictedNetworkBytes": 121637318, + "observedDependencyPhaseSeconds": 83.832037956, + "predictedDependencySeconds": 27.965600578986823, + "durationErrorPercent": -66.64091526241457, + "predictedServiceWaitSeconds": 0.0 + } + ], + "accepted": false, + "limitations": [ + "Makespan tolerance alone is insufficient; dominant transfer residuals must also be reviewed.", + "Historical phase wall times may include unseparated queues; no additional bulk slot queue is fitted from these samples.", + "No summed parallel phase durations are treated as makespan.", + "Compute, ordering and provisioning assumptions retained from the frozen original request.", + "Observed comparison run was not used to fit these transfer coefficients; this remains a retrospective test, not a new prospective real validation." + ] + }, + { + "schemaVersion": 1, + "comparisonRole": "historical-calibration-completed-run-holdout", + "observedRunId": "montage-clean-real-20261011-v1", + "modelSource": "montage-docker-image-real-20261010-v2:sha256:f54d5050efe2dcd74329b77c41dc7bcf1a3d533e89ee91d351ac77120a0f4392", + "simulatedMakespanSeconds": 1483.9818720184553, + "observedMakespanSeconds": 3150.6341450214386, + "makespanErrorPercent": -52.89894657037822, + "withinInitialMakespanTolerance": false, + "predictedNetworkBytes": 4925802756, + "observedNetworkBytes": 4399426793, + "networkBytesErrorPercent": 11.964648754640614, + "routeMismatchCount": 0, + "dominantTransfers": [ + { + "producer": "montage-independent-work-v1-mdifffitid0000025", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 242510991, + "predictedNetworkBytes": 276780410, + "observedDependencyPhaseSeconds": 458.625032877, + "predictedDependencySeconds": 97.56337293918989, + "durationErrorPercent": -78.72698480343185, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000028", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 230333020, + "predictedNetworkBytes": 262115992, + "observedDependencyPhaseSeconds": 457.641658918, + "predictedDependencySeconds": 93.72191713519663, + "durationErrorPercent": -79.52067620837165, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000027", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 220934982, + "predictedNetworkBytes": 251628190, + "observedDependencyPhaseSeconds": 439.837226243, + "predictedDependencySeconds": 90.4845919760927, + "durationErrorPercent": -79.42770948493977, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mconcatfitid0000049", + "consumer": "montage-independent-work-v1-mbgmodelid0000050", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 555568726, + "predictedNetworkBytes": 622425076, + "observedDependencyPhaseSeconds": 314.390931602, + "predictedDependencySeconds": 174.19739334300954, + "durationErrorPercent": -44.59210624957436, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-maddid0000056", + "consumer": "montage-independent-work-v1-mviewerid0000058", + "observedStrategy": "direct-runtime", + "executorStrategy": "direct-runtime", + "predictedStrategy": "direct-runtime", + "observedNetworkBytes": 619673320, + "predictedNetworkBytes": 641398880, + "observedDependencyPhaseSeconds": 302.049842513, + "predictedDependencySeconds": 385.8238161543371, + "durationErrorPercent": 27.735148922559553, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mconcatfitid0000011", + "consumer": "montage-independent-work-v1-mbgmodelid0000012", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 559032552, + "predictedNetworkBytes": 624143308, + "observedDependencyPhaseSeconds": 296.56120751, + "predictedDependencySeconds": 174.67468154617995, + "durationErrorPercent": -41.099956055348216, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000024", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 240230880, + "predictedNetworkBytes": 276787022, + "observedDependencyPhaseSeconds": 172.51386337099999, + "predictedDependencySeconds": 97.56498389980356, + "durationErrorPercent": -43.445134209309884, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000029", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 232804610, + "predictedNetworkBytes": 261977210, + "observedDependencyPhaseSeconds": 170.45803487, + "predictedDependencySeconds": 93.68336655128815, + "durationErrorPercent": -45.04021671801165, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000026", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 218669206, + "predictedNetworkBytes": 251595146, + "observedDependencyPhaseSeconds": 169.27667491, + "predictedDependencySeconds": 90.47235703991237, + "durationErrorPercent": -46.55355967500297, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000042", + "consumer": "montage-independent-work-v1-mbackgroundid0000054", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 108668676, + "predictedNetworkBytes": 121637318, + "observedDependencyPhaseSeconds": 117.85378738499999, + "predictedDependencySeconds": 36.64008095338909, + "durationErrorPercent": -68.91056132655733, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000020", + "consumer": "montage-independent-work-v1-mbackgroundid0000032", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 102529126, + "predictedNetworkBytes": 120553514, + "observedDependencyPhaseSeconds": 102.231519922, + "predictedDependencySeconds": 36.32354205254501, + "durationErrorPercent": -64.46933188486396, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000001", + "consumer": "montage-independent-work-v1-mbackgroundid0000013", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 105617907, + "predictedNetworkBytes": 122060266, + "observedDependencyPhaseSeconds": 98.08221692, + "predictedDependencySeconds": 36.76237220978078, + "durationErrorPercent": -62.518820063207045, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000023", + "consumer": "montage-independent-work-v1-mbackgroundid0000035", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 106193023, + "predictedNetworkBytes": 120553514, + "observedDependencyPhaseSeconds": 92.910540916, + "predictedDependencySeconds": 36.32354205254501, + "durationErrorPercent": -60.90482124586385, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000002", + "consumer": "montage-independent-work-v1-mbackgroundid0000014", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 107170830, + "predictedNetworkBytes": 122014006, + "observedDependencyPhaseSeconds": 89.978375582, + "predictedDependencySeconds": 36.74952217157488, + "durationErrorPercent": -59.15738427831036, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000039", + "consumer": "montage-independent-work-v1-mbackgroundid0000051", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 103997683, + "predictedNetworkBytes": 121637318, + "observedDependencyPhaseSeconds": 83.832037956, + "predictedDependencySeconds": 36.64008095338909, + "durationErrorPercent": -56.2934626823459, + "predictedServiceWaitSeconds": 0.0 + } + ], + "accepted": false, + "limitations": [ + "Makespan tolerance alone is insufficient; dominant transfer residuals must also be reviewed.", + "Historical phase wall times may include unseparated queues; no additional bulk slot queue is fitted from these samples.", + "No summed parallel phase durations are treated as makespan.", + "Compute, ordering and provisioning assumptions retained from the frozen original request.", + "Observed comparison run was not used to fit these transfer coefficients; this remains a retrospective test, not a new prospective real validation." + ] + }, + { + "schemaVersion": 1, + "comparisonRole": "calibration-fit-not-independent-validation", + "observedRunId": "montage-clean-real-20261011-v1", + "modelSource": "montage-clean-real-20261011-v1:sha256:f9d8a572c3ef77242bd9b10587f65eab6d75d06c838ab27f34ba101bb0489a7d", + "simulatedMakespanSeconds": 1737.7270017743192, + "observedMakespanSeconds": 3150.6341450214386, + "makespanErrorPercent": -44.84516697947185, + "withinInitialMakespanTolerance": false, + "predictedNetworkBytes": 4383304054, + "observedNetworkBytes": 4399426793, + "networkBytesErrorPercent": -0.36647362846571635, + "routeMismatchCount": 0, + "dominantTransfers": [ + { + "producer": "montage-independent-work-v1-mdifffitid0000025", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 242510991, + "predictedNetworkBytes": 243134592, + "observedDependencyPhaseSeconds": 458.625032877, + "predictedDependencySeconds": 220.03391805996426, + "durationErrorPercent": -52.023133870458445, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000028", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 230333020, + "predictedNetworkBytes": 230252800, + "observedDependencyPhaseSeconds": 457.641658918, + "predictedDependencySeconds": 211.05106020173486, + "durationErrorPercent": -53.88290028038053, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000027", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 220934982, + "predictedNetworkBytes": 221039910, + "observedDependencyPhaseSeconds": 439.837226243, + "predictedDependencySeconds": 200.51739745926903, + "durationErrorPercent": -54.41099900250649, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mconcatfitid0000049", + "consumer": "montage-independent-work-v1-mbgmodelid0000050", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 555568726, + "predictedNetworkBytes": 546762200, + "observedDependencyPhaseSeconds": 314.390931602, + "predictedDependencySeconds": 487.8831880165262, + "durationErrorPercent": 55.18360708767419, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-maddid0000056", + "consumer": "montage-independent-work-v1-mviewerid0000058", + "observedStrategy": "direct-runtime", + "executorStrategy": "direct-runtime", + "predictedStrategy": "direct-runtime", + "observedNetworkBytes": 619673320, + "predictedNetworkBytes": 619718380, + "observedDependencyPhaseSeconds": 302.049842513, + "predictedDependencySeconds": 302.08458283221194, + "durationErrorPercent": 0.011501518730457027, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mconcatfitid0000011", + "consumer": "montage-independent-work-v1-mbgmodelid0000012", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 559032552, + "predictedNetworkBytes": 548271562, + "observedDependencyPhaseSeconds": 296.56120751, + "predictedDependencySeconds": 489.2264203421446, + "durationErrorPercent": 64.9664244524119, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000024", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 240230880, + "predictedNetworkBytes": 243140400, + "observedDependencyPhaseSeconds": 172.51386337099999, + "predictedDependencySeconds": 220.037923618119, + "durationErrorPercent": 27.547965895886335, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000029", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 232804610, + "predictedNetworkBytes": 230130890, + "observedDependencyPhaseSeconds": 170.45803487, + "predictedDependencySeconds": 210.9406517999475, + "durationErrorPercent": 23.74931575435654, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000026", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 218669206, + "predictedNetworkBytes": 221010882, + "observedDependencyPhaseSeconds": 169.27667491, + "predictedDependencySeconds": 200.4834380431082, + "durationErrorPercent": 18.435359242317368, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000042", + "consumer": "montage-independent-work-v1-mbackgroundid0000054", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 108668676, + "predictedNetworkBytes": 106850914, + "observedDependencyPhaseSeconds": 117.85378738499999, + "predictedDependencySeconds": 100.35879799425697, + "durationErrorPercent": -14.844656059793047, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000020", + "consumer": "montage-independent-work-v1-mbackgroundid0000032", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 102529126, + "predictedNetworkBytes": 105898858, + "observedDependencyPhaseSeconds": 102.231519922, + "predictedDependencySeconds": 99.47236424922312, + "durationErrorPercent": -2.698928544623076, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000001", + "consumer": "montage-independent-work-v1-mbackgroundid0000013", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 105617907, + "predictedNetworkBytes": 107222448, + "observedDependencyPhaseSeconds": 98.08221692, + "predictedDependencySeconds": 100.70798261212713, + "durationErrorPercent": 2.6771067932414474, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000023", + "consumer": "montage-independent-work-v1-mbackgroundid0000035", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 106193023, + "predictedNetworkBytes": 105898858, + "observedDependencyPhaseSeconds": 92.910540916, + "predictedDependencySeconds": 99.47236424922312, + "durationErrorPercent": 7.062517630971099, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000002", + "consumer": "montage-independent-work-v1-mbackgroundid0000014", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 107170830, + "predictedNetworkBytes": 107181812, + "observedDependencyPhaseSeconds": 89.978375582, + "predictedDependencySeconds": 100.6799575183311, + "durationErrorPercent": 11.893504263786605, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000039", + "consumer": "montage-independent-work-v1-mbackgroundid0000051", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 103997683, + "predictedNetworkBytes": 106850914, + "observedDependencyPhaseSeconds": 83.832037956, + "predictedDependencySeconds": 100.35879799425697, + "durationErrorPercent": 19.714133690667524, + "predictedServiceWaitSeconds": 0.0 + } + ], + "accepted": false, + "limitations": [ + "Makespan tolerance alone is insufficient; dominant transfer residuals must also be reviewed.", + "Historical phase wall times may include unseparated queues; no additional bulk slot queue is fitted from these samples.", + "No summed parallel phase durations are treated as makespan.", + "Compute, ordering and provisioning assumptions retained from the frozen original request.", + "This comparison reuses calibration data and is not independent validation." + ] + }, + { + "schemaVersion": 1, + "comparisonRole": "calibration-fit-not-independent-validation", + "observedRunId": "montage-clean-real-20261011-v1", + "modelSource": "montage-clean-real-20261011-v1:sha256:0ab9d1bd45fbce2ed8b847b07f9e190f34be31709df3c48bfd7f93102426ded3", + "simulatedMakespanSeconds": 2835.6021967566785, + "observedMakespanSeconds": 3150.6341450214386, + "makespanErrorPercent": -9.999001272888714, + "withinInitialMakespanTolerance": true, + "predictedNetworkBytes": 4383304054, + "observedNetworkBytes": 4399426793, + "networkBytesErrorPercent": -0.36647362846571635, + "routeMismatchCount": 0, + "dominantTransfers": [ + { + "producer": "montage-independent-work-v1-mdifffitid0000025", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 242510991, + "predictedNetworkBytes": 243134592, + "observedDependencyPhaseSeconds": 458.625032877, + "predictedDependencySeconds": 364.8038058380528, + "durationErrorPercent": -20.457066298888527, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000028", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 230333020, + "predictedNetworkBytes": 230252800, + "observedDependencyPhaseSeconds": 457.641658918, + "predictedDependencySeconds": 681.5487379458395, + "durationErrorPercent": 48.92628865064906, + "predictedServiceWaitSeconds": 348.94570559301445 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000027", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 220934982, + "predictedNetworkBytes": 221039910, + "observedDependencyPhaseSeconds": 439.837226243, + "predictedDependencySeconds": 332.9376934434027, + "durationErrorPercent": -24.304339519578942, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mconcatfitid0000049", + "consumer": "montage-independent-work-v1-mbgmodelid0000050", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 555568726, + "predictedNetworkBytes": 546762200, + "observedDependencyPhaseSeconds": 314.390931602, + "predictedDependencySeconds": 757.8083899092965, + "durationErrorPercent": 141.04015534030614, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-maddid0000056", + "consumer": "montage-independent-work-v1-mviewerid0000058", + "observedStrategy": "direct-runtime", + "executorStrategy": "direct-runtime", + "predictedStrategy": "direct-runtime", + "observedNetworkBytes": 619673320, + "predictedNetworkBytes": 619718380, + "observedDependencyPhaseSeconds": 302.049842513, + "predictedDependencySeconds": 302.08458283221216, + "durationErrorPercent": 0.011501518730532304, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mconcatfitid0000011", + "consumer": "montage-independent-work-v1-mbgmodelid0000012", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 559032552, + "predictedNetworkBytes": 548271562, + "observedDependencyPhaseSeconds": 296.56120751, + "predictedDependencySeconds": 758.8805461359234, + "durationErrorPercent": 155.89339634393488, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000024", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 240230880, + "predictedNetworkBytes": 243140400, + "observedDependencyPhaseSeconds": 172.51386337099999, + "predictedDependencySeconds": 364.8097348575456, + "durationErrorPercent": 111.46690922630572, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000029", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 232804610, + "predictedNetworkBytes": 230130890, + "observedDependencyPhaseSeconds": 170.45803487, + "predictedDependencySeconds": 661.8891873144828, + "durationErrorPercent": 288.30037423537897, + "predictedServiceWaitSeconds": 318.46157297674966 + }, + { + "producer": "montage-independent-work-v1-mdifffitid0000026", + "consumer": "montage-independent-work-v1-mconcatfitid0000030", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 218669206, + "predictedNetworkBytes": 221010882, + "observedDependencyPhaseSeconds": 169.27667491, + "predictedDependencySeconds": 674.9876108725307, + "durationErrorPercent": 298.74815076052505, + "predictedServiceWaitSeconds": 348.95127133002956 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000042", + "consumer": "montage-independent-work-v1-mbackgroundid0000054", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 108668676, + "predictedNetworkBytes": 106850914, + "observedDependencyPhaseSeconds": 117.85378738499999, + "predictedDependencySeconds": 316.5188645576445, + "durationErrorPercent": 168.56910717994452, + "predictedServiceWaitSeconds": 154.61636908982086 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000020", + "consumer": "montage-independent-work-v1-mbackgroundid0000032", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 102529126, + "predictedNetworkBytes": 105898858, + "observedDependencyPhaseSeconds": 102.231519922, + "predictedDependencySeconds": 160.52457167759258, + "durationErrorPercent": 57.02062514581477, + "predictedServiceWaitSeconds": 0.0 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000001", + "consumer": "montage-independent-work-v1-mbackgroundid0000013", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 105617907, + "predictedNetworkBytes": 107222448, + "observedDependencyPhaseSeconds": 98.08221692, + "predictedDependencySeconds": 253.8837125207799, + "durationErrorPercent": 158.84785284559607, + "predictedServiceWaitSeconds": 91.59574461526995 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000023", + "consumer": "montage-independent-work-v1-mbackgroundid0000035", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 106193023, + "predictedNetworkBytes": 105898858, + "observedDependencyPhaseSeconds": 92.910540916, + "predictedDependencySeconds": 620.2671474056713, + "durationErrorPercent": 567.5960997433564, + "predictedServiceWaitSeconds": 459.7425757280787 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000002", + "consumer": "montage-independent-work-v1-mbackgroundid0000014", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 107170830, + "predictedNetworkBytes": 107181812, + "observedDependencyPhaseSeconds": 89.978375582, + "predictedDependencySeconds": 253.85484724124376, + "durationErrorPercent": 182.12872881873514, + "predictedServiceWaitSeconds": 91.59574461526995 + }, + { + "producer": "montage-independent-work-v1-mprojectid0000039", + "consumer": "montage-independent-work-v1-mbackgroundid0000051", + "observedStrategy": "gateway", + "executorStrategy": "gateway", + "predictedStrategy": "gateway", + "observedNetworkBytes": 103997683, + "predictedNetworkBytes": 106850914, + "observedDependencyPhaseSeconds": 83.832037956, + "predictedDependencySeconds": 163.61756127540298, + "durationErrorPercent": 95.17306898978066, + "predictedServiceWaitSeconds": 0.0 + } + ], + "accepted": false, + "limitations": [ + "Makespan tolerance alone is insufficient; dominant transfer residuals must also be reviewed.", + "Historical receiver phases may include unseparated queues; destination bulk pools must remain disabled until service is separately identified.", + "No summed parallel phase durations are treated as makespan.", + "Compute, ordering and provisioning assumptions retained from the frozen original request.", + "This comparison reuses calibration data and is not independent validation." + ] + } + ], + "provenance": { + "ako151-clean-v2-calibration": { + "sha256": "f9d8a572c3ef77242bd9b10587f65eab6d75d06c838ab27f34ba101bb0489a7d", + "evidence": { + "schemaVersion": 1, + "baselineRunId": "montage-clean-real-20261011-v1", + "baselineSHA256": "c55d80f3cf734f6486c3937b17a743cf2da6faade447fdd2d15e18e24d9d5e4f", + "computeWorkBasis": null, + "synchronizationContract": "whole-workspace-v1", + "workspaceVolumeBasis": "producer-final-snapshot-manifest", + "computeObservations": [ + { + "activityFamily": "madd", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 3, + "median": 0.9227643013000488, + "minimum": 0.9206876754760742, + "maximum": 8.156338691711426 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 3, + "median": 0.10252936681111653, + "minimum": 0.10229863060845269, + "maximum": 4.078169345855713 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mbackground", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 12, + "median": 1.0978449583053589, + "minimum": 0.7738721370697021, + "maximum": 9.189680099487305 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 12, + "median": 0.3890828953848945, + "minimum": 0.04005753475686778, + "maximum": 1.1718418598175049 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mbgmodel", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 3, + "median": 1.7851626873016357, + "minimum": 0.42560553550720215, + "maximum": 8.379759788513184 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 3, + "median": 0.19835140970018175, + "minimum": 0.060800790786743164, + "maximum": 0.4189879894256592 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "hpc", + "runtimeSeconds": { + "samples": 2, + "median": 2.273337483406067, + "minimum": 2.2540571689605713, + "maximum": 2.2926177978515625 + }, + "taskRuntimeFallbackSamples": 2, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 0.2955575788722319, + "minimum": 0.13259159817415125, + "maximum": 0.4585235595703125 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 1, + "median": 0.5490996837615967, + "minimum": 0.5490996837615967, + "maximum": 0.5490996837615967 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 1, + "median": 0.06101107597351074, + "minimum": 0.06101107597351074, + "maximum": 0.06101107597351074 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mdifffit", + "resourceClass": "hpc", + "runtimeSeconds": { + "samples": 18, + "median": 2.216379404067993, + "minimum": 1.6275734901428223, + "maximum": 3.6220107078552246 + }, + "taskRuntimeFallbackSamples": 18, + "wallSecondsPerReferenceSecond": { + "samples": 18, + "median": 0.12864546605518884, + "minimum": 0.07348955670992534, + "maximum": 0.27002583850513806 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mimgtbl", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 3, + "median": 0.10982584953308105, + "minimum": 0.09377503395080566, + "maximum": 1.9788739681243896 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 3, + "median": 0.0043930339813232425, + "minimum": 0.004262501543218439, + "maximum": 0.2198748853471544 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mproject", + "resourceClass": "hpc", + "runtimeSeconds": { + "samples": 12, + "median": 429.2951464653015, + "minimum": 296.09501814842224, + "maximum": 480.9227578639984 + }, + "taskRuntimeFallbackSamples": 12, + "wallSecondsPerReferenceSecond": { + "samples": 12, + "median": 0.4244157270059439, + "minimum": 0.26437055191823416, + "maximum": 1.1530097625105853 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mviewer", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 4, + "median": 2.4596585035324097, + "minimum": 1.3308792114257812, + "maximum": 22.134554862976074 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 4, + "median": 0.3088648319244385, + "minimum": 0.10351345118354349, + "maximum": 22.134554862976074 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + } + ], + "materializationPhaseObservations": [ + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "pack-and-receive", + "seconds": { + "samples": 20, + "median": 93.477485043, + "minimum": 23.984320886, + "maximum": 443.228494495 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 20, + "bytes": 1889644770, + "serviceSeconds": 3107.49033925, + "bytesPerSecond": 608093.5300529591, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 20, + "bytes": 3734106599, + "serviceSeconds": 3107.49033925, + "bytesPerSecond": 1201647.0499796423, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 8, + "median": 125191004.0, + "minimum": 9175961.0, + "maximum": 543960288.0 + }, + "archiveBytes": { + "samples": 20, + "median": 54575715.5, + "minimum": 51258241.0, + "maximum": 279482093.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 106346880.0, + "minimum": 105027840.0, + "maximum": 543960288.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.5039255042100858, + "minimum": 0.48783029128445954, + "maximum": 0.516872580582045 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 1467064.2688254993, + "minimum": 514336.08762021107, + "maximum": 4379020.798596232 + }, + "files": { + "samples": 8, + "median": 5.0, + "minimum": 3.0, + "maximum": 27.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 743603.5823995736, + "minimum": 259032.26729615868, + "maximum": 2223122.1494006827 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "archive-validation", + "seconds": { + "samples": 20, + "median": 0.5623665840000001, + "minimum": 0.514497417, + "maximum": 2.844069335 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 20, + "bytes": 1889644770, + "serviceSeconds": 19.35844647, + "bytesPerSecond": 97613451.20996168, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 20, + "bytes": 3734106599, + "serviceSeconds": 19.35844647, + "bytesPerSecond": 192892885.53122208, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 8, + "median": 125191004.0, + "minimum": 9175961.0, + "maximum": 543960288.0 + }, + "archiveBytes": { + "samples": 20, + "median": 54575715.5, + "minimum": 51258241.0, + "maximum": 279482093.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 106346880.0, + "minimum": 105027840.0, + "maximum": 543960288.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.5039255042100858, + "minimum": 0.48783029128445954, + "maximum": 0.516872580582045 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 191117125.75011408, + "minimum": 184823674.3538846, + "maximum": 204136768.2901312 + }, + "files": { + "samples": 8, + "median": 5.0, + "minimum": 3.0, + "maximum": 27.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 97510235.9022452, + "minimum": 92780099.73707034, + "maximum": 104536040.47379948 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "archive-upload", + "seconds": { + "samples": 20, + "median": 5.228915211, + "minimum": 4.934585543, + "maximum": 23.754565511 + }, + "secondsBases": [ + "phase-command-execution-excludes-channel-queue" + ], + "pooledArchiveServiceRate": { + "samples": 20, + "bytes": 1889644770, + "serviceSeconds": 186.044580421, + "bytesPerSecond": 10156946.070258675, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 20, + "bytes": 3734106599, + "serviceSeconds": 186.044580421, + "bytesPerSecond": 20071031.31168936, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": { + "samples": 20, + "median": 3.13335e-05, + "minimum": 5.542e-06, + "maximum": 9.775e-05 + }, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 8, + "median": 125191004.0, + "minimum": 9175961.0, + "maximum": 543960288.0 + }, + "archiveBytes": { + "samples": 20, + "median": 54575715.5, + "minimum": 51258241.0, + "maximum": 279482093.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 106346880.0, + "minimum": 105027840.0, + "maximum": 543960288.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.5039255042100858, + "minimum": 0.48783029128445954, + "maximum": 0.516872580582045 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 20885862.053935025, + "minimum": 13516582.554276312, + "maximum": 24103060.74761127 + }, + "files": { + "samples": 8, + "median": 5.0, + "minimum": 3.0, + "maximum": 27.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 10513116.819752198, + "minimum": 6809813.146297475, + "maximum": 12341189.114952732 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "receiver-verify-and-extract", + "seconds": { + "samples": 20, + "median": 1.6460796675, + "minimum": 1.158808168, + "maximum": 7.398465045 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 20, + "bytes": 1889644770, + "serviceSeconds": 50.131346858, + "bytesPerSecond": 37693875.95654532, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 20, + "bytes": 3734106599, + "serviceSeconds": 50.131346858, + "bytesPerSecond": 74486460.7283957, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 8, + "median": 125191004.0, + "minimum": 9175961.0, + "maximum": 543960288.0 + }, + "archiveBytes": { + "samples": 20, + "median": 54575715.5, + "minimum": 51258241.0, + "maximum": 279482093.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 106346880.0, + "minimum": 105027840.0, + "maximum": 543960288.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.5039255042100858, + "minimum": 0.48783029128445954, + "maximum": 0.516872580582045 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 81212680.52997111, + "minimum": 52652765.7823724, + "maximum": 100003423.27378607 + }, + "files": { + "samples": 8, + "median": 5.0, + "minimum": 3.0, + "maximum": 27.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 40677336.46764976, + "minimum": 26838049.28921842, + "maximum": 50364176.44315483 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-authenticated-connectivity-probe", + "seconds": { + "samples": 1, + "median": 3.210240794, + "minimum": 3.210240794, + "maximum": 3.210240794 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 619673320, + "serviceSeconds": 3.210240794, + "bytesPerSecond": 193030168.06657648, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 1108977176, + "serviceSeconds": 3.210240794, + "bytesPerSecond": 345449842.2899301, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 1, + "median": 1018425874.0, + "minimum": 1018425874.0, + "maximum": 1018425874.0 + }, + "archiveBytes": { + "samples": 1, + "median": 619673320.0, + "minimum": 619673320.0, + "maximum": 619673320.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 1108977176.0, + "minimum": 1108977176.0, + "maximum": 1108977176.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5587791465962506, + "minimum": 0.5587791465962506, + "maximum": 0.5587791465962506 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 345449842.2899301, + "minimum": 345449842.2899301, + "maximum": 345449842.2899301 + }, + "files": { + "samples": 1, + "median": 39.0, + "minimum": 39.0, + "maximum": 39.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 193030168.06657648, + "minimum": 193030168.06657648, + "maximum": 193030168.06657648 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-snapshot-acquire", + "seconds": { + "samples": 1, + "median": 250.851621405, + "minimum": 250.851621405, + "maximum": 250.851621405 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 619673320, + "serviceSeconds": 250.851621405, + "bytesPerSecond": 2470278.312451237, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 1108977176, + "serviceSeconds": 250.851621405, + "bytesPerSecond": 4420849.144959506, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 1, + "median": 1018425874.0, + "minimum": 1018425874.0, + "maximum": 1018425874.0 + }, + "archiveBytes": { + "samples": 1, + "median": 619673320.0, + "minimum": 619673320.0, + "maximum": 619673320.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 1108977176.0, + "minimum": 1108977176.0, + "maximum": 1108977176.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5587791465962506, + "minimum": 0.5587791465962506, + "maximum": 0.5587791465962506 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 4420849.144959506, + "minimum": 4420849.144959506, + "maximum": 4420849.144959506 + }, + "files": { + "samples": 1, + "median": 39.0, + "minimum": 39.0, + "maximum": 39.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 2470278.312451237, + "minimum": 2470278.312451237, + "maximum": 2470278.312451237 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-private-upload", + "seconds": { + "samples": 1, + "median": 28.724556513, + "minimum": 28.724556513, + "maximum": 28.724556513 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 619673320, + "serviceSeconds": 28.724556513, + "bytesPerSecond": 21572946.468975134, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 1108977176, + "serviceSeconds": 28.724556513, + "bytesPerSecond": 38607286.26038509, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 1, + "median": 1018425874.0, + "minimum": 1018425874.0, + "maximum": 1018425874.0 + }, + "archiveBytes": { + "samples": 1, + "median": 619673320.0, + "minimum": 619673320.0, + "maximum": 619673320.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 1108977176.0, + "minimum": 1108977176.0, + "maximum": 1108977176.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5587791465962506, + "minimum": 0.5587791465962506, + "maximum": 0.5587791465962506 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 38607286.26038509, + "minimum": 38607286.26038509, + "maximum": 38607286.26038509 + }, + "files": { + "samples": 1, + "median": 39.0, + "minimum": 39.0, + "maximum": 39.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 21572946.468975134, + "minimum": 21572946.468975134, + "maximum": 21572946.468975134 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-receiver-verify-and-extract", + "seconds": { + "samples": 1, + "median": 19.263423801, + "minimum": 19.263423801, + "maximum": 19.263423801 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 619673320, + "serviceSeconds": 19.263423801, + "bytesPerSecond": 32168389.503418997, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 1108977176, + "serviceSeconds": 19.263423801, + "bytesPerSecond": 57569058.722698666, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 1, + "median": 1018425874.0, + "minimum": 1018425874.0, + "maximum": 1018425874.0 + }, + "archiveBytes": { + "samples": 1, + "median": 619673320.0, + "minimum": 619673320.0, + "maximum": 619673320.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 1108977176.0, + "minimum": 1108977176.0, + "maximum": 1108977176.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5587791465962506, + "minimum": 0.5587791465962506, + "maximum": 0.5587791465962506 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 57569058.722698666, + "minimum": 57569058.722698666, + "maximum": 57569058.722698666 + }, + "files": { + "samples": 1, + "median": 39.0, + "minimum": 39.0, + "maximum": 39.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 32168389.503418997, + "minimum": 32168389.503418997, + "maximum": 32168389.503418997 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + } + ], + "consumerMergeObservations": [ + { + "activityFamily": "madd", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 3, + "median": 67.094164362, + "minimum": 66.79656049, + "maximum": 219.086229683 + }, + "checkSeconds": { + "samples": 3, + "median": 25.7431940460001, + "minimum": 25.456530666999925, + "maximum": 121.27468780899994 + }, + "copySeconds": { + "samples": 3, + "median": 34.58182319599996, + "minimum": 33.813945880999654, + "maximum": 62.06325912700004 + }, + "verifySeconds": { + "samples": 3, + "median": 6.73655051500009, + "minimum": 6.551817965000055, + "maximum": 35.192216692999864 + } + }, + { + "activityFamily": "mbackground", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 12, + "median": 18.883627134, + "minimum": 17.8304088, + "maximum": 19.754857759 + }, + "checkSeconds": { + "samples": 12, + "median": 4.557633406999969, + "minimum": 3.6671020270000554, + "maximum": 4.961447119000013 + }, + "copySeconds": { + "samples": 12, + "median": 10.148011529499968, + "minimum": 9.761975659999962, + "maximum": 10.434541308000007 + }, + "verifySeconds": { + "samples": 12, + "median": 3.693423046499902, + "minimum": 2.9794699060000767, + "maximum": 4.000663433 + } + }, + { + "activityFamily": "mbgmodel", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 3, + "median": 15.077407549, + "minimum": 14.266392464, + "maximum": 15.232841298 + }, + "checkSeconds": { + "samples": 3, + "median": 2.53756751800006, + "minimum": 2.5070188539998526, + "maximum": 2.962898128000006 + }, + "copySeconds": { + "samples": 3, + "median": 8.130230359999814, + "minimum": 8.074467681999977, + "maximum": 9.150769504999971 + }, + "verifySeconds": { + "samples": 3, + "median": 3.020684265, + "minimum": 3.0046015890000035, + "maximum": 3.463600305 + } + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "hpc", + "mergeSeconds": { + "samples": 2, + "median": 18.891379676, + "minimum": 17.266011092, + "maximum": 20.51674826 + }, + "checkSeconds": { + "samples": 2, + "median": 6.206628285348415, + "minimum": 5.660585141740739, + "maximum": 6.752671428956091 + }, + "copySeconds": { + "samples": 2, + "median": 6.046090563759208, + "minimum": 5.871328831650317, + "maximum": 6.220852295868099 + }, + "verifySeconds": { + "samples": 2, + "median": 4.663542255293578, + "minimum": 4.222140168771148, + "maximum": 5.104944341816008 + } + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 1, + "median": 24.153126054, + "minimum": 24.153126054, + "maximum": 24.153126054 + }, + "checkSeconds": { + "samples": 1, + "median": 8.702594806999969, + "minimum": 8.702594806999969, + "maximum": 8.702594806999969 + }, + "copySeconds": { + "samples": 1, + "median": 11.903956375999996, + "minimum": 11.903956375999996, + "maximum": 11.903956375999996 + }, + "verifySeconds": { + "samples": 1, + "median": 3.0493237859999454, + "minimum": 3.0493237859999454, + "maximum": 3.0493237859999454 + } + }, + { + "activityFamily": "mdifffit", + "resourceClass": "hpc", + "mergeSeconds": { + "samples": 18, + "median": 10.650899734, + "minimum": 5.614199461, + "maximum": 21.22848251 + }, + "checkSeconds": { + "samples": 18, + "median": 1.4091542228125036, + "minimum": 0.677861861884594, + "maximum": 3.194015149027109 + }, + "copySeconds": { + "samples": 18, + "median": 1.472872169688344, + "minimum": 1.2712753061205149, + "maximum": 1.878188787959516 + }, + "verifySeconds": { + "samples": 18, + "median": 4.185988076031208, + "minimum": 1.6743122031912208, + "maximum": 5.521449169144034 + } + }, + { + "activityFamily": "mimgtbl", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 3, + "median": 53.298720149, + "minimum": 51.926005982, + "maximum": 53.377830025 + }, + "checkSeconds": { + "samples": 3, + "median": 18.728920129000016, + "minimum": 17.38217044000021, + "maximum": 18.955533912999954 + }, + "copySeconds": { + "samples": 3, + "median": 27.39036550999981, + "minimum": 27.380674623000004, + "maximum": 27.65483295599995 + }, + "verifySeconds": { + "samples": 3, + "median": 6.6541218470001695, + "minimum": 6.2337451810000175, + "maximum": 6.684228208999912 + } + }, + { + "activityFamily": "mviewer", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 4, + "median": 60.991183819, + "minimum": 29.459673139, + "maximum": 104.951331755 + }, + "checkSeconds": { + "samples": 4, + "median": 13.8288311230001, + "minimum": 5.738225310999951, + "maximum": 34.67292259200008 + }, + "copySeconds": { + "samples": 4, + "median": 25.007742583499976, + "minimum": 16.94268330900013, + "maximum": 49.24239904699971 + }, + "verifySeconds": { + "samples": 4, + "median": 13.40475499099989, + "minimum": 6.247800998000002, + "maximum": 36.95691540500002 + } + } + ], + "mergeRateObservations": [ + { + "targetClass": "e2-micro", + "source": "baseline:montage-clean-real-20261011-v1", + "observations": { + "checkSecondsPerInputByte": { + "samples": 26, + "median": 6.2190376479285e-09, + "minimum": 4.04158434198331e-09, + "maximum": 3.1265677366835255e-08 + }, + "copySecondsPerUniqueByte": { + "samples": 26, + "median": 1.6090916291863342e-08, + "minimum": 1.4574432595848423e-08, + "maximum": 5.8709123754770746e-08 + }, + "verifySecondsPerUniqueByte": { + "samples": 26, + "median": 5.9409084403851915e-09, + "minimum": 4.745400286178283e-09, + "maximum": 3.3325226348030825e-08 + } + }, + "commandOverheadSeconds": null, + "channelWaitSeconds": null + }, + { + "targetClass": "hpc", + "source": "baseline:montage-clean-real-20261011-v1", + "observations": { + "checkSecondsPerInputByte": { + "samples": 20, + "median": 6.241897995174089e-09, + "minimum": 3.225024006512041e-09, + "maximum": 1.5199312527158336e-08 + }, + "copySecondsPerUniqueByte": { + "samples": 20, + "median": 7.273321197353105e-09, + "minimum": 5.994391983044094e-09, + "maximum": 1.1467828457107587e-08 + }, + "verifySecondsPerUniqueByte": { + "samples": 20, + "median": 1.9239898531383915e-08, + "minimum": 7.761874480480946e-09, + "maximum": 2.5959619920885475e-08 + } + }, + "commandOverheadSeconds": null, + "channelWaitSeconds": null + } + ], + "limitations": [ + "Single baseline; ranges are observations, not confidence intervals.", + "Do not replay per-activity durations or total makespan as a holdout prediction.", + "No destination cache hit or shared-storage route may be assumed without evidence.", + "Workspace manifests and topology-specific contention are required before applying these observations.", + "Compute samples are contention-conditioned; do not apply interference slowdown a second time." + ] + } + }, + "ako151-source-pool-v3-calibration": { + "sha256": "0ab9d1bd45fbce2ed8b847b07f9e190f34be31709df3c48bfd7f93102426ded3", + "evidence": { + "schemaVersion": 1, + "baselineRunId": "montage-clean-real-20261011-v1", + "baselineSHA256": "c55d80f3cf734f6486c3937b17a743cf2da6faade447fdd2d15e18e24d9d5e4f", + "computeWorkBasis": null, + "synchronizationContract": "whole-workspace-v1", + "workspaceVolumeBasis": "producer-final-snapshot-manifest", + "computeObservations": [ + { + "activityFamily": "madd", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 3, + "median": 0.9227643013000488, + "minimum": 0.9206876754760742, + "maximum": 8.156338691711426 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 3, + "median": 0.10252936681111653, + "minimum": 0.10229863060845269, + "maximum": 4.078169345855713 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mbackground", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 12, + "median": 1.0978449583053589, + "minimum": 0.7738721370697021, + "maximum": 9.189680099487305 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 12, + "median": 0.3890828953848945, + "minimum": 0.04005753475686778, + "maximum": 1.1718418598175049 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mbgmodel", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 3, + "median": 1.7851626873016357, + "minimum": 0.42560553550720215, + "maximum": 8.379759788513184 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 3, + "median": 0.19835140970018175, + "minimum": 0.060800790786743164, + "maximum": 0.4189879894256592 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "hpc", + "runtimeSeconds": { + "samples": 2, + "median": 2.273337483406067, + "minimum": 2.2540571689605713, + "maximum": 2.2926177978515625 + }, + "taskRuntimeFallbackSamples": 2, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 0.2955575788722319, + "minimum": 0.13259159817415125, + "maximum": 0.4585235595703125 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 1, + "median": 0.5490996837615967, + "minimum": 0.5490996837615967, + "maximum": 0.5490996837615967 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 1, + "median": 0.06101107597351074, + "minimum": 0.06101107597351074, + "maximum": 0.06101107597351074 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mdifffit", + "resourceClass": "hpc", + "runtimeSeconds": { + "samples": 18, + "median": 2.216379404067993, + "minimum": 1.6275734901428223, + "maximum": 3.6220107078552246 + }, + "taskRuntimeFallbackSamples": 18, + "wallSecondsPerReferenceSecond": { + "samples": 18, + "median": 0.12864546605518884, + "minimum": 0.07348955670992534, + "maximum": 0.27002583850513806 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mimgtbl", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 3, + "median": 0.10982584953308105, + "minimum": 0.09377503395080566, + "maximum": 1.9788739681243896 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 3, + "median": 0.0043930339813232425, + "minimum": 0.004262501543218439, + "maximum": 0.2198748853471544 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mproject", + "resourceClass": "hpc", + "runtimeSeconds": { + "samples": 12, + "median": 429.2951464653015, + "minimum": 296.09501814842224, + "maximum": 480.9227578639984 + }, + "taskRuntimeFallbackSamples": 12, + "wallSecondsPerReferenceSecond": { + "samples": 12, + "median": 0.4244157270059439, + "minimum": 0.26437055191823416, + "maximum": 1.1530097625105853 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mviewer", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 4, + "median": 2.4596585035324097, + "minimum": 1.3308792114257812, + "maximum": 22.134554862976074 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 4, + "median": 0.3088648319244385, + "minimum": 0.10351345118354349, + "maximum": 22.134554862976074 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + } + ], + "materializationPhaseObservations": [ + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "pack-and-receive", + "seconds": { + "samples": 9, + "median": 99.334554129, + "minimum": 15.046990423, + "maximum": 329.915067318 + }, + "secondsBases": [ + "source-command-execution-excludes-channel-queue" + ], + "pooledArchiveServiceRate": { + "samples": 9, + "bytes": 1073826598, + "serviceSeconds": 1380.179567089, + "bytesPerSecond": 778033.9773214125, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 9, + "bytes": 2130542471, + "serviceSeconds": 1380.179567089, + "bytesPerSecond": 1543670.4917271198, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": { + "samples": 9, + "median": 0.652852501, + "minimum": 8.7709e-05, + "maximum": 329.915110567 + }, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": { + "samples": 9, + "median": 0.649784334, + "minimum": 0.643699041, + "maximum": 3.3186203760000126 + }, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 8, + "median": 125191004.0, + "minimum": 9175961.0, + "maximum": 543960288.0 + }, + "archiveBytes": { + "samples": 20, + "median": 54575715.5, + "minimum": 51258241.0, + "maximum": 279482093.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 106346880.0, + "minimum": 105027840.0, + "maximum": 543960288.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.5039255042100858, + "minimum": 0.48783029128445954, + "maximum": 0.516872580582045 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 1467064.2688254993, + "minimum": 514336.08762021107, + "maximum": 4379020.798596232 + }, + "files": { + "samples": 8, + "median": 5.0, + "minimum": 3.0, + "maximum": 27.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 743603.5823995736, + "minimum": 259032.26729615868, + "maximum": 2223122.1494006827 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "archive-validation", + "seconds": { + "samples": 20, + "median": 0.5623665840000001, + "minimum": 0.514497417, + "maximum": 2.844069335 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 20, + "bytes": 1889644770, + "serviceSeconds": 19.35844647, + "bytesPerSecond": 97613451.20996168, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 20, + "bytes": 3734106599, + "serviceSeconds": 19.35844647, + "bytesPerSecond": 192892885.53122208, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 8, + "median": 125191004.0, + "minimum": 9175961.0, + "maximum": 543960288.0 + }, + "archiveBytes": { + "samples": 20, + "median": 54575715.5, + "minimum": 51258241.0, + "maximum": 279482093.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 106346880.0, + "minimum": 105027840.0, + "maximum": 543960288.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.5039255042100858, + "minimum": 0.48783029128445954, + "maximum": 0.516872580582045 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 191117125.75011408, + "minimum": 184823674.3538846, + "maximum": 204136768.2901312 + }, + "files": { + "samples": 8, + "median": 5.0, + "minimum": 3.0, + "maximum": 27.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 97510235.9022452, + "minimum": 92780099.73707034, + "maximum": 104536040.47379948 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "archive-upload", + "seconds": { + "samples": 20, + "median": 5.228915211, + "minimum": 4.934585543, + "maximum": 23.754565511 + }, + "secondsBases": [ + "phase-command-execution-excludes-channel-queue" + ], + "pooledArchiveServiceRate": { + "samples": 20, + "bytes": 1889644770, + "serviceSeconds": 186.044580421, + "bytesPerSecond": 10156946.070258675, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 20, + "bytes": 3734106599, + "serviceSeconds": 186.044580421, + "bytesPerSecond": 20071031.31168936, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": { + "samples": 20, + "median": 3.13335e-05, + "minimum": 5.542e-06, + "maximum": 9.775e-05 + }, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 8, + "median": 125191004.0, + "minimum": 9175961.0, + "maximum": 543960288.0 + }, + "archiveBytes": { + "samples": 20, + "median": 54575715.5, + "minimum": 51258241.0, + "maximum": 279482093.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 106346880.0, + "minimum": 105027840.0, + "maximum": 543960288.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.5039255042100858, + "minimum": 0.48783029128445954, + "maximum": 0.516872580582045 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 20885862.053935025, + "minimum": 13516582.554276312, + "maximum": 24103060.74761127 + }, + "files": { + "samples": 8, + "median": 5.0, + "minimum": 3.0, + "maximum": 27.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 10513116.819752198, + "minimum": 6809813.146297475, + "maximum": 12341189.114952732 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "receiver-verify-and-extract", + "seconds": { + "samples": 20, + "median": 1.6460796675, + "minimum": 1.158808168, + "maximum": 7.398465045 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 20, + "bytes": 1889644770, + "serviceSeconds": 50.131346858, + "bytesPerSecond": 37693875.95654532, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 20, + "bytes": 3734106599, + "serviceSeconds": 50.131346858, + "bytesPerSecond": 74486460.7283957, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 8, + "median": 125191004.0, + "minimum": 9175961.0, + "maximum": 543960288.0 + }, + "archiveBytes": { + "samples": 20, + "median": 54575715.5, + "minimum": 51258241.0, + "maximum": 279482093.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 106346880.0, + "minimum": 105027840.0, + "maximum": 543960288.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.5039255042100858, + "minimum": 0.48783029128445954, + "maximum": 0.516872580582045 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 81212680.52997111, + "minimum": 52652765.7823724, + "maximum": 100003423.27378607 + }, + "files": { + "samples": 8, + "median": 5.0, + "minimum": 3.0, + "maximum": 27.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 40677336.46764976, + "minimum": 26838049.28921842, + "maximum": 50364176.44315483 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-authenticated-connectivity-probe", + "seconds": { + "samples": 1, + "median": 3.210240794, + "minimum": 3.210240794, + "maximum": 3.210240794 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 619673320, + "serviceSeconds": 3.210240794, + "bytesPerSecond": 193030168.06657648, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 1108977176, + "serviceSeconds": 3.210240794, + "bytesPerSecond": 345449842.2899301, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 1, + "median": 1018425874.0, + "minimum": 1018425874.0, + "maximum": 1018425874.0 + }, + "archiveBytes": { + "samples": 1, + "median": 619673320.0, + "minimum": 619673320.0, + "maximum": 619673320.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 1108977176.0, + "minimum": 1108977176.0, + "maximum": 1108977176.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5587791465962506, + "minimum": 0.5587791465962506, + "maximum": 0.5587791465962506 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 345449842.2899301, + "minimum": 345449842.2899301, + "maximum": 345449842.2899301 + }, + "files": { + "samples": 1, + "median": 39.0, + "minimum": 39.0, + "maximum": 39.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 193030168.06657648, + "minimum": 193030168.06657648, + "maximum": 193030168.06657648 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-snapshot-acquire", + "seconds": { + "samples": 1, + "median": 250.851621405, + "minimum": 250.851621405, + "maximum": 250.851621405 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 619673320, + "serviceSeconds": 250.851621405, + "bytesPerSecond": 2470278.312451237, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 1108977176, + "serviceSeconds": 250.851621405, + "bytesPerSecond": 4420849.144959506, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 1, + "median": 1018425874.0, + "minimum": 1018425874.0, + "maximum": 1018425874.0 + }, + "archiveBytes": { + "samples": 1, + "median": 619673320.0, + "minimum": 619673320.0, + "maximum": 619673320.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 1108977176.0, + "minimum": 1108977176.0, + "maximum": 1108977176.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5587791465962506, + "minimum": 0.5587791465962506, + "maximum": 0.5587791465962506 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 4420849.144959506, + "minimum": 4420849.144959506, + "maximum": 4420849.144959506 + }, + "files": { + "samples": 1, + "median": 39.0, + "minimum": 39.0, + "maximum": 39.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 2470278.312451237, + "minimum": 2470278.312451237, + "maximum": 2470278.312451237 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-private-upload", + "seconds": { + "samples": 1, + "median": 28.724556513, + "minimum": 28.724556513, + "maximum": 28.724556513 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 619673320, + "serviceSeconds": 28.724556513, + "bytesPerSecond": 21572946.468975134, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 1108977176, + "serviceSeconds": 28.724556513, + "bytesPerSecond": 38607286.26038509, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 1, + "median": 1018425874.0, + "minimum": 1018425874.0, + "maximum": 1018425874.0 + }, + "archiveBytes": { + "samples": 1, + "median": 619673320.0, + "minimum": 619673320.0, + "maximum": 619673320.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 1108977176.0, + "minimum": 1108977176.0, + "maximum": 1108977176.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5587791465962506, + "minimum": 0.5587791465962506, + "maximum": 0.5587791465962506 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 38607286.26038509, + "minimum": 38607286.26038509, + "maximum": 38607286.26038509 + }, + "files": { + "samples": 1, + "median": 39.0, + "minimum": 39.0, + "maximum": 39.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 21572946.468975134, + "minimum": 21572946.468975134, + "maximum": 21572946.468975134 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-receiver-verify-and-extract", + "seconds": { + "samples": 1, + "median": 19.263423801, + "minimum": 19.263423801, + "maximum": 19.263423801 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 619673320, + "serviceSeconds": 19.263423801, + "bytesPerSecond": 32168389.503418997, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 1108977176, + "serviceSeconds": 19.263423801, + "bytesPerSecond": 57569058.722698666, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 1, + "median": 1018425874.0, + "minimum": 1018425874.0, + "maximum": 1018425874.0 + }, + "archiveBytes": { + "samples": 1, + "median": 619673320.0, + "minimum": 619673320.0, + "maximum": 619673320.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 1108977176.0, + "minimum": 1108977176.0, + "maximum": 1108977176.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5587791465962506, + "minimum": 0.5587791465962506, + "maximum": 0.5587791465962506 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 57569058.722698666, + "minimum": 57569058.722698666, + "maximum": 57569058.722698666 + }, + "files": { + "samples": 1, + "median": 39.0, + "minimum": 39.0, + "maximum": 39.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 32168389.503418997, + "minimum": 32168389.503418997, + "maximum": 32168389.503418997 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + } + ], + "consumerMergeObservations": [ + { + "activityFamily": "madd", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 3, + "median": 67.094164362, + "minimum": 66.79656049, + "maximum": 219.086229683 + }, + "checkSeconds": { + "samples": 3, + "median": 25.7431940460001, + "minimum": 25.456530666999925, + "maximum": 121.27468780899994 + }, + "copySeconds": { + "samples": 3, + "median": 34.58182319599996, + "minimum": 33.813945880999654, + "maximum": 62.06325912700004 + }, + "verifySeconds": { + "samples": 3, + "median": 6.73655051500009, + "minimum": 6.551817965000055, + "maximum": 35.192216692999864 + } + }, + { + "activityFamily": "mbackground", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 12, + "median": 18.883627134, + "minimum": 17.8304088, + "maximum": 19.754857759 + }, + "checkSeconds": { + "samples": 12, + "median": 4.557633406999969, + "minimum": 3.6671020270000554, + "maximum": 4.961447119000013 + }, + "copySeconds": { + "samples": 12, + "median": 10.148011529499968, + "minimum": 9.761975659999962, + "maximum": 10.434541308000007 + }, + "verifySeconds": { + "samples": 12, + "median": 3.693423046499902, + "minimum": 2.9794699060000767, + "maximum": 4.000663433 + } + }, + { + "activityFamily": "mbgmodel", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 3, + "median": 15.077407549, + "minimum": 14.266392464, + "maximum": 15.232841298 + }, + "checkSeconds": { + "samples": 3, + "median": 2.53756751800006, + "minimum": 2.5070188539998526, + "maximum": 2.962898128000006 + }, + "copySeconds": { + "samples": 3, + "median": 8.130230359999814, + "minimum": 8.074467681999977, + "maximum": 9.150769504999971 + }, + "verifySeconds": { + "samples": 3, + "median": 3.020684265, + "minimum": 3.0046015890000035, + "maximum": 3.463600305 + } + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "hpc", + "mergeSeconds": { + "samples": 2, + "median": 18.891379676, + "minimum": 17.266011092, + "maximum": 20.51674826 + }, + "checkSeconds": { + "samples": 2, + "median": 6.206628285348415, + "minimum": 5.660585141740739, + "maximum": 6.752671428956091 + }, + "copySeconds": { + "samples": 2, + "median": 6.046090563759208, + "minimum": 5.871328831650317, + "maximum": 6.220852295868099 + }, + "verifySeconds": { + "samples": 2, + "median": 4.663542255293578, + "minimum": 4.222140168771148, + "maximum": 5.104944341816008 + } + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 1, + "median": 24.153126054, + "minimum": 24.153126054, + "maximum": 24.153126054 + }, + "checkSeconds": { + "samples": 1, + "median": 8.702594806999969, + "minimum": 8.702594806999969, + "maximum": 8.702594806999969 + }, + "copySeconds": { + "samples": 1, + "median": 11.903956375999996, + "minimum": 11.903956375999996, + "maximum": 11.903956375999996 + }, + "verifySeconds": { + "samples": 1, + "median": 3.0493237859999454, + "minimum": 3.0493237859999454, + "maximum": 3.0493237859999454 + } + }, + { + "activityFamily": "mdifffit", + "resourceClass": "hpc", + "mergeSeconds": { + "samples": 18, + "median": 10.650899734, + "minimum": 5.614199461, + "maximum": 21.22848251 + }, + "checkSeconds": { + "samples": 18, + "median": 1.4091542228125036, + "minimum": 0.677861861884594, + "maximum": 3.194015149027109 + }, + "copySeconds": { + "samples": 18, + "median": 1.472872169688344, + "minimum": 1.2712753061205149, + "maximum": 1.878188787959516 + }, + "verifySeconds": { + "samples": 18, + "median": 4.185988076031208, + "minimum": 1.6743122031912208, + "maximum": 5.521449169144034 + } + }, + { + "activityFamily": "mimgtbl", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 3, + "median": 53.298720149, + "minimum": 51.926005982, + "maximum": 53.377830025 + }, + "checkSeconds": { + "samples": 3, + "median": 18.728920129000016, + "minimum": 17.38217044000021, + "maximum": 18.955533912999954 + }, + "copySeconds": { + "samples": 3, + "median": 27.39036550999981, + "minimum": 27.380674623000004, + "maximum": 27.65483295599995 + }, + "verifySeconds": { + "samples": 3, + "median": 6.6541218470001695, + "minimum": 6.2337451810000175, + "maximum": 6.684228208999912 + } + }, + { + "activityFamily": "mviewer", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 4, + "median": 60.991183819, + "minimum": 29.459673139, + "maximum": 104.951331755 + }, + "checkSeconds": { + "samples": 4, + "median": 13.8288311230001, + "minimum": 5.738225310999951, + "maximum": 34.67292259200008 + }, + "copySeconds": { + "samples": 4, + "median": 25.007742583499976, + "minimum": 16.94268330900013, + "maximum": 49.24239904699971 + }, + "verifySeconds": { + "samples": 4, + "median": 13.40475499099989, + "minimum": 6.247800998000002, + "maximum": 36.95691540500002 + } + } + ], + "mergeRateObservations": [ + { + "targetClass": "e2-micro", + "source": "baseline:montage-clean-real-20261011-v1", + "observations": { + "checkSecondsPerInputByte": { + "samples": 26, + "median": 6.2190376479285e-09, + "minimum": 4.04158434198331e-09, + "maximum": 3.1265677366835255e-08 + }, + "copySecondsPerUniqueByte": { + "samples": 26, + "median": 1.6090916291863342e-08, + "minimum": 1.4574432595848423e-08, + "maximum": 5.8709123754770746e-08 + }, + "verifySecondsPerUniqueByte": { + "samples": 26, + "median": 5.9409084403851915e-09, + "minimum": 4.745400286178283e-09, + "maximum": 3.3325226348030825e-08 + } + }, + "commandOverheadSeconds": null, + "channelWaitSeconds": null + }, + { + "targetClass": "hpc", + "source": "baseline:montage-clean-real-20261011-v1", + "observations": { + "checkSecondsPerInputByte": { + "samples": 20, + "median": 6.1298139012793435e-09, + "minimum": 3.1751234028133156e-09, + "maximum": 1.2935774743283055e-08 + }, + "copySecondsPerUniqueByte": { + "samples": 20, + "median": 6.6847097203047835e-09, + "minimum": 6.024563031240022e-09, + "maximum": 1.2348273488129912e-08 + }, + "verifySecondsPerUniqueByte": { + "samples": 20, + "median": 1.6028533003778777e-08, + "minimum": 8.072685871648882e-09, + "maximum": 2.3498275561055637e-08 + } + }, + "commandOverheadSeconds": { + "samples": 20, + "median": 1.2050064484193266, + "minimum": 0.8645295389928158, + "maximum": 1.4323608704028086 + }, + "channelWaitSeconds": { + "samples": 20, + "median": 2.3327023555000004, + "minimum": 8.0375e-05, + "maximum": 11.018675047 + } + } + ], + "limitations": [ + "Single baseline; ranges are observations, not confidence intervals.", + "Do not replay per-activity durations or total makespan as a holdout prediction.", + "No destination cache hit or shared-storage route may be assumed without evidence.", + "Workspace manifests and topology-specific contention are required before applying these observations.", + "Compute samples are contention-conditioned; do not apply interference slowdown a second time." + ], + "auxiliarySourceServiceEvidence": { + "runId": "montage-calibration-real-20261011-v1", + "sha256": "56fe504f6d8491ed12b53a954c2ad33cc0776a655cb18c23620bc57108e5057d", + "completedTransferSamples": 9, + "role": "completed commands from interrupted diagnostic; not independent validation", + "queueNotAddedToService": true, + "completedHPCMergeSamples": 20 + } + } + }, + "ako151-source-pool-v3-model": { + "sha256": "f0ddab1065b6f1b07a05d287f4ee718f6d66e47b3d596b8611a7beaa29a2e9e9", + "evidence": { + "schemaVersion": 1, + "source": "montage-clean-real-20261011-v1:sha256:0ab9d1bd45fbce2ed8b847b07f9e190f34be31709df3c48bfd7f93102426ded3", + "relayResourceId": "montage-relay-controller", + "resources": { + "montage-relay-controller": { + "class": "relay", + "domain": "relay" + }, + "montage-gcp-e2-medium-1": { + "class": "e2-medium", + "domain": "cloud" + }, + "montage-gcp-e2-medium-2": { + "class": "e2-medium", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-01": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-02": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-03": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-04": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-05": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-06": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-07": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-08": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-09": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-10": { + "class": "e2-micro", + "domain": "cloud" + }, + "plafrim-execution-ssh-node-bora001": { + "class": "hpc", + "domain": "hpc", + "bulkServiceGroupId": "plafrim-reference-ssh-bulk", + "bulkServiceSlots": 3, + "bulkServiceSource": "AKO-148 SSHChannelLimit4 reserve1; diagnostic source waits prove two waves of three" + }, + "plafrim-execution-ssh-node-diablo03": { + "class": "hpc", + "domain": "hpc", + "bulkServiceGroupId": "plafrim-reference-ssh-bulk", + "bulkServiceSlots": 3, + "bulkServiceSource": "AKO-148 SSHChannelLimit4 reserve1; diagnostic source waits prove two waves of three" + } + }, + "profiles": [ + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "source": "montage-clean-real-20261011-v1:sha256:0ab9d1bd45fbce2ed8b847b07f9e190f34be31709df3c48bfd7f93102426ded3", + "samples": 20, + "archiveToWorkspaceRatio": 0.5039255042100858, + "uploadArchiveBytesPerSecond": 10156946.070258675, + "ingressArchiveBytesPerSecond": 778033.9773214125, + "probeSeconds": 0.649784334, + "ingressIncludesSourcePreparation": true, + "validationSecondsPerByte": 5.1842243805209585e-09, + "receiverSecondsPerByte": 1.3425258633865797e-08 + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "source": "montage-clean-real-20261011-v1:sha256:0ab9d1bd45fbce2ed8b847b07f9e190f34be31709df3c48bfd7f93102426ded3", + "samples": 1, + "archiveToWorkspaceRatio": 0.5587791465962506, + "uploadArchiveBytesPerSecond": 21572946.468975134, + "probeSeconds": 3.210240794, + "snapshotSecondsPerByte": 2.2620088747886009e-07, + "receiverSecondsPerByte": 1.7370442077520265e-08 + } + ], + "mergeProfiles": [ + { + "targetClass": "e2-micro", + "source": "montage-clean-real-20261011-v1:sha256:0ab9d1bd45fbce2ed8b847b07f9e190f34be31709df3c48bfd7f93102426ded3", + "samples": 26, + "checkSecondsPerInputByte": 6.2190376479285e-09, + "copySecondsPerUniqueByte": 1.6090916291863342e-08, + "verifySecondsPerUniqueByte": 5.9409084403851915e-09 + }, + { + "targetClass": "hpc", + "source": "montage-clean-real-20261011-v1:sha256:0ab9d1bd45fbce2ed8b847b07f9e190f34be31709df3c48bfd7f93102426ded3", + "samples": 20, + "checkSecondsPerInputByte": 6.1298139012793435e-09, + "copySecondsPerUniqueByte": 6.6847097203047835e-09, + "verifySecondsPerUniqueByte": 1.6028533003778777e-08, + "commandOverheadSeconds": 1.2050064484193266 + } + ] + } + }, + "ako151-shared-source-v2-capacity": { + "sha256": "7435b48a745043efeb9316f5faf618873efcf742582e3f6662ca69e6d86b1362", + "evidence": { + "samples": 9, + "busySeconds": 527.3644335269928, + "concurrentCommandsMaximum": 3, + "durationWeightedMedianBytesPerSecond": 1081607.64706648, + "minimumBytesPerSecond": 358234.9543482617, + "maximumBytesPerSecond": 5569715.58658991, + "interpretation": "Aggregate source pack/ingress service, NOT NIC bandwidth; assumes uniform byte progress within each queue-free command.", + "limitations": [ + "Small auxiliary diagnostic sample; capacity is conditioned on workload and concurrent source work.", + "Admission queue excluded from command intervals." + ], + "runId": "montage-calibration-real-20261011-v1", + "sha256": "56fe504f6d8491ed12b53a954c2ad33cc0776a655cb18c23620bc57108e5057d" + } + } + }, + "gate": { + "accepted": false, + "newPlanningReleased": false, + "independentRealPrepared": false, + "reason": "Shared source-capacity fit meets makespan tolerance but dominant dependency residuals still exceed 100%; do not accept compensating errors." + }, + "remainingLimitations": [ + "Receiver command service is not identifiable separately from queue in completed references. No destination pool is added to those mixed samples.", + "Nine completed source commands and measured HPC merges from an interrupted diagnostic are auxiliary calibration evidence, not a completed-run benchmark.", + "Aggregate source capacity varies with workload/concurrency; constant-rate fit cannot explain every dominant dependency.", + "New planning must retain only evidenced route classes. Unobserved placement/storage identity must not be silently inferred.", + "Provisioning and orchestration timing retained from the frozen original forecast; not yet independently validated.", + "Compute and lifecycle cost bases unchanged; e2-medium interference remains extrapolated and network/public-IP billing remains incomplete." + ] +} diff --git a/experiments/montage-micro-hpc-e2/transfer-wall-sensitivity-20261011.json b/experiments/montage-micro-hpc-e2/transfer-wall-sensitivity-20261011.json new file mode 100644 index 00000000..7ae13f36 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/transfer-wall-sensitivity-20261011.json @@ -0,0 +1,32 @@ +{ + "issue": "AKO-158", + "role": "exploratory-effective-wall-profile-fit-not-independent-validation", + "referenceRunId": "montage-prism-transfer-real-20261011-v2", + "wholeWorkspaceVolumesUsed": true, + "newRealSubmitted": false, + "previousPlanSeconds": 3720.168973798004, + "previousSimulationSeconds": 3825.5614337389516, + "newPlanSeconds": 4615.267278717876, + "newSimulationSeconds": 4648.1042131221075, + "fixedPlacementSimulationSeconds": 4309.4108525254105, + "completedSimulationTasks": 58, + "profileSamples": { "hpcToCloudRelay": 42, "cloudToCloudPeer": 17 }, + "dominantDependency": { + "previousPredictedSeconds": 288.8601371575005, + "newPredictedSeconds": 636.8082884753348, + "observedSeconds": 1294.134133049, + "newDurationErrorPercent": -50.79271365982715, + "predictedNetworkBytes": 618568774, + "observedNetworkBytes": 596317206 + }, + "fixedPlacementActivityWallWithin20Percent": { "previous": 19, "new": 13, "comparable": 56 }, + "accepted": false, + "limitations": [ + "Uniform effective rates by route do not explain concurrency-conditioned dominant transfers.", + "Unknown SSH/cache waiting and CPU/IO contention remain in wall coefficients; rates are not queue-free network bandwidth.", + "Explicit SSH groups are disabled in this sensitivity to avoid adding admission twice; simulated network sharing can still overlap observed sharing.", + "Logical output bytes are not whole workspace bytes; calibration uses preserved source workspace sizeBytes.", + "Replanned placement is slower than frozen-placement native simulation; planner optimality is not established.", + "A longer makespan alone does not prove a more faithful model; the per-activity gate worsened." + ] +} diff --git a/experiments/montage-micro-hpc-e2/v5-comparison.json b/experiments/montage-micro-hpc-e2/v5-comparison.json new file mode 100644 index 00000000..398fd840 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/v5-comparison.json @@ -0,0 +1,824 @@ +{ + "schema_version": 1, + "run_id": "montage-micro-hpc-e2-real-20261007-v5", + "audit": { + "expected_count": 58, + "missing": [ + + ], + "unexpected": [ + + ], + "not_completed": [ + + ], + "incomplete_transfer_count": 0, + "barrier_violation_count": 0, + "unreleased_lease_count": 0 + }, + "plan": { + "makespanSeconds": 3567.8707086633945, + "cost": 0 + }, + "plan_assignment_compute_work_seconds": 27666.338780192465, + "plan_assignment_transfer_work_seconds": 24.256941990412088, + "simulation": { + "makespanSeconds": 3567.8162834323634, + "cost": 0, + "breakdown": { + "computeSeconds": 27654.47752709201, + "transferSeconds": 29.90346456857492, + "queueSeconds": 361.2112485549678, + "interferenceSeconds": 3388.4775270920195, + "overheadSeconds": 0 + }, + "transferredBytes": 820143360 + }, + "real_api_reported": { + "makespanSeconds": 10852.481158733368, + "cost": 0, + "breakdown": { + "computeSeconds": 23113.980337381363, + "transferSeconds": 36232.68752288818, + "queueSeconds": 23406.0823969841, + "interferenceSeconds": 0, + "overheadSeconds": 38.18167543411255 + }, + "transferredBytes": 26801976910 + }, + "real_api_run_timestamp_wall_seconds": 9324.0, + "real_network_bytes_from_routes": 59724006790, + "transfer_route_count": 122, + "activities": [ + { + "activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "resource_id": "montage-gcp-fog-e2-micro-06", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 3.604032768000252, + "observed_runtime_seconds": 144.2626724243164, + "preparation_elapsed_seconds": 488.9450030326843, + "overlapping_transfer_work_seconds": 2444.7227578163147, + "launch_wait_seconds": 31.158469676971436, + "queue_seconds": 523.267852306366, + "network_bytes": 4605163608 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.9640327680012888, + "observed_runtime_seconds": 341.35736656188965, + "preparation_elapsed_seconds": 927.6489601135254, + "overlapping_transfer_work_seconds": 4638.23574924469, + "launch_wait_seconds": 40.61918663978577, + "queue_seconds": 1648.296317577362, + "network_bytes": 4529756266 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "resource_id": "montage-gcp-fog-e2-micro-05", + "attempt": 1, + "planned_compute_seconds": 8, + "planned_transfer_seconds": 0.9640327680012888, + "observed_runtime_seconds": 277.3620390892029, + "preparation_elapsed_seconds": 983.138617515564, + "overlapping_transfer_work_seconds": 4915.685327529907, + "launch_wait_seconds": 49.484028339385986, + "queue_seconds": 1712.2788462638855, + "network_bytes": 4575758268 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013", + "resource_id": "montage-gcp-fog-e2-micro-03", + "attempt": 1, + "planned_compute_seconds": 4, + "planned_transfer_seconds": 0.4014915200004907, + "observed_runtime_seconds": 258.24932408332825, + "preparation_elapsed_seconds": 232.3864860534668, + "overlapping_transfer_work_seconds": 464.77296710014343, + "launch_wait_seconds": 10.384948492050171, + "queue_seconds": 324.903817653656, + "network_bytes": 1025486888 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014", + "resource_id": "montage-gcp-fog-e2-micro-04", + "attempt": 1, + "planned_compute_seconds": 4, + "planned_transfer_seconds": 0.08049152000012327, + "observed_runtime_seconds": 214.96390891075134, + "preparation_elapsed_seconds": 268.17702317237854, + "overlapping_transfer_work_seconds": 536.3540370464325, + "launch_wait_seconds": 12.128744840621948, + "queue_seconds": 365.60555768013, + "network_bytes": 1027045703 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 1, + "planned_transfer_seconds": 0.5620032768001693, + "observed_runtime_seconds": 3.8778188228607178, + "preparation_elapsed_seconds": 74.99215316772461, + "overlapping_transfer_work_seconds": 147.86728477478027, + "launch_wait_seconds": 5.495436429977417, + "queue_seconds": 0.16001176834106445, + "network_bytes": 290209781 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016", + "resource_id": "montage-gcp-fog-e2-micro-10", + "attempt": 1, + "planned_compute_seconds": 4, + "planned_transfer_seconds": 0.08049152000012327, + "observed_runtime_seconds": 247.22693276405334, + "preparation_elapsed_seconds": 238.2373025417328, + "overlapping_transfer_work_seconds": 476.4743835926056, + "launch_wait_seconds": 12.872168064117432, + "queue_seconds": 336.51263546943665, + "network_bytes": 1029485875 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032", + "resource_id": "montage-gcp-fog-e2-micro-08", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.0011310720001347363, + "observed_runtime_seconds": 259.78463220596313, + "preparation_elapsed_seconds": 99.46577787399292, + "overlapping_transfer_work_seconds": 198.93100094795227, + "launch_wait_seconds": 10.76372480392456, + "queue_seconds": 323.8190002441406, + "network_bytes": 1009412120 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 100, + "planned_transfer_seconds": 0.0011310720001347363, + "observed_runtime_seconds": 352.403440952301, + "preparation_elapsed_seconds": 136.8810441493988, + "overlapping_transfer_work_seconds": 273.76100039482117, + "launch_wait_seconds": 9.436262845993042, + "queue_seconds": 230.86548972129822, + "network_bytes": 1011214234 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034", + "resource_id": "montage-gcp-e2-medium-1", + "attempt": 1, + "planned_compute_seconds": 23, + "planned_transfer_seconds": 0.0011310720001347363, + "observed_runtime_seconds": 373.4897518157959, + "preparation_elapsed_seconds": 118.25735688209534, + "overlapping_transfer_work_seconds": 236.514151096344, + "launch_wait_seconds": 9.066216707229614, + "queue_seconds": 209.75061225891113, + "network_bytes": 1013537982 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035", + "resource_id": "montage-gcp-e2-medium-2", + "attempt": 1, + "planned_compute_seconds": 9, + "planned_transfer_seconds": 0.0011310720001347363, + "observed_runtime_seconds": 374.20631527900696, + "preparation_elapsed_seconds": 115.68949055671692, + "overlapping_transfer_work_seconds": 231.37880039215088, + "launch_wait_seconds": 7.428061008453369, + "queue_seconds": 208.3925702571869, + "network_bytes": 1013076250 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "resource_id": "montage-gcp-fog-e2-micro-06", + "attempt": 1, + "planned_compute_seconds": 64, + "planned_transfer_seconds": 0.8811310720000165, + "observed_runtime_seconds": 2.8815789222717285, + "preparation_elapsed_seconds": 334.8788957595825, + "overlapping_transfer_work_seconds": 669.7576863765717, + "launch_wait_seconds": 9.73267149925232, + "queue_seconds": 577.6622266769409, + "network_bytes": 839940359 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052", + "resource_id": "montage-gcp-fog-e2-micro-04", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.8811310720000165, + "observed_runtime_seconds": 61.57319521903992, + "preparation_elapsed_seconds": 299.0221507549286, + "overlapping_transfer_work_seconds": 598.0435538291931, + "launch_wait_seconds": 8.033943176269531, + "queue_seconds": 518.9747133255005, + "network_bytes": 842208331 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 48, + "planned_transfer_seconds": 0.08313107200046943, + "observed_runtime_seconds": 312.6725015640259, + "preparation_elapsed_seconds": 174.88084959983826, + "overlapping_transfer_work_seconds": 349.7616677284241, + "launch_wait_seconds": 10.62543773651123, + "queue_seconds": 270.4502375125885, + "network_bytes": 1024955175 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054", + "resource_id": "montage-gcp-fog-e2-micro-05", + "attempt": 1, + "planned_compute_seconds": 48, + "planned_transfer_seconds": 0.295797738667261, + "observed_runtime_seconds": 325.73548889160156, + "preparation_elapsed_seconds": 165.5724949836731, + "overlapping_transfer_work_seconds": 331.1447367668152, + "launch_wait_seconds": 10.566657066345215, + "queue_seconds": 258.26020646095276, + "network_bytes": 1024911402 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "resource_id": "plafrim-execution-ssh-node-diablo03", + "attempt": 1, + "planned_compute_seconds": 9, + "planned_transfer_seconds": 0.080163839999841, + "observed_runtime_seconds": 1.8560967445373535, + "preparation_elapsed_seconds": 128.3770616054535, + "overlapping_transfer_work_seconds": 126.8277416229248, + "launch_wait_seconds": 5.583914518356323, + "queue_seconds": 0.7973895072937012, + "network_bytes": 739563580 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "resource_id": "montage-gcp-fog-e2-micro-06", + "attempt": 1, + "planned_compute_seconds": 28, + "planned_transfer_seconds": 0.0010327680001864792, + "observed_runtime_seconds": 2.708998203277588, + "preparation_elapsed_seconds": 168.7302565574646, + "overlapping_transfer_work_seconds": 168.7302565574646, + "launch_wait_seconds": 8.619796514511108, + "queue_seconds": 341.4066047668457, + "network_bytes": 726584277 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "resource_id": "montage-gcp-fog-e2-micro-01", + "attempt": 1, + "planned_compute_seconds": 80, + "planned_transfer_seconds": 0.080163839999841, + "observed_runtime_seconds": 49.80984282493591, + "preparation_elapsed_seconds": 118.15841174125671, + "overlapping_transfer_work_seconds": 118.15841174125671, + "launch_wait_seconds": 12.123093843460083, + "queue_seconds": 294.3410120010376, + "network_bytes": 735942580 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "resource_id": "montage-gcp-e2-medium-1", + "attempt": 1, + "planned_compute_seconds": 17, + "planned_transfer_seconds": 0.00626214400108438, + "observed_runtime_seconds": 1.7220439910888672, + "preparation_elapsed_seconds": 141.55394673347473, + "overlapping_transfer_work_seconds": 849.3142881393433, + "launch_wait_seconds": 13.466747999191284, + "queue_seconds": 698.6337170600891, + "network_bytes": 2505808115 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "resource_id": "montage-gcp-fog-e2-micro-05", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.006196608000664128, + "observed_runtime_seconds": 1.7506978511810303, + "preparation_elapsed_seconds": 491.46055603027344, + "overlapping_transfer_work_seconds": 2948.7576208114624, + "launch_wait_seconds": 32.99739909172058, + "queue_seconds": 755.4324867725372, + "network_bytes": 2467665263 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 5, + "planned_transfer_seconds": 0.2424980735995632, + "observed_runtime_seconds": 2.7940571308135986, + "preparation_elapsed_seconds": 206.196453332901, + "overlapping_transfer_work_seconds": 1226.9844052791595, + "launch_wait_seconds": 14.425408840179443, + "queue_seconds": 0.4984259605407715, + "network_bytes": 1260005826 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000005", + "resource_id": "montage-gcp-fog-e2-micro-01", + "attempt": 1, + "planned_compute_seconds": 120, + "planned_transfer_seconds": 0.5620000000003529, + "observed_runtime_seconds": 251.65013813972473, + "preparation_elapsed_seconds": 107.92653584480286, + "overlapping_transfer_work_seconds": 215.85226464271545, + "launch_wait_seconds": 16.633808374404907, + "queue_seconds": 328.3393430709839, + "network_bytes": 573380833 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000006", + "resource_id": "montage-gcp-fog-e2-micro-06", + "attempt": 1, + "planned_compute_seconds": 112, + "planned_transfer_seconds": 0.5225000000004911, + "observed_runtime_seconds": 4.42097544670105, + "preparation_elapsed_seconds": 43.949525117874146, + "overlapping_transfer_work_seconds": 87.8982663154602, + "launch_wait_seconds": 23.008190631866455, + "queue_seconds": 127.67748355865479, + "network_bytes": 576120792 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000007", + "resource_id": "montage-gcp-fog-e2-micro-05", + "attempt": 1, + "planned_compute_seconds": 96, + "planned_transfer_seconds": 0.5620000000003529, + "observed_runtime_seconds": 21.61045813560486, + "preparation_elapsed_seconds": 67.0773537158966, + "overlapping_transfer_work_seconds": 134.1543927192688, + "launch_wait_seconds": 129.41329097747803, + "queue_seconds": 352.7021062374115, + "network_bytes": 575821004 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000008", + "resource_id": "montage-gcp-fog-e2-micro-03", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.24099999999998545, + "observed_runtime_seconds": 73.99173498153687, + "preparation_elapsed_seconds": 44.43785071372986, + "overlapping_transfer_work_seconds": 88.87482810020447, + "launch_wait_seconds": 5.970422267913818, + "queue_seconds": 256.57098746299744, + "network_bytes": 577679603 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009", + "resource_id": "montage-gcp-e2-medium-2", + "attempt": 1, + "planned_compute_seconds": 11, + "planned_transfer_seconds": 0.4819999999999709, + "observed_runtime_seconds": 155.6325740814209, + "preparation_elapsed_seconds": 59.58903241157532, + "overlapping_transfer_work_seconds": 119.17725443840027, + "launch_wait_seconds": 5.364755630493164, + "queue_seconds": 218.84639406204224, + "network_bytes": 577379820 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000010", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 96, + "planned_transfer_seconds": 0.44250000000010914, + "observed_runtime_seconds": 146.79116415977478, + "preparation_elapsed_seconds": 69.10107159614563, + "overlapping_transfer_work_seconds": 138.2019395828247, + "launch_wait_seconds": 4.509028911590576, + "queue_seconds": 227.49827790260315, + "network_bytes": 580119778 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000024", + "resource_id": "montage-gcp-fog-e2-micro-08", + "attempt": 1, + "planned_compute_seconds": 64, + "planned_transfer_seconds": 0.1610000000000582, + "observed_runtime_seconds": 61.62095808982849, + "preparation_elapsed_seconds": 51.17185592651367, + "overlapping_transfer_work_seconds": 102.34290957450867, + "launch_wait_seconds": 10.996396780014038, + "queue_seconds": 268.9418365955353, + "network_bytes": 567456347 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000025", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 80, + "planned_transfer_seconds": 0.40200000000004366, + "observed_runtime_seconds": 34.83442044258118, + "preparation_elapsed_seconds": 53.62738823890686, + "overlapping_transfer_work_seconds": 107.25428891181946, + "launch_wait_seconds": 36.151440143585205, + "queue_seconds": 295.8706965446472, + "network_bytes": 569780096 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000026", + "resource_id": "montage-gcp-fog-e2-micro-10", + "attempt": 1, + "planned_compute_seconds": 88.6288311173912, + "planned_transfer_seconds": 0.40200000000004366, + "observed_runtime_seconds": 63.93125605583191, + "preparation_elapsed_seconds": 59.42932486534119, + "overlapping_transfer_work_seconds": 118.85849523544312, + "launch_wait_seconds": 3.719320774078369, + "queue_seconds": 266.6180467605591, + "network_bytes": 569318364 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000027", + "resource_id": "montage-gcp-fog-e2-micro-01", + "attempt": 1, + "planned_compute_seconds": 60, + "planned_transfer_seconds": 0.40200000000004366, + "observed_runtime_seconds": 137.68307757377625, + "preparation_elapsed_seconds": 70.1656928062439, + "overlapping_transfer_work_seconds": 140.33001279830933, + "launch_wait_seconds": 119.70827746391296, + "queue_seconds": 334.59805512428284, + "network_bytes": 571582210 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000028", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 1, + "planned_compute_seconds": 79.99999999999977, + "planned_transfer_seconds": 0.24099999999998545, + "observed_runtime_seconds": 27.769585847854614, + "preparation_elapsed_seconds": 80.59860730171204, + "overlapping_transfer_work_seconds": 161.1967978477478, + "launch_wait_seconds": 16.360780239105225, + "queue_seconds": 103.72483086585999, + "network_bytes": 571120478 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000029", + "resource_id": "montage-gcp-fog-e2-micro-04", + "attempt": 3, + "planned_compute_seconds": 52, + "planned_transfer_seconds": 0.4819999999999709, + "observed_runtime_seconds": 33.249143838882446, + "preparation_elapsed_seconds": 138.3420331478119, + "overlapping_transfer_work_seconds": 276.68331241607666, + "launch_wait_seconds": 24.244794845581055, + "queue_seconds": 400.5515911579132, + "network_bytes": 573444227 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000043", + "resource_id": "plafrim-execution-ssh-node-diablo03", + "attempt": 1, + "planned_compute_seconds": 11, + "planned_transfer_seconds": 0.009000000000014552, + "observed_runtime_seconds": 2.673428535461426, + "preparation_elapsed_seconds": 7.0569868087768555, + "overlapping_transfer_work_seconds": 14.069963216781616, + "launch_wait_seconds": 4.43604850769043, + "queue_seconds": 0.11738777160644531, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000044", + "resource_id": "montage-gcp-fog-e2-micro-09", + "attempt": 1, + "planned_compute_seconds": 44, + "planned_transfer_seconds": 0.8799999999998818, + "observed_runtime_seconds": 151.11678051948547, + "preparation_elapsed_seconds": 146.70770525932312, + "overlapping_transfer_work_seconds": 293.41430139541626, + "launch_wait_seconds": 31.818129539489746, + "queue_seconds": 314.8557949066162, + "network_bytes": 393010139 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045", + "resource_id": "plafrim-execution-ssh-node-diablo03", + "attempt": 3, + "planned_compute_seconds": 16, + "planned_transfer_seconds": 0.6410000000005311, + "observed_runtime_seconds": 1.636733055114746, + "preparation_elapsed_seconds": 62.057613372802734, + "overlapping_transfer_work_seconds": 115.78968286514282, + "launch_wait_seconds": 4.56097936630249, + "queue_seconds": 0.553372859954834, + "network_bytes": 288968704 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000046", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 14, + "planned_transfer_seconds": 0.4810000000002219, + "observed_runtime_seconds": 2.3873519897460938, + "preparation_elapsed_seconds": 120.45770072937012, + "overlapping_transfer_work_seconds": 238.87659621238708, + "launch_wait_seconds": 4.847031116485596, + "queue_seconds": 0.3953056335449219, + "network_bytes": 289012477 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000047", + "resource_id": "montage-gcp-fog-e2-micro-10", + "attempt": 3, + "planned_compute_seconds": 77.55022722771719, + "planned_transfer_seconds": 1.174666666667008, + "observed_runtime_seconds": 174.70377326011658, + "preparation_elapsed_seconds": 123.43664693832397, + "overlapping_transfer_work_seconds": 246.8722746372223, + "launch_wait_seconds": 24.040416717529297, + "queue_seconds": 229.3669376373291, + "network_bytes": 395234337 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mdifffitid0000048", + "resource_id": "montage-gcp-fog-e2-micro-09", + "attempt": 1, + "planned_compute_seconds": 76, + "planned_transfer_seconds": 0.29466666666712626, + "observed_runtime_seconds": 140.66779828071594, + "preparation_elapsed_seconds": 162.5476999282837, + "overlapping_transfer_work_seconds": 325.0953495502472, + "launch_wait_seconds": 26.42296862602234, + "queue_seconds": 325.29792165756226, + "network_bytes": 577981179 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "resource_id": "montage-gcp-fog-e2-micro-01", + "attempt": 1, + "planned_compute_seconds": 88, + "planned_transfer_seconds": 3.6030000000000655, + "observed_runtime_seconds": 788.4004962444305, + "preparation_elapsed_seconds": 576.3223161697388, + "overlapping_transfer_work_seconds": 2305.2867839336395, + "launch_wait_seconds": 55.247376680374146, + "queue_seconds": 632.1917777061462, + "network_bytes": 3411916818 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "resource_id": "montage-gcp-fog-e2-micro-03", + "attempt": 1, + "planned_compute_seconds": 100, + "planned_transfer_seconds": 1.1240000000011605, + "observed_runtime_seconds": 285.1898350715637, + "preparation_elapsed_seconds": 451.6803328990936, + "overlapping_transfer_work_seconds": 1806.7202920913696, + "launch_wait_seconds": 44.296215534210205, + "queue_seconds": 1135.2299027442932, + "network_bytes": 3354769357 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "resource_id": "montage-gcp-fog-e2-micro-07", + "attempt": 1, + "planned_compute_seconds": 36, + "planned_transfer_seconds": 0.9630000000011023, + "observed_runtime_seconds": 3.6926822662353516, + "preparation_elapsed_seconds": 720.3201177120209, + "overlapping_transfer_work_seconds": 2881.2765712738037, + "launch_wait_seconds": 54.475027561187744, + "queue_seconds": 1414.4761848449707, + "network_bytes": 3391791631 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000001", + "resource_id": "montage-gcp-e2-medium-2", + "attempt": 1, + "planned_compute_seconds": 2509.0870506376314, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 520.7625739574432, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 106.51812052726746, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000002", + "resource_id": "montage-gcp-fog-e2-micro-04", + "attempt": 3, + "planned_compute_seconds": 1656, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1818.560084104538, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 241.39356970787048, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000003", + "resource_id": "montage-gcp-fog-e2-micro-03", + "attempt": 3, + "planned_compute_seconds": 2200, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1531.1265585422516, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 241.1799294948578, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000004", + "resource_id": "montage-gcp-fog-e2-micro-10", + "attempt": 3, + "planned_compute_seconds": 3299.664954999394, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 2182.6583817005157, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 85.58447527885437, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000020", + "resource_id": "montage-gcp-fog-e2-micro-08", + "attempt": 3, + "planned_compute_seconds": 2112, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1986.8708395957947, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 73.08072113990784, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000021", + "resource_id": "montage-gcp-fog-e2-micro-02", + "attempt": 3, + "planned_compute_seconds": 1444, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1844.214260816574, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 73.48668694496155, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000022", + "resource_id": "montage-gcp-e2-medium-1", + "attempt": 1, + "planned_compute_seconds": 1953.1603327863493, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 622.9507427215576, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 162.1806902885437, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000023", + "resource_id": "montage-gcp-e2-medium-2", + "attempt": 1, + "planned_compute_seconds": 1974.0870506376316, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 514.642370223999, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 106.49829602241516, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000039", + "resource_id": "plafrim-execution-ssh-node-diablo03", + "attempt": 1, + "planned_compute_seconds": 1540, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 436.16385865211487, + "preparation_elapsed_seconds": 6.399613618850708, + "overlapping_transfer_work_seconds": 6.399613618850708, + "launch_wait_seconds": 4.573583364486694, + "queue_seconds": 1.1591229438781738, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000040", + "resource_id": "plafrim-execution-ssh-node-bora001", + "attempt": 1, + "planned_compute_seconds": 1537, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 432.9233865737915, + "preparation_elapsed_seconds": 6.401705980300903, + "overlapping_transfer_work_seconds": 6.401705980300903, + "launch_wait_seconds": 4.801985025405884, + "queue_seconds": 0.7325694561004639, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000041", + "resource_id": "montage-gcp-fog-e2-micro-09", + "attempt": 3, + "planned_compute_seconds": 3012, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 1855.756742477417, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 73.04916024208069, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mprojectid0000042", + "resource_id": "montage-gcp-e2-medium-1", + "attempt": 1, + "planned_compute_seconds": 2379.1603327863495, + "planned_transfer_seconds": 0, + "observed_runtime_seconds": 622.9535145759583, + "preparation_elapsed_seconds": null, + "overlapping_transfer_work_seconds": null, + "launch_wait_seconds": null, + "queue_seconds": 162.17444109916687, + "network_bytes": 0 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000019", + "resource_id": "montage-gcp-e2-medium-2", + "attempt": 1, + "planned_compute_seconds": 17, + "planned_transfer_seconds": 0.1610000000000582, + "observed_runtime_seconds": 1243.1292638778687, + "preparation_elapsed_seconds": 134.41252398490906, + "overlapping_transfer_work_seconds": 134.41252398490906, + "launch_wait_seconds": 8.319751501083374, + "queue_seconds": 1471.7621870040894, + "network_bytes": 1249544888 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000038", + "resource_id": "montage-gcp-fog-e2-micro-08", + "attempt": 1, + "planned_compute_seconds": 32, + "planned_transfer_seconds": 0.1610000000000582, + "observed_runtime_seconds": 548.611704826355, + "preparation_elapsed_seconds": 633.8678057193756, + "overlapping_transfer_work_seconds": 633.8678057193756, + "launch_wait_seconds": 56.27052426338196, + "queue_seconds": 840.4918801784515, + "network_bytes": 1230918523 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000057", + "resource_id": "montage-gcp-fog-e2-micro-09", + "attempt": 1, + "planned_compute_seconds": 4, + "planned_transfer_seconds": 0.1610000000000582, + "observed_runtime_seconds": 566.2262053489685, + "preparation_elapsed_seconds": 634.1015605926514, + "overlapping_transfer_work_seconds": 634.1015605926514, + "launch_wait_seconds": 37.13513255119324, + "queue_seconds": 822.1827816963196, + "network_bytes": 1239729945 + }, + { + "activity_id": "montage-58-fog-hpc-e2-20261006-mviewerid0000058", + "resource_id": "montage-gcp-e2-medium-1", + "attempt": 1, + "planned_compute_seconds": 7, + "planned_transfer_seconds": 0.4830000000001746, + "observed_runtime_seconds": 212.08677911758423, + "preparation_elapsed_seconds": 991.3245203495026, + "overlapping_transfer_work_seconds": 2973.971189260483, + "launch_wait_seconds": 34.37017464637756, + "queue_seconds": 1175.9244356155396, + "network_bytes": 3720193357 + } + ], + "limitations": [ + "Recovered run: not a clean uninterrupted performance benchmark.", + "API runtime includes delayed terminal observation; native Docker durations differ.", + "API aggregate counters can contain older attempts; audit uses latest distinct IDs.", + "Transfer work is summed overlapping route time, not wall-clock preparation.", + "Plan assignment work sums are not makespan; API timestamp wall and recovered makespan can differ.", + "Cost zero is uncalibrated, not free cloud execution.", + "e2-medium interference extrapolated to HPC/micro; micro speedup 0.25 is approximate.", + "Whole-workspace real traffic differs from estimated plan edge volumes.", + "Zero reported interference does not prove absence of physical contention." + ], + "product": { + "filename": "mosaic-color-v5.png", + "bytes": 4920343, + "sha256": "0f75c2e811e89ac56ae44784fa22e4e01368abf19a10cbe59a77bd19446fdf4a" + } +} diff --git a/experiments/montage-micro-hpc-e2/v5-native-stage-evidence.json b/experiments/montage-micro-hpc-e2/v5-native-stage-evidence.json new file mode 100644 index 00000000..20bb8924 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/v5-native-stage-evidence.json @@ -0,0 +1,46 @@ +{ + "run": "montage-micro-hpc-e2-real-20261007-v5", + "measurement": "Native Docker State.StartedAt/FinishedAt read through daemon console API; preparation from transfer phase records", + "stages": [ + {"activity": "mAdd18", "preparation_seconds": 488.945003, "native_command_seconds": 1.316860377, "network_bytes": 4605163608}, + {"activity": "mAdd37", "preparation_seconds": 927.648960, "barrier_seconds": 535.706014, "merge_seconds": 391.944042, "native_command_seconds": 9.753975640, "network_bytes": 4529756266}, + {"activity": "mAdd56", "preparation_seconds": 983.138618, "barrier_seconds": 540.045425, "merge_seconds": 443.092478, "native_command_seconds": 10.281894171, "network_bytes": 4575758268}, + {"activity": "mViewer19", "preparation_seconds": 134.412524, "native_command_seconds": 2.511764320, "network_bytes": 1249544888}, + {"activity": "mViewer38", "preparation_seconds": 633.867806, "native_command_seconds": 23.484506687, "network_bytes": 1230918523}, + {"activity": "mViewer57", "preparation_seconds": 634.101561, "native_command_seconds": 23.988753783, "network_bytes": 1239729945}, + {"activity": "mViewer58", "preparation_seconds": 991.3245203495, "barrier_seconds": 918.044581002, "merge_seconds": 73.085048784, "native_command_seconds": 5.963964158, "network_bytes": 3720193357} + ], + "interpretation": [ + "Full-workspace preparation dominates these final stages, not the image command.", + "Micro fan-in merge and compression/upload consume substantial time; no isolated bandwidth benchmark was performed.", + "Destination single-pass verification and reserved SSH control capacity were implemented after these measurements, not active during v5.", + "Synchronous idle cloud stop waits delayed observation; AKO-117 was implemented after v5 and is not validated by these measurements.", + "Run metrics include recoveries and delayed observation, so aggregate compute is not pure command time." + ], + "corrections": [ + {"issue": "AKO-102", "change": "Eight parallel predecessor routes per consumer"}, + {"issue": "AKO-103", "change": "Safe Terraform stop/start updates without create/delete/replace"}, + {"issue": "AKO-104", "change": "Verified per-generation SSH host trust"}, + {"issue": "AKO-105", "change": "Stop failed configuration only when a physical provider instance exists"}, + {"issue": "AKO-106", "change": "Bind workspace SSH to exact verified instance generation"}, + {"issue": "AKO-107", "change": "Deduplicate queued infrastructure and reconcile unused operations"}, + {"issue": "AKO-108", "change": "Batch independent exact cloud allocations"}, + {"issue": "AKO-109", "change": "Serialize runtime control per handle; protect observation cancellation"}, + {"issue": "AKO-110", "change": "Record transfer phase timings"}, + {"issue": "AKO-111", "change": "Compressed full-workspace archives with integrity verification"}, + {"issue": "AKO-112", "change": "Ready exact preserved stopped producer before recovery inspection"}, + {"issue": "AKO-113", "change": "Native remote Docker completion timestamp", "v5_active": false}, + {"issue": "AKO-114", "change": "Isolate Docker telemetry by actual recovery attempt", "v5_active": false}, + {"issue": "AKO-115", "change": "Verify unique destination paths once after merge", "v5_active": false}, + {"issue": "AKO-116", "change": "Reserve one SSH control channel from bulk transfer saturation", "v5_active": false}, + {"issue": "AKO-117", "change": "Independent bounded idle cloud stop progress", "v5_active": false} + ], + "cleanup": { + "cloud_instances_destroyed": 12, + "remaining_non_destroyed_instances": 0, + "active_cloud_operations": 0, + "hpc_squeue_empty": true, + "hpc_console_evidence": "console-command-ae1f3087c643d2532a2629e4", + "final_png_preserved_locally": true + } +} diff --git a/experiments/montage-micro-hpc-e2/v6-preservation.json b/experiments/montage-micro-hpc-e2/v6-preservation.json new file mode 100644 index 00000000..d6378fd1 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/v6-preservation.json @@ -0,0 +1,12 @@ +{ + "issue": "AKO-121", + "runId": "montage-micro-hpc-e2-real-20261008-v6", + "export": "Whole run workspace archives via daemon console API, bounded 4 MiB chunks", + "validation": "Remote and local SHA-256 match; complete tar listing succeeds", + "archives": [ + {"target": "montage-gcp-e2-medium-1", "bytes": 280542634, "sha256": "2471390d50a0632ea79b5cb8aecab1b3c53fb0d7b9f660ed4520ab2d59ba26f7"}, + {"target": "montage-gcp-e2-medium-2", "bytes": 278595230, "sha256": "0ddf5f376174c85f91732a76fb34d9edcb78c5bc4f39c9eafc9301491edf4d12"}, + {"target": "montage-gcp-fog-e2-micro-04", "bytes": 288542219, "sha256": "6416577b8358e49fba59ae60703cf55b8448dbcb694f19a5ab5d1a6a770d3998"} + ], + "limitations": ["Archives stay in ignored local storage, not Git", "Previously automatically destroyed micro10 is not covered by these archives"] +} diff --git a/experiments/montage-micro-hpc-e2/workspace-projection-v2.json b/experiments/montage-micro-hpc-e2/workspace-projection-v2.json new file mode 100644 index 00000000..ca978799 --- /dev/null +++ b/experiments/montage-micro-hpc-e2/workspace-projection-v2.json @@ -0,0 +1,7717 @@ +{ + "schemaVersion": 1, + "baselineRunId": "montage-docker-image-real-20261010-v2", + "auditedEvidenceSHA256": "ce8343153d9016ce2149fa5e70b3624ca148c4747e6dd9887dae1fa2b6ab46c6", + "contract": "whole-workspace-v1", + "runtimePackagingFiles": 45, + "activityModels": { + "montage-58-fog-hpc-e2-20261006-maddid0000018": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "1-mosaic.fits", + "sizeBytes": 25922880, + "contentId": "sha256:89840b15af5665d27288c810de30e71d9843d8edc57d7af5721b64d3e002dcb5" + }, + { + "path": "1-mosaic_area.fits", + "sizeBytes": 25922880, + "contentId": "sha256:d1ce65c62e27ba55577c34fcfb51f6ca4004d5f974518dcdad8441e319407372" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-maddid0000037": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "2-mosaic.fits", + "sizeBytes": 25922880, + "contentId": "sha256:39926300e3c3f28d4bcf0535eb5fd746ad98f18e677ea4823c1a2e85b0b58480" + }, + { + "path": "2-mosaic_area.fits", + "sizeBytes": 25922880, + "contentId": "sha256:841833f6feb118ef94e494434c943868a8cc13aad4f90237e9257b7ffb2f28f0" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-maddid0000056": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "3-mosaic.fits", + "sizeBytes": 25922880, + "contentId": "sha256:4a2b76a0f949a8c84d13ce4a9d69f9cb5fa6ad5e5a946b9357acbdc7266e040c" + }, + { + "path": "3-mosaic_area.fits", + "sizeBytes": 25922880, + "contentId": "sha256:140db1601b6a7956031bd5009e253371785964bc3d74f40fe94f4a1cff657667" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "cposs2ukstu_blue_001_001.fits", + "sizeBytes": 53193600, + "contentId": "sha256:09962cd502f62448d022d59b4b50314c30589fbbc7cf01b476b635f0f1066d22" + }, + { + "path": "cposs2ukstu_blue_001_001_area.fits", + "sizeBytes": 53193600, + "contentId": "sha256:ce2f3af33a9bb12b443eeef4f0f5fb40543c32883bcb62076a76e133092b9258" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "cposs2ukstu_blue_001_002.fits", + "sizeBytes": 53173440, + "contentId": "sha256:8b801df4b6399a42b75a28d2b7334b35164779828bdb5688bb123417e9ef08ae" + }, + { + "path": "cposs2ukstu_blue_001_002_area.fits", + "sizeBytes": 53173440, + "contentId": "sha256:baa881cf48b44433ec2a4ad756d3a9288f2ee2d43ffda273aaee04a1a2240c37" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "cposs2ukstu_blue_002_001.fits", + "sizeBytes": 53173440, + "contentId": "sha256:71a38c4f837b6f6acdef42a8d1d97ed38809893763170e06eca3f45b9c1a35a4" + }, + { + "path": "cposs2ukstu_blue_002_001_area.fits", + "sizeBytes": 53173440, + "contentId": "sha256:3f1e2441d8467e04aadb1a901894e9cf4eed0c548a51fcaaf442219dbda784f0" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "cposs2ukstu_blue_002_002.fits", + "sizeBytes": 53173440, + "contentId": "sha256:9567a22d11603b8c0c98e17573831299bb0dcc22be2ab02cffa738a66f795ea8" + }, + { + "path": "cposs2ukstu_blue_002_002_area.fits", + "sizeBytes": 53173440, + "contentId": "sha256:e32ead58ebd3a89dd970b0fc73776b290d77267d9b4052860d1fa7e2f88fd45a" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "cposs2ukstu_red_001_001.fits", + "sizeBytes": 52536960, + "contentId": "sha256:7d191eac9fbcafc7d22b30eccf6fd71568bfc72fe6c6bb6d30af8030aa0c0fdc" + }, + { + "path": "cposs2ukstu_red_001_001_area.fits", + "sizeBytes": 52536960, + "contentId": "sha256:2dc17ea041bfde20474aabc2c2a84f55fb6901b1810c7bc0ba84dff1a24d9ffa" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "cposs2ukstu_red_001_002.fits", + "sizeBytes": 52557120, + "contentId": "sha256:c4191ea37c711918c911de4af1524696d7adaa78e4998f2153ca2873f2b93508" + }, + { + "path": "cposs2ukstu_red_001_002_area.fits", + "sizeBytes": 52557120, + "contentId": "sha256:e41d0413b616ccda390a95d3dbed1c2004b669873def40290f506cf4d61a3798" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "cposs2ukstu_red_002_001.fits", + "sizeBytes": 52513920, + "contentId": "sha256:6282d94d4426c56298ff96355fee3b35646a19ea429e3ef913b368d70c6e3a27" + }, + { + "path": "cposs2ukstu_red_002_001_area.fits", + "sizeBytes": 52513920, + "contentId": "sha256:b0ea648d5ec32fab135571d3fa55204d3bff8686bd02430cdf19cc5087d89556" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "cposs2ukstu_red_002_002.fits", + "sizeBytes": 52536960, + "contentId": "sha256:9af79de619227b9327152da9dfd9a6d06343eec0d48bd0fd14fb7613ea6e27fd" + }, + { + "path": "cposs2ukstu_red_002_002_area.fits", + "sizeBytes": 52536960, + "contentId": "sha256:5d5084a21330ad1e8dbb24ee541c36713008c8d062a8011c2d8d0dd64bfd98c3" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "cposs2ukstu_ir_001_001.fits", + "sizeBytes": 53009280, + "contentId": "sha256:03c9cdd701a0c07a900601a6de714d73ad20df2fce47f599e25f8c66fd405943" + }, + { + "path": "cposs2ukstu_ir_001_001_area.fits", + "sizeBytes": 53009280, + "contentId": "sha256:ae39747027dc42a90596c7c89af1359b1e03474873e77643a9a24c9e9ce442bf" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "cposs2ukstu_ir_001_002.fits", + "sizeBytes": 53029440, + "contentId": "sha256:f9dd66c75385833a94f94dc6fb95ddf0e8ac13caddf7a874eba6d028e439be3d" + }, + { + "path": "cposs2ukstu_ir_001_002_area.fits", + "sizeBytes": 53029440, + "contentId": "sha256:be1461abf377eff28ee293f75e46108471d440e2d3da1b4670eccf6ca30ff70a" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "cposs2ukstu_ir_002_001.fits", + "sizeBytes": 53029440, + "contentId": "sha256:7bd5196b17210e40636b701366e403d86412d8b73e2fac9f8cdf192d811f1575" + }, + { + "path": "cposs2ukstu_ir_002_001_area.fits", + "sizeBytes": 53029440, + "contentId": "sha256:bc0665a46216ffd9456671a46d4ca495c1a11dc83de77e3e7048ba48ae4dfd9d" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "cposs2ukstu_ir_002_002.fits", + "sizeBytes": 53009280, + "contentId": "sha256:0832d94f62cba5175b33fedadd2f08981a86db87ccf987a0d6d4c05b68ecba3a" + }, + { + "path": "cposs2ukstu_ir_002_002_area.fits", + "sizeBytes": 53009280, + "contentId": "sha256:8ab9c50dd73ee761e15cf1e7c36a02bb32cb4d98b81f058264f0418bffcd67a4" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "1-corrections.tbl", + "sizeBytes": 276, + "contentId": "sha256:158d6e680abfbabbf777b69cc97eca2cd4c2306e5bae826f0ef38327ea37d363" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "2-corrections.tbl", + "sizeBytes": 276, + "contentId": "sha256:e047834476d76b7c4c3926a2af9e87b482962f2f0906f4b2cbc6c32305cadd55" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "3-corrections.tbl", + "sizeBytes": 276, + "contentId": "sha256:089005eb8651264f18c51011237582b0300ed8841dfeb7c2f53148b45a6e05b5" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "1-fits.tbl", + "sizeBytes": 1457, + "contentId": "sha256:154f992c09503d6092af14e506fc9a127dd93cb8598622ae2da8015c700c4850" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "2-fits.tbl", + "sizeBytes": 1457, + "contentId": "sha256:59474ddb76094bef5dbcf0e82c53fc6d40b8b01a05e9c1306cc8694740d77ad0" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "3-fits.tbl", + "sizeBytes": 1457, + "contentId": "sha256:63c753f41a61538b335c707ce8a06a2f1cd3cdd2b6792a72a4c0a4d6c4fda56e" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000005": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "1-diff.000001.000002.fits", + "sizeBytes": 15796800, + "contentId": "sha256:8c6bff04264dcab637ed9278945dd41169655c428485059f4ea7f724c697cd73" + }, + { + "path": "1-diff.000001.000002_area.fits", + "sizeBytes": 15796800, + "contentId": "sha256:cf4bf661a2d77de0e9da5754651a75621a44865c95e72d08ccad1a66cfbfadde" + }, + { + "path": "1-fit.000001.000002.txt", + "sizeBytes": 286, + "contentId": "sha256:b5a68a8d9f6c7998cd29a61561b07535d0e44d3969755e1996d07e496133e4ba" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000006": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "1-diff.000001.000003.fits", + "sizeBytes": 15816960, + "contentId": "sha256:943d001e1d7731d1306baa7912b1ca7b96c36a280352eeea23b9d6b5555b9d0e" + }, + { + "path": "1-diff.000001.000003_area.fits", + "sizeBytes": 15816960, + "contentId": "sha256:fbe0ab009e20bd5d1a200a659e606c6a9ab9ccecd94a499b34008617ed379596" + }, + { + "path": "1-fit.000001.000003.txt", + "sizeBytes": 290, + "contentId": "sha256:e71185716c537fd1f358074627f468c6b9b017d34274c684c2fac76516dd9797" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000007": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "1-diff.000001.000004.fits", + "sizeBytes": 4622400, + "contentId": "sha256:f2c146f620676ecca0b2fc6cb99506efd88c1b73385c1fdf1e421771e520cb37" + }, + { + "path": "1-diff.000001.000004_area.fits", + "sizeBytes": 4622400, + "contentId": "sha256:e810370b8d38eebecae6ba5bcdb640d63119c771085dcba1081d60cbf3340cdc" + }, + { + "path": "1-fit.000001.000004.txt", + "sizeBytes": 283, + "contentId": "sha256:6d752726c646970350ea7b4139a5ff6e26131381807a701f9dff1bae0e99b02f" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000008": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "1-diff.000002.000003.fits", + "sizeBytes": 4628160, + "contentId": "sha256:bce19f0facc5eccfe7da14273c0ef35805f1ed8e6cf016103dcf96c01b81c48b" + }, + { + "path": "1-diff.000002.000003_area.fits", + "sizeBytes": 4628160, + "contentId": "sha256:685b3e3308671b3b957f9a7009ce82118699989364b8e4d7070e8e5644084a04" + }, + { + "path": "1-fit.000002.000003.txt", + "sizeBytes": 284, + "contentId": "sha256:1109f42a4bf85bb3625f7710a292e300f6aa7c67a528a1557c4aaca3340ee6cd" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000009": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "1-diff.000002.000004.fits", + "sizeBytes": 9213120, + "contentId": "sha256:06930349dfac879ce216815077e2b299591f014779dabdb5dbc961bace35bfdf" + }, + { + "path": "1-diff.000002.000004_area.fits", + "sizeBytes": 9213120, + "contentId": "sha256:36134255c25961247ecf638a5eda9884ad47be764193f4b0dfab09c41cd104e6" + }, + { + "path": "1-fit.000002.000004.txt", + "sizeBytes": 283, + "contentId": "sha256:57db0f74d87f07b54f670bc008931680f2d38bce5f37c3f5d7a280ab70489f67" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000010": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "1-diff.000003.000004.fits", + "sizeBytes": 9207360, + "contentId": "sha256:7b14d04c5411a245e3dada61f987a4f191e98dccf926fee92885bc9f70090c4a" + }, + { + "path": "1-diff.000003.000004_area.fits", + "sizeBytes": 9207360, + "contentId": "sha256:f830b0e68bc9f5729be28945b88b47ce07c7fe6139cbf0978e928263d01211d1" + }, + { + "path": "1-fit.000003.000004.txt", + "sizeBytes": 285, + "contentId": "sha256:70505a5fb45ce8ffd08d868cff0e600ce3f3e5925d3ca9b206262eee8d811995" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000024": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "2-diff.000001.000002.fits", + "sizeBytes": 15528960, + "contentId": "sha256:e7a4912f1906cc229f06dd4febc193a59b8854ea490b9ece6242dc0b1634834b" + }, + { + "path": "2-diff.000001.000002_area.fits", + "sizeBytes": 15528960, + "contentId": "sha256:ab9cb15ef4d5d55739f03df95fb85d1d7ca7694c05a29ad81feddc73ecc19627" + }, + { + "path": "2-fit.000001.000002.txt", + "sizeBytes": 291, + "contentId": "sha256:542c6545c0cb4cb51e99b15b0211402d2ef7c544527092c304d8d6531aa1c741" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000025": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "2-diff.000001.000003.fits", + "sizeBytes": 15569280, + "contentId": "sha256:f294ae4035f1023ff3afa0d37c49108360120a201717ae116f63c2822870bf08" + }, + { + "path": "2-diff.000001.000003_area.fits", + "sizeBytes": 15569280, + "contentId": "sha256:b599939517e6d2adf913f18cedfc8d13007499e7697dfb059ad6575e12834452" + }, + { + "path": "2-fit.000001.000003.txt", + "sizeBytes": 288, + "contentId": "sha256:5634c3006eaa9029b62223a34c727969c59f43e883d0fa112128e8346d3ceffe" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000026": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "2-diff.000001.000004.fits", + "sizeBytes": 4570560, + "contentId": "sha256:7a6a0e714d6a335dcd31c83b6f8783c25ded4fd7a8ec98901fde53530061068a" + }, + { + "path": "2-diff.000001.000004_area.fits", + "sizeBytes": 4570560, + "contentId": "sha256:5c88047807ad0d8c2e3da80871341df22c751caf865b75127b9e6710559addfe" + }, + { + "path": "2-fit.000001.000004.txt", + "sizeBytes": 280, + "contentId": "sha256:d4045eadf80246d8962771ff22f0a038692334fb0a9e2272ca18adee499b18f3" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000027": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "2-diff.000002.000003.fits", + "sizeBytes": 4587840, + "contentId": "sha256:a7775d2361ea0b9033e8c3447b3a1bd16f611388ef938ccb75330e01954cf7c1" + }, + { + "path": "2-diff.000002.000003_area.fits", + "sizeBytes": 4587840, + "contentId": "sha256:82713e6aaf1d2ba38d807491fd6d36518122bb38d08f5c94725e030bec2e17e2" + }, + { + "path": "2-fit.000002.000003.txt", + "sizeBytes": 281, + "contentId": "sha256:17ad134f9a5fb1ccf474532fa097737fd7fabb3e5ad19bba6f4c5f2490df77dc" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000028": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "2-diff.000002.000004.fits", + "sizeBytes": 9135360, + "contentId": "sha256:9c43fb9937c4e29783b78a7c4e2ca4396f18a345ed7946ed4cb08492547dcd9d" + }, + { + "path": "2-diff.000002.000004_area.fits", + "sizeBytes": 9135360, + "contentId": "sha256:b1aed9d6343b9694d6afb7c556bf935ad40db741b085692d14e6f980e915b437" + }, + { + "path": "2-fit.000002.000004.txt", + "sizeBytes": 284, + "contentId": "sha256:e334047be840a321918420cafc80f51d1480d356ad9fb2efed9083de53a5f852" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000029": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "2-diff.000003.000004.fits", + "sizeBytes": 9118080, + "contentId": "sha256:ac72260481e182f8df56455a011dea8dad18a2ed9ba790d9c090dc2d1482cb58" + }, + { + "path": "2-diff.000003.000004_area.fits", + "sizeBytes": 9118080, + "contentId": "sha256:6a75af01d6f8ffd8539160667d92991b40cfe7c91c80f8035e809d4d46920ea7" + }, + { + "path": "2-fit.000003.000004.txt", + "sizeBytes": 283, + "contentId": "sha256:14a841ce9ff5dbfd4c0c590f8aabe23d06dce08e2968a4f847f0262cc1356c2c" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000043": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "3-diff.000001.000002.fits", + "sizeBytes": 15747840, + "contentId": "sha256:adc9c78828bb0307f9f47c542d6977d80d9c399995915b10970469376d5d8ccb" + }, + { + "path": "3-diff.000001.000002_area.fits", + "sizeBytes": 15747840, + "contentId": "sha256:8c0cc2f2aefcb8a3b6e7dc1961e3b114cf621687f8db7dbd5291b93331add4dc" + }, + { + "path": "3-fit.000001.000002.txt", + "sizeBytes": 288, + "contentId": "sha256:3e4555db8eaf92a9494191c27047adac7303895a2e1dec902fb19a822332b57c" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000044": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "3-diff.000001.000003.fits", + "sizeBytes": 15782400, + "contentId": "sha256:84765f9f66dc7bdbaac73ab083813ae8ae89f135bcfb2d586002c384134dcf8e" + }, + { + "path": "3-diff.000001.000003_area.fits", + "sizeBytes": 15782400, + "contentId": "sha256:b21fa8d40ca5042d2c0fc75f6f451577e0fe91d1e7cb167988692780e4d1d9d2" + }, + { + "path": "3-fit.000001.000003.txt", + "sizeBytes": 288, + "contentId": "sha256:9417d7390ad9dacd8429c4eb9e5f9bdc3e13608778c68cfda9dc65a073340321" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000045": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "3-diff.000001.000004.fits", + "sizeBytes": 4616640, + "contentId": "sha256:c03c75e4dd9f64ee9cbd921f137119b62f5fdcc8b4cb9b3a4085eff460d0c3f8" + }, + { + "path": "3-diff.000001.000004_area.fits", + "sizeBytes": 4616640, + "contentId": "sha256:b442985b4328251d3b38fbb19b5320bda0a444f37e8559a7237beb6a8d1a7135" + }, + { + "path": "3-fit.000001.000004.txt", + "sizeBytes": 283, + "contentId": "sha256:d4000f53c9da9699b429328f5aede0b075f47f7368f3fe45a42e69a395b564e6" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000046": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "3-diff.000002.000003.fits", + "sizeBytes": 4628160, + "contentId": "sha256:9939c6963adefe007241ef6111dca97735405e2415601dce07a0c1415d55ec18" + }, + { + "path": "3-diff.000002.000003_area.fits", + "sizeBytes": 4628160, + "contentId": "sha256:54c591516b5be2791b93ecc81793201d029d60f6990ab1610be8d9622c04ff1f" + }, + { + "path": "3-fit.000002.000003.txt", + "sizeBytes": 279, + "contentId": "sha256:70ac1d436cf86199f2eea3d498d91b0fbd38c53a1d535f3b7843f592ab810726" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000047": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "3-diff.000002.000004.fits", + "sizeBytes": 9201600, + "contentId": "sha256:ee6a7331ff93197c6605d5264d97b0f11a63a48c1be7f557b160922ba3500b02" + }, + { + "path": "3-diff.000002.000004_area.fits", + "sizeBytes": 9201600, + "contentId": "sha256:50b1bd1d961aa96ee4ec0df82af3d69ba0dd0ad64b094e080c3a23e7ff61920c" + }, + { + "path": "3-fit.000002.000004.txt", + "sizeBytes": 284, + "contentId": "sha256:217ee95db242500b96ffbba1964c7b415dfa4b5297254fb0fff0384e27bba5ef" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000048": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "3-diff.000003.000004.fits", + "sizeBytes": 9195840, + "contentId": "sha256:34ea44ed7851f7aee357dc3386b74e1bc485633cd86fd265e8fad02043f6c22f" + }, + { + "path": "3-diff.000003.000004_area.fits", + "sizeBytes": 9195840, + "contentId": "sha256:53633221e71fe7973f49beba4f72fea1f4bf1ce25bddae02a2c3d7d9558ac595" + }, + { + "path": "3-fit.000003.000004.txt", + "sizeBytes": 285, + "contentId": "sha256:41e20488381492d4fa220be79ef9bf81e482c7902c06004e245faad46355219c" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mimgtblid0000017": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "1-updated-corrected.tbl", + "sizeBytes": 2606, + "contentId": "sha256:fc1cd87713e1ce3ed80cee8ffbd902dc41b205da471afef0f50738fe65c6be35" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mimgtblid0000036": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "2-updated-corrected.tbl", + "sizeBytes": 2600, + "contentId": "sha256:8dbb346fe2f398abc1a9e52bbaec523b8c2e39424b7660ef02e0932322748dc3" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mimgtblid0000055": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "3-updated-corrected.tbl", + "sizeBytes": 2594, + "contentId": "sha256:8802f70abdb5f47ea64cd593362801b120adac8b22127d0baf3047260b55cc7e" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000001": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "outputs": [ + { + "path": "pposs2ukstu_blue_001_001.fits", + "sizeBytes": 53193600, + "contentId": "sha256:ec2bf0002e132ca80cd10605e058aa4271cbcbb3a1a72e15acf45c584382e69a" + }, + { + "path": "pposs2ukstu_blue_001_001_area.fits", + "sizeBytes": 53193600, + "contentId": "sha256:ce2f3af33a9bb12b443eeef4f0f5fb40543c32883bcb62076a76e133092b9258" + } + ], + "removedPaths": [ + + ], + "inputsByResourceType": { + "cloud_vm": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "hpc_machine": [ + + ] + } + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000002": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "outputs": [ + { + "path": "pposs2ukstu_blue_001_002.fits", + "sizeBytes": 53173440, + "contentId": "sha256:75266ba7896de92e1402fba78c07cc40bedaace51c4949524af2aa5fb3509936" + }, + { + "path": "pposs2ukstu_blue_001_002_area.fits", + "sizeBytes": 53173440, + "contentId": "sha256:baa881cf48b44433ec2a4ad756d3a9288f2ee2d43ffda273aaee04a1a2240c37" + } + ], + "removedPaths": [ + + ], + "inputsByResourceType": { + "cloud_vm": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "hpc_machine": [ + + ] + } + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000003": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "outputs": [ + { + "path": "pposs2ukstu_blue_002_001.fits", + "sizeBytes": 53173440, + "contentId": "sha256:609732a1e2502e5263aa625ef524a7107f0f85d3237e4603bfb953b05636bab2" + }, + { + "path": "pposs2ukstu_blue_002_001_area.fits", + "sizeBytes": 53173440, + "contentId": "sha256:3f1e2441d8467e04aadb1a901894e9cf4eed0c548a51fcaaf442219dbda784f0" + } + ], + "removedPaths": [ + + ], + "inputsByResourceType": { + "cloud_vm": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "hpc_machine": [ + + ] + } + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000004": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "outputs": [ + { + "path": "pposs2ukstu_blue_002_002.fits", + "sizeBytes": 53173440, + "contentId": "sha256:899094163675e42718fdfb5b76c06258d3600238b84e8d6f0ef184a435e35abe" + }, + { + "path": "pposs2ukstu_blue_002_002_area.fits", + "sizeBytes": 53173440, + "contentId": "sha256:e32ead58ebd3a89dd970b0fc73776b290d77267d9b4052860d1fa7e2f88fd45a" + } + ], + "removedPaths": [ + + ], + "inputsByResourceType": { + "cloud_vm": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "hpc_machine": [ + + ] + } + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000020": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "outputs": [ + { + "path": "pposs2ukstu_red_001_001.fits", + "sizeBytes": 52536960, + "contentId": "sha256:b0a16774b8c6744534cf78e6e6b853d0293de06f7b56cab2deb69c1401aa344f" + }, + { + "path": "pposs2ukstu_red_001_001_area.fits", + "sizeBytes": 52536960, + "contentId": "sha256:2dc17ea041bfde20474aabc2c2a84f55fb6901b1810c7bc0ba84dff1a24d9ffa" + } + ], + "removedPaths": [ + + ], + "inputsByResourceType": { + "cloud_vm": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "hpc_machine": [ + + ] + } + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000021": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "outputs": [ + { + "path": "pposs2ukstu_red_001_002.fits", + "sizeBytes": 52557120, + "contentId": "sha256:a6de5f6b873acba801507eb2f8012c61e8a5403d8c513e3cee4f1d089aae6548" + }, + { + "path": "pposs2ukstu_red_001_002_area.fits", + "sizeBytes": 52557120, + "contentId": "sha256:e41d0413b616ccda390a95d3dbed1c2004b669873def40290f506cf4d61a3798" + } + ], + "removedPaths": [ + + ], + "inputsByResourceType": { + "cloud_vm": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "hpc_machine": [ + + ] + } + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000022": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "outputs": [ + { + "path": "pposs2ukstu_red_002_001.fits", + "sizeBytes": 52513920, + "contentId": "sha256:c2286fb0e9f3e64998431968712050dd0b4709aa83ae98eaf5feb44c050aed71" + }, + { + "path": "pposs2ukstu_red_002_001_area.fits", + "sizeBytes": 52513920, + "contentId": "sha256:b0ea648d5ec32fab135571d3fa55204d3bff8686bd02430cdf19cc5087d89556" + } + ], + "removedPaths": [ + + ], + "inputsByResourceType": { + "cloud_vm": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "hpc_machine": [ + + ] + } + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000023": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "outputs": [ + { + "path": "pposs2ukstu_red_002_002.fits", + "sizeBytes": 52536960, + "contentId": "sha256:6b70f22effdb4416be395416191134e46ce69959f2ab83e0f23b5c9b454c425e" + }, + { + "path": "pposs2ukstu_red_002_002_area.fits", + "sizeBytes": 52536960, + "contentId": "sha256:5d5084a21330ad1e8dbb24ee541c36713008c8d062a8011c2d8d0dd64bfd98c3" + } + ], + "removedPaths": [ + + ], + "inputsByResourceType": { + "cloud_vm": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "hpc_machine": [ + + ] + } + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000039": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "outputs": [ + { + "path": "pposs2ukstu_ir_001_001.fits", + "sizeBytes": 53009280, + "contentId": "sha256:de19eaf066d0bbbd75c513d658ea98fa596a70fdb2a3c049c78b17ce0e6148a5" + }, + { + "path": "pposs2ukstu_ir_001_001_area.fits", + "sizeBytes": 53009280, + "contentId": "sha256:ae39747027dc42a90596c7c89af1359b1e03474873e77643a9a24c9e9ce442bf" + } + ], + "removedPaths": [ + + ], + "inputsByResourceType": { + "cloud_vm": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "hpc_machine": [ + + ] + } + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000040": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "outputs": [ + { + "path": "pposs2ukstu_ir_001_002.fits", + "sizeBytes": 53029440, + "contentId": "sha256:6b08934fd323f5d13d4574820236d35407983d8c6da296fc4ef4e16b1b507614" + }, + { + "path": "pposs2ukstu_ir_001_002_area.fits", + "sizeBytes": 53029440, + "contentId": "sha256:be1461abf377eff28ee293f75e46108471d440e2d3da1b4670eccf6ca30ff70a" + } + ], + "removedPaths": [ + + ], + "inputsByResourceType": { + "cloud_vm": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "hpc_machine": [ + + ] + } + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000041": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "outputs": [ + { + "path": "pposs2ukstu_ir_002_001.fits", + "sizeBytes": 53029440, + "contentId": "sha256:adc3094a01ee350260dde0bccaccf8e02b79e5e86b5c2dc46b63bb3e6fb2c319" + }, + { + "path": "pposs2ukstu_ir_002_001_area.fits", + "sizeBytes": 53029440, + "contentId": "sha256:bc0665a46216ffd9456671a46d4ca495c1a11dc83de77e3e7048ba48ae4dfd9d" + } + ], + "removedPaths": [ + + ], + "inputsByResourceType": { + "cloud_vm": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "hpc_machine": [ + + ] + } + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000042": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "outputs": [ + { + "path": "pposs2ukstu_ir_002_002.fits", + "sizeBytes": 53009280, + "contentId": "sha256:efce4975948c7c15a4199d1b6c3d54f0e6b773d5f36ed4d8a09d585c1b3771e5" + }, + { + "path": "pposs2ukstu_ir_002_002_area.fits", + "sizeBytes": 53009280, + "contentId": "sha256:8ab9c50dd73ee761e15cf1e7c36a02bb32cb4d98b81f058264f0418bffcd67a4" + } + ], + "removedPaths": [ + + ], + "inputsByResourceType": { + "cloud_vm": [ + { + "path": "1-corrected.tbl", + "sizeBytes": 952, + "contentId": "sha256:6dbcaac981c043a607e25f1cf7b2584a47482d52f51910611f57e3668c968b1d" + }, + { + "path": "1-diffs.tbl", + "sizeBytes": 816, + "contentId": "sha256:0f6112e4b85271a65dae65e332a2d4d58d6f9a431bff5442a131928815f5688f" + }, + { + "path": "1-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:c7abba1727d9d1e68e97047aa42067e5270a22446ade80dc0346d6ffb7085466" + }, + { + "path": "1-projected.tbl", + "sizeBytes": 952, + "contentId": "sha256:578ddb3658eedb59337d9a56825f6e5bf8ab599fdda175b58f26c1fb45635167" + }, + { + "path": "1-raw.tbl", + "sizeBytes": 948, + "contentId": "sha256:3a66ad0b3f37c9b4aaa2fea7f24dec86f335f02abd0cbd4396f9990ee55b61c8" + }, + { + "path": "1-stat.tbl", + "sizeBytes": 1200, + "contentId": "sha256:8eb42521ccf00796c0873d67989312b8ce32ab4c2c6355327183c86ffe2a98b8" + }, + { + "path": "2-corrected.tbl", + "sizeBytes": 946, + "contentId": "sha256:06b74a0dcd1e1b619e7f14d80b885e61fc06a3ca84751c02c50d84d612d358d1" + }, + { + "path": "2-diffs.tbl", + "sizeBytes": 800, + "contentId": "sha256:2318157c454db7ddba094915fa42c42f809befba566d4064740fc948b70daf45" + }, + { + "path": "2-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:b6be4998f9001834ea549e655092c9ac6b5c7484bb34c8e94858894e49373508" + }, + { + "path": "2-projected.tbl", + "sizeBytes": 946, + "contentId": "sha256:7c6f65e9a7bd58161dc34310c8b9e5e98e731a5be32aa33763c154fbe24d6f80" + }, + { + "path": "2-raw.tbl", + "sizeBytes": 942, + "contentId": "sha256:05c8b7940d6fa7fff83ca013d64fb8dccf1c41259727a19779325e72a92116a3" + }, + { + "path": "2-stat.tbl", + "sizeBytes": 1180, + "contentId": "sha256:f5526e2d981098ab81784bcd3c298fcf9d32fa3ab5107947c8ce516ac317b998" + }, + { + "path": "3-corrected.tbl", + "sizeBytes": 940, + "contentId": "sha256:9909e186fe2b08e15947159d48a08ee3520eecf32e01d97f1883c994714b7951" + }, + { + "path": "3-diffs.tbl", + "sizeBytes": 784, + "contentId": "sha256:b1c67573706e728c036ac887e774371f3c2f3cfc07ca8e06bcbdcd5a09c45f4d" + }, + { + "path": "3-images.tbl", + "sizeBytes": 2246, + "contentId": "sha256:7ebdd229be45b03482b4fef1098edc4b2f7c384448e682181abb12a69039e540" + }, + { + "path": "3-projected.tbl", + "sizeBytes": 940, + "contentId": "sha256:4fa98024f1a85e26ee4e8254751c4768c7a455cf99c81bfc8b2fd136f4e4981d" + }, + { + "path": "3-raw.tbl", + "sizeBytes": 936, + "contentId": "sha256:b281c5d5b0b0f368e7d95228fe144feba31686c6c312a4a4ab16cd4019a1fc8b" + }, + { + "path": "3-stat.tbl", + "sizeBytes": 1160, + "contentId": "sha256:b950889a89d1e298a5ffdb66650e9ff1cd5a1efa48361ac63e64d5ce13f3f3a2" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_blue_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_ir_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_001_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_001.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "__MACOSX/data-out-0-50d/._poss2ukstu_red_002_002.fits", + "sizeBytes": 163, + "contentId": "sha256:a226d06ffcff45cf2e87ba142cb9d0a2a81659ebc97b468b0a5152b5fcbc7d72" + }, + { + "path": "montage-workflow.yml", + "sizeBytes": 55999, + "contentId": "sha256:eed57440859b6aa7e3929931d08e96dd90a2a46d6a853f8f663837946774cbdb" + }, + { + "path": "poss2ukstu_blue_001_001.fits", + "sizeBytes": 9655401, + "contentId": "sha256:d2b2629d7de19ab483807a7231e75023bbdf4a47905115026fb5898407ddbc14" + }, + { + "path": "poss2ukstu_blue_001_002.fits", + "sizeBytes": 8587351, + "contentId": "sha256:500ec211d0e638e50dc911a1dd81b075c11223d3067410086210948b48481883" + }, + { + "path": "poss2ukstu_blue_002_001.fits", + "sizeBytes": 8814262, + "contentId": "sha256:083ce09fec7f05cafa9d8809c529011c8f9e11151c21c4b7547accd6def7dda0" + }, + { + "path": "poss2ukstu_blue_002_002.fits", + "sizeBytes": 8238776, + "contentId": "sha256:34ebeb66332efc704e88f07215ada93a79c5150b841c968c2a15953bc96a9247" + }, + { + "path": "poss2ukstu_ir_001_001.fits", + "sizeBytes": 7327980, + "contentId": "sha256:93401b425358b80a604d82648f8e0f396acae2cd67fa3f70956b07b2a87c5d21" + }, + { + "path": "poss2ukstu_ir_001_002.fits", + "sizeBytes": 7167256, + "contentId": "sha256:b65cddc600c6f13a2538127f945a18b3baa4b16904b9e74563c69ebd2558226f" + }, + { + "path": "poss2ukstu_ir_002_001.fits", + "sizeBytes": 7217490, + "contentId": "sha256:ef004bd622f6bd6594b914faa538aff8104f33234c0494b60daf65efa887fb97" + }, + { + "path": "poss2ukstu_ir_002_002.fits", + "sizeBytes": 7120717, + "contentId": "sha256:b0382772a213784cac08b5d0a2dd2f4017c04de6327c592e8ef7e16631b61c7f" + }, + { + "path": "poss2ukstu_red_001_001.fits", + "sizeBytes": 6900714, + "contentId": "sha256:7f0bc89a03f6657cd7e352c2eee72465910b79bc7a22b0efffa60f23a873611d" + }, + { + "path": "poss2ukstu_red_001_002.fits", + "sizeBytes": 6494409, + "contentId": "sha256:ba2004aeeec3dbe5848f92c1b822c5d8346ac3e734f269174f44a96742fb3944" + }, + { + "path": "poss2ukstu_red_002_001.fits", + "sizeBytes": 6570387, + "contentId": "sha256:e8d0aa4a8b4659b43a63df37a189872e39e2b3b2179ee0ecdabf43c0c7683fb1" + }, + { + "path": "poss2ukstu_red_002_002.fits", + "sizeBytes": 6376872, + "contentId": "sha256:54546b64cceb235d5691077a279400d46336f06c698c345215c848ae0b98778e" + }, + { + "path": "region-oversized.hdr", + "sizeBytes": 277, + "contentId": "sha256:ab4d0ce04df6238607f0248451c13cc4fc3b8e44705b3a9dc7520f10c8a5996f" + }, + { + "path": "region.hdr", + "sizeBytes": 275, + "contentId": "sha256:209ddd299c1fbb3f711a0f20273fd20d2a9ef9ebbe94069e00fad4b0627359af" + } + ], + "hpc_machine": [ + + ] + } + }, + "montage-58-fog-hpc-e2-20261006-mviewerid0000019": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "1-mosaic.png", + "sizeBytes": 1443126, + "contentId": "sha256:90b53e98f8de8ba50db292fb7727c1698a8f0139ba53ca877f05b34f7e4a75cb" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mviewerid0000038": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "2-mosaic.png", + "sizeBytes": 1850031, + "contentId": "sha256:3ac1c8aee8f6f96813ffa29cddee00d9a632a5626d75feb36853671379d37082" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mviewerid0000057": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "3-mosaic.png", + "sizeBytes": 2142272, + "contentId": "sha256:407f7524a7c3644731f39743990c9debc28f03e1eda4c21545a190a7fead104f" + } + ], + "removedPaths": [ + + ] + }, + "montage-58-fog-hpc-e2-20261006-mviewerid0000058": { + "contract": "whole-workspace-v1", + "source": "baseline:montage-docker-image-real-20261010-v2", + "inputs": [ + + ], + "outputs": [ + { + "path": "mosaic-color.png", + "sizeBytes": 4920451, + "contentId": "sha256:63fa4d8447274bdb8a44f6be8cf5a23c0a2f0baa479bd2ae1ec3b541df4e19f2" + } + ], + "removedPaths": [ + + ] + } + }, + "dependencies": [ + { + "activityId": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-maddid0000018", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-maddid0000037", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-maddid0000056", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000001" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000002" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000003" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000004" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000020" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000021" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000022" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000023" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000039" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000040" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000041" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000042" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000005" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000006" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000007" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000008" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000010" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000024" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000025" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000026" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000027" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000028" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000029" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000043" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000044" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000046" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000047" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000048" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000005", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000001" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000005", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000002" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000006", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000001" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000006", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000003" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000007", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000001" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000007", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000004" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000008", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000002" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000008", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000003" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000002" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000009", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000004" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000010", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000003" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000010", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000004" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000024", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000020" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000024", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000021" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000025", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000020" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000025", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000022" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000026", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000020" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000026", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000023" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000027", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000021" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000027", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000022" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000028", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000021" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000028", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000023" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000029", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000022" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000029", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000023" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000043", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000039" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000043", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000040" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000044", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000039" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000044", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000041" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000039" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000045", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000042" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000046", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000040" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000046", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000041" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000047", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000040" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000047", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000042" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000048", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000041" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mdifffitid0000048", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mprojectid0000042" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mimgtblid0000017", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mimgtblid0000036", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mimgtblid0000055", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mviewerid0000019", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-maddid0000018" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mviewerid0000038", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-maddid0000037" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mviewerid0000057", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-maddid0000056" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mviewerid0000058", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-maddid0000018" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mviewerid0000058", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-maddid0000037" + }, + { + "activityId": "montage-58-fog-hpc-e2-20261006-mviewerid0000058", + "dependsOnActivityId": "montage-58-fog-hpc-e2-20261006-maddid0000056" + } + ], + "observedProducerWorkspaceBytes": { + "montage-58-fog-hpc-e2-20261006-maddid0000018": 1111828392, + "montage-58-fog-hpc-e2-20261006-maddid0000037": 1100003102, + "montage-58-fog-hpc-e2-20261006-maddid0000056": 1109057816, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013": 740939386, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014": 740899066, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015": 740899066, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016": 740899066, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032": 732938742, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033": 732979062, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034": 732892662, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035": 732938742, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051": 739073142, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052": 739113462, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053": 739113462, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054": 739073142, + "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012": 634552186, + "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031": 627864822, + "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050": 633054582, + "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011": 634551910, + "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030": 627864546, + "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049": 633054306, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000005": 334879268, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000006": 334919592, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000007": 312530465, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000008": 312501666, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000009": 321671585, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000010": 321660067, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000024": 331797673, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000025": 331791910, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000026": 309840542, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000027": 309869343, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000028": 319010466, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000029": 318889505, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000043": 334124710, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000044": 334193830, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000045": 221270683, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000046": 311925661, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000047": 321032226, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000048": 321020707, + "montage-58-fog-hpc-e2-20261006-mimgtblid0000017": 1059982632, + "montage-58-fog-hpc-e2-20261006-mimgtblid0000036": 1048157342, + "montage-58-fog-hpc-e2-20261006-mimgtblid0000055": 1057212056, + "montage-58-fog-hpc-e2-20261006-mprojectid0000001": 196938502, + "montage-58-fog-hpc-e2-20261006-mprojectid0000002": 196898182, + "montage-58-fog-hpc-e2-20261006-mprojectid0000003": 196898182, + "montage-58-fog-hpc-e2-20261006-mprojectid0000004": 196898182, + "montage-58-fog-hpc-e2-20261006-mprojectid0000020": 195625222, + "montage-58-fog-hpc-e2-20261006-mprojectid0000021": 195665542, + "montage-58-fog-hpc-e2-20261006-mprojectid0000022": 195579142, + "montage-58-fog-hpc-e2-20261006-mprojectid0000023": 195625222, + "montage-58-fog-hpc-e2-20261006-mprojectid0000039": 106018560, + "montage-58-fog-hpc-e2-20261006-mprojectid0000040": 196610182, + "montage-58-fog-hpc-e2-20261006-mprojectid0000041": 196610182, + "montage-58-fog-hpc-e2-20261006-mprojectid0000042": 106018560, + "montage-58-fog-hpc-e2-20261006-mviewerid0000019": 1113271518, + "montage-58-fog-hpc-e2-20261006-mviewerid0000038": 1101853133, + "montage-58-fog-hpc-e2-20261006-mviewerid0000057": 1111200088, + "montage-58-fog-hpc-e2-20261006-mviewerid0000058": 3144707157 + }, + "observedConsumerInputVolumes": { + "montage-58-fog-hpc-e2-20261006-maddid0000018": { + "inputTotalBytes": 4023619216, + "inputUniqueBytes": 1059982632, + "inputFiles": 82 + }, + "montage-58-fog-hpc-e2-20261006-maddid0000037": { + "inputTotalBytes": 3979906550, + "inputUniqueBytes": 1048157342, + "inputFiles": 82 + }, + "montage-58-fog-hpc-e2-20261006-maddid0000056": { + "inputTotalBytes": 4013585264, + "inputUniqueBytes": 1057212056, + "inputFiles": 82 + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013": { + "inputTotalBytes": 831490688, + "inputUniqueBytes": 634552186, + "inputFiles": 73 + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014": { + "inputTotalBytes": 831450368, + "inputUniqueBytes": 634552186, + "inputFiles": 73 + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015": { + "inputTotalBytes": 831450368, + "inputUniqueBytes": 634552186, + "inputFiles": 73 + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016": { + "inputTotalBytes": 831450368, + "inputUniqueBytes": 634552186, + "inputFiles": 73 + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032": { + "inputTotalBytes": 823490044, + "inputUniqueBytes": 627864822, + "inputFiles": 73 + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033": { + "inputTotalBytes": 823530364, + "inputUniqueBytes": 627864822, + "inputFiles": 73 + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034": { + "inputTotalBytes": 823443964, + "inputUniqueBytes": 627864822, + "inputFiles": 73 + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035": { + "inputTotalBytes": 823490044, + "inputUniqueBytes": 627864822, + "inputFiles": 73 + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051": { + "inputTotalBytes": 739073142, + "inputUniqueBytes": 633054582, + "inputFiles": 73 + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052": { + "inputTotalBytes": 829664764, + "inputUniqueBytes": 633054582, + "inputFiles": 73 + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053": { + "inputTotalBytes": 829664764, + "inputUniqueBytes": 633054582, + "inputFiles": 73 + }, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054": { + "inputTotalBytes": 739073142, + "inputUniqueBytes": 633054582, + "inputFiles": 73 + }, + "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012": { + "inputTotalBytes": 634551910, + "inputUniqueBytes": 634551910, + "inputFiles": 72 + }, + "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031": { + "inputTotalBytes": 627864546, + "inputUniqueBytes": 627864546, + "inputFiles": 72 + }, + "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050": { + "inputTotalBytes": 633054306, + "inputUniqueBytes": 633054306, + "inputFiles": 72 + }, + "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011": { + "inputTotalBytes": 1938162643, + "inputUniqueBytes": 634550453, + "inputFiles": 71 + }, + "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030": { + "inputTotalBytes": 1921199439, + "inputUniqueBytes": 627863089, + "inputFiles": 71 + }, + "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049": { + "inputTotalBytes": 1843567817, + "inputUniqueBytes": 633052849, + "inputFiles": 71 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000005": { + "inputTotalBytes": 393836684, + "inputUniqueBytes": 303285382, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000006": { + "inputTotalBytes": 393836684, + "inputUniqueBytes": 303285382, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000007": { + "inputTotalBytes": 393836684, + "inputUniqueBytes": 303285382, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000008": { + "inputTotalBytes": 393796364, + "inputUniqueBytes": 303245062, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000009": { + "inputTotalBytes": 393796364, + "inputUniqueBytes": 303245062, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000010": { + "inputTotalBytes": 393796364, + "inputUniqueBytes": 303245062, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000024": { + "inputTotalBytes": 391290764, + "inputUniqueBytes": 300739462, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000025": { + "inputTotalBytes": 391204364, + "inputUniqueBytes": 300653062, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000026": { + "inputTotalBytes": 391250444, + "inputUniqueBytes": 300699142, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000027": { + "inputTotalBytes": 391244684, + "inputUniqueBytes": 300693382, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000028": { + "inputTotalBytes": 391290764, + "inputUniqueBytes": 300739462, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000029": { + "inputTotalBytes": 391204364, + "inputUniqueBytes": 300653062, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000043": { + "inputTotalBytes": 302628742, + "inputUniqueBytes": 302628742, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000044": { + "inputTotalBytes": 302628742, + "inputUniqueBytes": 302628742, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000045": { + "inputTotalBytes": 212037120, + "inputUniqueBytes": 212037120, + "inputFiles": 4 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000046": { + "inputTotalBytes": 393220364, + "inputUniqueBytes": 302669062, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000047": { + "inputTotalBytes": 302628742, + "inputUniqueBytes": 302628742, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000048": { + "inputTotalBytes": 302628742, + "inputUniqueBytes": 302628742, + "inputFiles": 49 + }, + "montage-58-fog-hpc-e2-20261006-mimgtblid0000017": { + "inputTotalBytes": 2963636584, + "inputUniqueBytes": 1059980026, + "inputFiles": 81 + }, + "montage-58-fog-hpc-e2-20261006-mimgtblid0000036": { + "inputTotalBytes": 2931749208, + "inputUniqueBytes": 1048154742, + "inputFiles": 81 + }, + "montage-58-fog-hpc-e2-20261006-mimgtblid0000055": { + "inputTotalBytes": 2956373208, + "inputUniqueBytes": 1057209462, + "inputFiles": 81 + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000001": { + "inputTotalBytes": 0, + "inputUniqueBytes": 0, + "inputFiles": 0 + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000002": { + "inputTotalBytes": 0, + "inputUniqueBytes": 0, + "inputFiles": 0 + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000003": { + "inputTotalBytes": 0, + "inputUniqueBytes": 0, + "inputFiles": 0 + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000004": { + "inputTotalBytes": 0, + "inputUniqueBytes": 0, + "inputFiles": 0 + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000020": { + "inputTotalBytes": 0, + "inputUniqueBytes": 0, + "inputFiles": 0 + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000021": { + "inputTotalBytes": 0, + "inputUniqueBytes": 0, + "inputFiles": 0 + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000022": { + "inputTotalBytes": 0, + "inputUniqueBytes": 0, + "inputFiles": 0 + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000023": { + "inputTotalBytes": 0, + "inputUniqueBytes": 0, + "inputFiles": 0 + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000039": { + "inputTotalBytes": 0, + "inputUniqueBytes": 0, + "inputFiles": 0 + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000040": { + "inputTotalBytes": 0, + "inputUniqueBytes": 0, + "inputFiles": 0 + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000041": { + "inputTotalBytes": 0, + "inputUniqueBytes": 0, + "inputFiles": 0 + }, + "montage-58-fog-hpc-e2-20261006-mprojectid0000042": { + "inputTotalBytes": 0, + "inputUniqueBytes": 0, + "inputFiles": 0 + }, + "montage-58-fog-hpc-e2-20261006-mviewerid0000019": { + "inputTotalBytes": 1111828392, + "inputUniqueBytes": 1111828392, + "inputFiles": 84 + }, + "montage-58-fog-hpc-e2-20261006-mviewerid0000038": { + "inputTotalBytes": 1100003102, + "inputUniqueBytes": 1100003102, + "inputFiles": 84 + }, + "montage-58-fog-hpc-e2-20261006-mviewerid0000057": { + "inputTotalBytes": 1109057816, + "inputUniqueBytes": 1109057816, + "inputFiles": 84 + }, + "montage-58-fog-hpc-e2-20261006-mviewerid0000058": { + "inputTotalBytes": 3320889310, + "inputUniqueBytes": 3139786706, + "inputFiles": 162 + } + }, + "baselineAssignedResourceTypes": { + "montage-58-fog-hpc-e2-20261006-mprojectid0000042": "hpc_machine", + "montage-58-fog-hpc-e2-20261006-mprojectid0000039": "hpc_machine", + "montage-58-fog-hpc-e2-20261006-mprojectid0000001": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mprojectid0000040": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mprojectid0000004": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mprojectid0000041": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mprojectid0000003": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mprojectid0000020": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mprojectid0000002": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mprojectid0000021": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000005": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000006": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000007": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000010": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mprojectid0000023": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mprojectid0000022": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000024": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000028": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000025": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000027": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000026": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000008": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000048": "hpc_machine", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000045": "hpc_machine", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000046": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000047": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000043": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000044": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000029": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mimgtblid0000036": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-maddid0000037": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mviewerid0000038": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mdifffitid0000009": "hpc_machine", + "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011": "hpc_machine", + "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015": "hpc_machine", + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051": "hpc_machine", + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mimgtblid0000055": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-maddid0000056": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mviewerid0000057": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mimgtblid0000017": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-maddid0000018": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mviewerid0000058": "cloud_vm", + "montage-58-fog-hpc-e2-20261006-mviewerid0000019": "cloud_vm" + }, + "activityReferenceDurations": { + "montage-58-fog-hpc-e2-20261006-maddid0000018": 9, + "montage-58-fog-hpc-e2-20261006-maddid0000037": 9, + "montage-58-fog-hpc-e2-20261006-maddid0000056": 2, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000013": 1, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000014": 1, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000015": 1, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000016": 1, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000032": 9, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000033": 25, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000034": 23, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000035": 9, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000051": 16, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000052": 9, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000053": 12, + "montage-58-fog-hpc-e2-20261006-mbackgroundid0000054": 12, + "montage-58-fog-hpc-e2-20261006-mbgmodelid0000012": 9, + "montage-58-fog-hpc-e2-20261006-mbgmodelid0000031": 7, + "montage-58-fog-hpc-e2-20261006-mbgmodelid0000050": 20, + "montage-58-fog-hpc-e2-20261006-mconcatfitid0000011": 17, + "montage-58-fog-hpc-e2-20261006-mconcatfitid0000030": 9, + "montage-58-fog-hpc-e2-20261006-mconcatfitid0000049": 5, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000005": 30, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000006": 28, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000007": 24, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000008": 9, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000009": 11, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000010": 24, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000024": 16, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000025": 20, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000026": 16, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000027": 15, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000028": 20, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000029": 13, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000043": 11, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000044": 11, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000045": 16, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000046": 14, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000047": 14, + "montage-58-fog-hpc-e2-20261006-mdifffitid0000048": 19, + "montage-58-fog-hpc-e2-20261006-mimgtblid0000017": 22, + "montage-58-fog-hpc-e2-20261006-mimgtblid0000036": 25, + "montage-58-fog-hpc-e2-20261006-mimgtblid0000055": 9, + "montage-58-fog-hpc-e2-20261006-mprojectid0000001": 1667, + "montage-58-fog-hpc-e2-20261006-mprojectid0000002": 414, + "montage-58-fog-hpc-e2-20261006-mprojectid0000003": 550, + "montage-58-fog-hpc-e2-20261006-mprojectid0000004": 824, + "montage-58-fog-hpc-e2-20261006-mprojectid0000020": 528, + "montage-58-fog-hpc-e2-20261006-mprojectid0000021": 361, + "montage-58-fog-hpc-e2-20261006-mprojectid0000022": 1120, + "montage-58-fog-hpc-e2-20261006-mprojectid0000023": 1132, + "montage-58-fog-hpc-e2-20261006-mprojectid0000039": 1540, + "montage-58-fog-hpc-e2-20261006-mprojectid0000040": 1537, + "montage-58-fog-hpc-e2-20261006-mprojectid0000041": 753, + "montage-58-fog-hpc-e2-20261006-mprojectid0000042": 1546, + "montage-58-fog-hpc-e2-20261006-mviewerid0000019": 17, + "montage-58-fog-hpc-e2-20261006-mviewerid0000038": 8, + "montage-58-fog-hpc-e2-20261006-mviewerid0000057": 1, + "montage-58-fog-hpc-e2-20261006-mviewerid0000058": 7 + }, + "limitations": [ + "File sizes and identities are previous-run observations for the same immutable image and dataset, not future observations.", + "Runtime packaging variant is supported only for evidenced cloud_vm and hpc_machine types.", + "No transfer deduplication, snapshot reuse or shared storage is inferred by this manifest model." + ] +} diff --git a/internal/api/handlers/workflow_engine_api_handler/cloud_orphan_test.go b/internal/api/handlers/workflow_engine_api_handler/cloud_orphan_test.go new file mode 100644 index 00000000..30e07fe8 --- /dev/null +++ b/internal/api/handlers/workflow_engine_api_handler/cloud_orphan_test.go @@ -0,0 +1,73 @@ +package workflow_engine_api_handler + +import ( + "context" + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" + queue "github.com/UFFeScience/akoflow/internal/domain/queue" + "net/http/httptest" + "testing" +) + +type orphanOperations struct { + ports.CloudOperationStore + op domain.CloudOperationRun +} + +func (s *orphanOperations) FindCloudOperation(context.Context, string) (*domain.CloudOperationRun, error) { + v := s.op + return &v, nil +} +func (s *orphanOperations) UpdateCloudOperation(_ context.Context, v domain.CloudOperationRun) error { + s.op = v + return nil +} + +type orphanQueue struct { + QueueCancellationStore + job *queue.Job +} + +func (s orphanQueue) FindLatestByAggregate(context.Context, string, string, string) (*queue.Job, error) { + return s.job, nil +} + +type orphanExecution struct { + ExecutionQuery + status domain.ExecutionRunStatus +} + +func (s orphanExecution) FindRun(context.Context, string) (*domain.ExecutionRun, error) { + return &domain.ExecutionRun{Status: s.status}, nil +} + +func TestReconcileOrphanCloudOperationGuards(t *testing.T) { + for _, tc := range []struct { + name string + job *queue.Job + status domain.ExecutionRunStatus + want int + }{ + {"orphan", nil, domain.ExecutionRunFailed, 200}, + {"queue job exists", &queue.Job{}, domain.ExecutionRunFailed, 409}, + {"workflow active", nil, domain.ExecutionRunRunning, 409}, + } { + t.Run(tc.name, func(t *testing.T) { + s := &orphanOperations{op: domain.CloudOperationRun{ID: "orphan", Status: "queued", Kind: "start", ExecutionRunID: "run"}} + h := &Handler{cloudOperations: s, queue: orphanQueue{job: tc.job}, executions: orphanExecution{status: tc.status}} + r := httptest.NewRequest("POST", "/", nil) + r.SetPathValue("operationId", "orphan") + w := httptest.NewRecorder() + h.ReconcileOrphanCloudOperation(w, r) + if w.Code != tc.want { + t.Fatalf("status=%d body=%s", w.Code, w.Body) + } + if tc.want == 200 && (s.op.Status != "cancelled" || s.op.FinishedAt == nil) { + t.Fatal("orphan not terminal") + } + if tc.want != 200 && s.op.Status != "queued" { + t.Fatal("live operation changed") + } + }) + } +} diff --git a/internal/api/handlers/workflow_engine_api_handler/cloud_reprovision_identity_test.go b/internal/api/handlers/workflow_engine_api_handler/cloud_reprovision_identity_test.go new file mode 100644 index 00000000..71eeb765 --- /dev/null +++ b/internal/api/handlers/workflow_engine_api_handler/cloud_reprovision_identity_test.go @@ -0,0 +1,22 @@ +package workflow_engine_api_handler + +import ( + "strings" + "testing" +) + +func TestProvisionOperationPreservesInstanceIdentity(t *testing.T) { + for _, tc := range []struct{ existing, requested, want string }{ + {"", "cloud-instance-original", "cloud-instance-original"}, + {"cloud-instance-original", "cloud-instance-original", "cloud-instance-original"}, + {"", " cloud-instance-original ", "cloud-instance-original"}, + } { + if got := provisionOperationInstanceID(tc.existing, tc.requested); got != tc.want { + t.Fatalf("got %q, want %q", got, tc.want) + } + } + first, second := provisionOperationInstanceID("", ""), provisionOperationInstanceID("", "") + if !strings.HasPrefix(first, "cloud-instance-") || first == second { + t.Fatal("fresh provisioning must generate distinct instance identities") + } +} diff --git a/internal/api/handlers/workflow_engine_api_handler/execution_billing.go b/internal/api/handlers/workflow_engine_api_handler/execution_billing.go new file mode 100644 index 00000000..aac3ebb0 --- /dev/null +++ b/internal/api/handlers/workflow_engine_api_handler/execution_billing.go @@ -0,0 +1,99 @@ +package workflow_engine_api_handler + +import ( + "context" + "sort" + "time" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +// These are lifecycle estimates, not activity charges or a provider invoice. +// One concrete generation contributes once, even when many tasks share it. +type executionCloudBilling struct { + CostBasis string `json:"costBasis"` + ExcludedCosts []string `json:"excludedCosts"` + Status string `json:"status"` + Currency string `json:"currency,omitempty"` + EstimatedCost *float64 `json:"estimatedCost"` + KnownCost float64 `json:"knownCost"` + Instances []executionInstanceBilling `json:"instances"` + AsOf time.Time `json:"asOf"` +} +type executionInstanceBilling struct { + InstanceID string `json:"instanceId"` + Status string `json:"status"` + Billing *domain.InstanceBilling `json:"billing,omitempty"` +} + +func (h *Handler) executionCloudBilling(ctx context.Context, tasks []domain.TaskExecution, operations []domain.CloudOperationRun) (executionCloudBilling, error) { + ids := map[string]bool{} + for _, task := range tasks { + if task.CloudInstanceID != "" { + ids[task.CloudInstanceID] = true + } + } + for _, operation := range operations { + if operation.InstanceID != "" { + ids[operation.InstanceID] = true + } + } + result := executionCloudBilling{ + Status: "estimated", + AsOf: time.Now().UTC(), + Instances: []executionInstanceBilling{}, + CostBasis: "VM compute and boot-disk lifecycle at on-demand marginal rates; not a provider invoice", + ExcludedCosts: []string{"network-egress", "public-ipv4", "image-storage", "discounts-and-tax"}, + } + ordered := make([]string, 0, len(ids)) + for id := range ids { + ordered = append(ordered, id) + } + sort.Strings(ordered) + for _, id := range ordered { + row := executionInstanceBilling{InstanceID: id, Status: "missing"} + if h.cloud != nil { + instance, err := h.cloud.FindProvisionedInstance(ctx, id) + if err != nil { + return result, err + } + if instance != nil { + row.Status, row.Billing = instance.Status, instance.Billing + } + } + result.Instances = append(result.Instances, row) + } + return sumExecutionCloudBilling(result), nil +} + +func sumExecutionCloudBilling(result executionCloudBilling) executionCloudBilling { + seen := map[string]bool{} + for _, row := range result.Instances { + if seen[row.InstanceID] { + continue + } + seen[row.InstanceID] = true + billing := row.Billing + if billing == nil { + result.Status = "incomplete" + continue + } + billing.RefreshCost(result.AsOf) + if billing.PricingStatus != "estimated" || billing.PriceCurrency == "" { + result.Status = "incomplete" + } + if result.Currency == "" { + result.Currency = billing.PriceCurrency + } + if billing.PriceCurrency != result.Currency { + result.Status = "incomplete" + continue + } + result.KnownCost += billing.CurrentCost + } + if result.Status == "estimated" { + total := result.KnownCost + result.EstimatedCost = &total + } + return result +} diff --git a/internal/api/handlers/workflow_engine_api_handler/execution_billing_test.go b/internal/api/handlers/workflow_engine_api_handler/execution_billing_test.go new file mode 100644 index 00000000..2ee4a273 --- /dev/null +++ b/internal/api/handlers/workflow_engine_api_handler/execution_billing_test.go @@ -0,0 +1,25 @@ +package workflow_engine_api_handler + +import ( + "github.com/UFFeScience/akoflow/internal/domain" + "testing" + "time" +) + +func TestLifecycleBillingCountsSharedGenerationOnce(t *testing.T) { + start := time.Unix(100, 0) + end := start.Add(time.Hour) + billing := &domain.InstanceBilling{StartedAt: start, ComputeStartedAt: &start, ComputePricePerSecond: 1.0 / 3600, DiskPricePerSecond: 0.1 / 3600, PricingStatus: "estimated", PriceCurrency: "USD"} + row := executionInstanceBilling{InstanceID: "vm-1", Billing: billing} + result := sumExecutionCloudBilling(executionCloudBilling{Status: "estimated", AsOf: end, Instances: []executionInstanceBilling{row, row}}) + if result.EstimatedCost == nil || *result.EstimatedCost != 1.1 { + t.Fatalf("double counted VM: %+v", result) + } +} + +func TestMissingLifecycleTariffIsNotFree(t *testing.T) { + result := sumExecutionCloudBilling(executionCloudBilling{Status: "estimated", AsOf: time.Now(), Instances: []executionInstanceBilling{{InstanceID: "vm-1"}}}) + if result.Status != "incomplete" || result.EstimatedCost != nil { + t.Fatalf("missing tariff became free: %+v", result) + } +} diff --git a/internal/api/handlers/workflow_engine_api_handler/execution_job_retry.go b/internal/api/handlers/workflow_engine_api_handler/execution_job_retry.go new file mode 100644 index 00000000..4b29c316 --- /dev/null +++ b/internal/api/handlers/workflow_engine_api_handler/execution_job_retry.go @@ -0,0 +1,14 @@ +package workflow_engine_api_handler + +import ( + "github.com/UFFeScience/akoflow/internal/domain" + domainqueue "github.com/UFFeScience/akoflow/internal/domain/queue" +) + +// Replaying a failed real workflow can repeat commands and cloud side effects. +// A failed run must instead be resumed through the explicit recovery API. +func configureExecutionJobRetries(job *domainqueue.Job, mode domain.ExecutionMode) { + if mode == domain.ExecutionModeReal || mode == domain.ExecutionModeInteractive { + job.MaxAttempts = 1 + } +} diff --git a/internal/api/handlers/workflow_engine_api_handler/execution_job_retry_test.go b/internal/api/handlers/workflow_engine_api_handler/execution_job_retry_test.go new file mode 100644 index 00000000..3703c0cc --- /dev/null +++ b/internal/api/handlers/workflow_engine_api_handler/execution_job_retry_test.go @@ -0,0 +1,26 @@ +package workflow_engine_api_handler + +import ( + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" + domainqueue "github.com/UFFeScience/akoflow/internal/domain/queue" +) + +func TestExecutionJobRequiresExplicitRealRecovery(t *testing.T) { + for _, mode := range []domain.ExecutionMode{domain.ExecutionModeReal, domain.ExecutionModeInteractive} { + job := domainqueue.Job{MaxAttempts: 5} + configureExecutionJobRetries(&job, mode) + if job.MaxAttempts != 1 { + t.Fatalf("mode %s can replay real side effects: %d attempts", mode, job.MaxAttempts) + } + } +} + +func TestNonRealExecutionPreservesQueueRetryPolicy(t *testing.T) { + job := domainqueue.Job{MaxAttempts: 5} + configureExecutionJobRetries(&job, domain.ExecutionMode("simulation")) + if job.MaxAttempts != 5 { + t.Fatalf("simulation retry policy changed: %d", job.MaxAttempts) + } +} diff --git a/internal/api/handlers/workflow_engine_api_handler/recovery_handle_gate_test.go b/internal/api/handlers/workflow_engine_api_handler/recovery_handle_gate_test.go new file mode 100644 index 00000000..126ab1df --- /dev/null +++ b/internal/api/handlers/workflow_engine_api_handler/recovery_handle_gate_test.go @@ -0,0 +1,33 @@ +package workflow_engine_api_handler + +import ( + "context" + "github.com/UFFeScience/akoflow/internal/domain" + "testing" +) + +type recoveryHandleQuery struct { + executionQueryStub + handles []domain.ActivityHandle +} + +func (s recoveryHandleQuery) ListHandles(context.Context, string) ([]domain.ActivityHandle, error) { + return s.handles, nil +} + +func TestRecoveryRejectsActiveHandlesFromEarlierAttempts(t *testing.T) { + for _, status := range []domain.ActivityHandleStatus{domain.HandleStarting, domain.HandleRunning, domain.HandleStopped, domain.HandleCompleted, domain.HandleFailed} { + h := &Handler{executions: recoveryHandleQuery{executionQueryStub: executionQueryStub{run: &domain.ExecutionRun{ID: "run", Status: domain.ExecutionRunFailed}}, handles: []domain.ActivityHandle{{ActivityID: "activity", Status: status}}}} + preview, err := h.executionRecoveryPreview(context.Background(), "run") + if err != nil { + t.Fatal(err) + } + active := status == domain.HandleStarting || status == domain.HandleRunning + if preview.Recoverable == active { + t.Fatalf("status=%s preview=%+v", status, preview) + } + if active && (len(preview.CleanupPending) != 1 || preview.CleanupPending[0] != "activity") { + t.Fatalf("missing cleanup reason: %+v", preview) + } + } +} diff --git a/internal/api/handlers/workflow_engine_api_handler/storage_budget_test.go b/internal/api/handlers/workflow_engine_api_handler/storage_budget_test.go new file mode 100644 index 00000000..8f0c11ab --- /dev/null +++ b/internal/api/handlers/workflow_engine_api_handler/storage_budget_test.go @@ -0,0 +1,26 @@ +package workflow_engine_api_handler + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCreateExecutionRejectsExhaustedQuotaBeforeCreatingRun(t *testing.T) { + // No execution command service is installed: reaching CreateRun would return + // 503. The quota failure must instead be detected before any side effect. + h := &Handler{} + request := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{ + "workflow":{"activities":[{"id":"a","metadata":{"workspaceProjection":{"contract":"whole-workspace-v1","source":"measured","inputs":[{"path":"input.fits","contentId":"hash","sizeBytes":100}]}}}]}, + "resources":[{"id":"node","metadata":{"workspaceStorageBudget":{"storageId":"home-quota","availableBytes":0,"observedAt":"2026-10-11T00:00:00Z","source":"quota-api"}}}], + "plan":{"assignments":[{"activityId":"a","resourceId":"node"}]} + }`)) + request.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + h.CreateExecution(response, request) + require.Equal(t, http.StatusUnprocessableEntity, response.Code) + require.Contains(t, response.Body.String(), "quota") +} diff --git a/internal/api/handlers/workflow_engine_api_handler/storage_override.go b/internal/api/handlers/workflow_engine_api_handler/storage_override.go new file mode 100644 index 00000000..08e1df58 --- /dev/null +++ b/internal/api/handlers/workflow_engine_api_handler/storage_override.go @@ -0,0 +1,31 @@ +package workflow_engine_api_handler + +import ( + "fmt" + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" +) + +// Only preflight uses these copies. The persisted execution retains the live +// quota observation and explicit user risk acceptance. OS quota stays enforced. +func preflightStorageResources(request ports.ExecutionRequest) ([]domain.Resource, error) { + resources := append([]domain.Resource(nil), request.Resources...) + for i := range resources { + r := &resources[i] + raw := r.Metadata["workspaceStorageQuotaOverride"] + if raw == nil { + continue + } + o, ok := raw.(map[string]any) + if !ok || r.Type != domain.ResourceHPCMachine || o["runId"] != request.Run.ID || request.Run.ID == "" || o["acceptedRisk"] != true || o["source"] != "explicit-user-authorization" || o["reason"] == "" || o["reason"] == nil { + return nil, fmt.Errorf("invalid or unscoped HPC quota preflight override for %s", r.ID) + } + m := map[string]any{} + for k, v := range r.Metadata { + m[k] = v + } + delete(m, "workspaceStorageBudget") + r.Metadata = m + } + return resources, nil +} diff --git a/internal/api/handlers/workflow_engine_api_handler/storage_override_test.go b/internal/api/handlers/workflow_engine_api_handler/storage_override_test.go new file mode 100644 index 00000000..15a249c6 --- /dev/null +++ b/internal/api/handlers/workflow_engine_api_handler/storage_override_test.go @@ -0,0 +1,24 @@ +package workflow_engine_api_handler + +import ( + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/stretchr/testify/require" + "testing" +) + +func TestQuotaOverrideScopedAndKeepsObservedSnapshot(t *testing.T) { + m := map[string]any{"workspaceStorageBudget": "live", "workspaceStorageQuotaOverride": map[string]any{"runId": "run", "acceptedRisk": true, "source": "explicit-user-authorization", "reason": "rolling cleanup experiment"}} + r := ports.ExecutionRequest{Run: domain.ExecutionRun{ID: "run"}, Resources: []domain.Resource{{ID: "hpc", Type: domain.ResourceHPCMachine, Metadata: m}}} + out, err := preflightStorageResources(r) + require.NoError(t, err) + require.NotContains(t, out[0].Metadata, "workspaceStorageBudget") + require.Equal(t, "live", m["workspaceStorageBudget"]) + r.Run.ID = "different" + _, err = preflightStorageResources(r) + require.Error(t, err) + r.Run.ID = "run" + r.Resources[0].Type = domain.ResourceCloudVM + _, err = preflightStorageResources(r) + require.Error(t, err) +} diff --git a/internal/api/handlers/workflow_engine_api_handler/workflow_engine_api_handler.go b/internal/api/handlers/workflow_engine_api_handler/workflow_engine_api_handler.go index 6a1f52ee..dbba67ec 100644 --- a/internal/api/handlers/workflow_engine_api_handler/workflow_engine_api_handler.go +++ b/internal/api/handlers/workflow_engine_api_handler/workflow_engine_api_handler.go @@ -33,6 +33,7 @@ import ( domainevents "github.com/UFFeScience/akoflow/internal/domain/events" domaininstance "github.com/UFFeScience/akoflow/internal/domain/instance" domainqueue "github.com/UFFeScience/akoflow/internal/domain/queue" + "github.com/UFFeScience/akoflow/internal/domain/storagebudget" cloudcredential "github.com/UFFeScience/akoflow/internal/infrastructure/credentials/cloud" "github.com/UFFeScience/akoflow/internal/infrastructure/credentials/sshkey" "github.com/UFFeScience/akoflow/internal/infrastructure/credentials/token" @@ -46,7 +47,7 @@ const maxRequestBodyBytes = 32 << 20 type ExecutionQuery interface { FindRun(context.Context, string) (*domain.ExecutionRun, error) ListRuns(context.Context) ([]domain.ExecutionRun, error) - ListRunsPage(context.Context, int, int, string, string, string) (domain.ExecutionRunPage, error) + ListRunsPage(context.Context, int, int, string, string, string, ...string) (domain.ExecutionRunPage, error) ListTasks(context.Context, string) ([]domain.TaskExecution, error) ListTransfers(context.Context, string) ([]domain.DataTransfer, error) ListHandles(context.Context, string) ([]domain.ActivityHandle, error) @@ -1082,86 +1083,14 @@ func (h *Handler) Search(w http.ResponseWriter, r *http.Request) { results = append(results, item) } } - if types["workflow"] { - items, err := h.workflows.List(r.Context()) - if err != nil { - writeError(w, http.StatusInternalServerError, err) - return - } - for _, item := range items { - add(SearchResult{Type: "workflow", ID: item.ID, Title: item.Name, Subtitle: item.Namespace, Path: "/workflows/" + url.PathEscape(item.ID)}, item.ID, item.ExternalID, item.Name, item.Namespace) - } - } - if types["execution"] { - items, err := h.executions.ListRuns(r.Context()) - if err != nil { - writeError(w, http.StatusInternalServerError, err) - return - } - for _, item := range items { - add(SearchResult{Type: "execution", ID: item.ID, Title: firstNonEmpty(item.Title, item.ID), Subtitle: firstNonEmpty(item.FailureReason, item.ResourceID, item.RuntimeID), Status: string(item.Status), Path: "/executions/" + url.PathEscape(item.ID)}, item.ID, item.Title, item.ResourceID, item.RuntimeID, item.FailureReason, string(item.Status)) - } - } - if types["artifact"] { - items, err := h.data.ListArtifacts(r.Context(), false) - if err != nil { - writeError(w, http.StatusInternalServerError, err) - return - } - for _, item := range items { - add(SearchResult{Type: "artifact", ID: item.ID, Title: firstNonEmpty(item.Name, item.ID), Subtitle: item.Version, Path: "/artifacts/" + url.PathEscape(item.ID)}, item.ID, item.Name, item.Version) - } - } - if types["environment"] { - items, err := h.environments.List(r.Context()) - if err != nil { - writeError(w, http.StatusInternalServerError, err) - return - } - for _, item := range items { - value := item.Environment - add(SearchResult{Type: "environment", ID: value.ID, Title: value.Name, Subtitle: value.Description, Status: string(value.Status), Path: "/environments/" + url.PathEscape(value.ID)}, value.ID, value.Name, value.Description, string(value.Status)) - } - } - if types["resource"] { - items, err := h.resources.List(r.Context()) - if err != nil { - writeError(w, http.StatusInternalServerError, err) - return - } - for _, item := range items { - add(SearchResult{Type: "resource", ID: item.ID, Title: item.Name, Subtitle: string(item.Type), Path: "/resources/" + url.PathEscape(item.ID)}, item.ID, item.Name, item.ProviderID, item.Region, item.Zone, string(item.Type), item.EnvironmentVersionID) - } - } - if types["plan"] { - items, err := h.plans.List(r.Context()) - if err != nil { - writeError(w, http.StatusInternalServerError, err) - return - } - for _, item := range items { - add(SearchResult{Type: "plan", ID: item.ID, Title: item.ID, Subtitle: item.Algorithm, Path: "/plans/" + url.PathEscape(item.ID)}, item.ID, item.Algorithm, item.Objective, item.WorkflowVersionID, item.ExecutionScopeID) - } + if !h.searchExecutionRecords(w, r, types, add) { + return } - if types["scope"] { - items, err := h.scopes.ListScopes(r.Context()) - if err != nil { - writeError(w, http.StatusInternalServerError, err) - return - } - for _, item := range items { - add(SearchResult{Type: "scope", ID: item.ID, Title: item.Name, Subtitle: item.NetworkTopologyID, Path: "/execution-scopes/" + url.PathEscape(item.ID)}, item.ID, item.Name, item.NetworkTopologyID, strings.Join(item.EnvironmentVersionIDs, " ")) - } + if !h.searchInfrastructureRecords(w, r, types, add) { + return } - if types["materialization"] { - items, err := h.data.ListArtifactMaterializations(r.Context(), "") - if err != nil { - writeError(w, http.StatusInternalServerError, err) - return - } - for _, item := range items { - add(SearchResult{Type: "materialization", ID: item.ID, Title: item.VariantID, Subtitle: firstNonEmpty(item.DestinationPath, item.ResourceID), Status: string(item.Status), Path: "/materializations?runId=" + url.QueryEscape(item.RunID)}, item.ID, item.VariantID, item.Digest, item.ResourceID, item.RunID, item.ActivityID, item.DestinationPath, string(item.Status)) - } + if !h.searchPlanningRecords(w, r, types, add) { + return } sort.SliceStable(results, func(i, j int) bool { return results[i].Score > results[j].Score }) if len(results) > limit { @@ -1211,11 +1140,11 @@ func firstNonEmpty(values ...string) string { } func (h *Handler) ListExecutions(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Has("page") || r.URL.Query().Has("pageSize") { + if r.URL.Query().Has("page") || r.URL.Query().Has("pageSize") || r.URL.Query().Has("workflowVersionId") { page := positiveInteger(r.URL.Query().Get("page"), 1) pageSize := positiveInteger(r.URL.Query().Get("pageSize"), 20) result, err := h.executions.ListRunsPage(r.Context(), page, pageSize, - r.URL.Query().Get("kind"), r.URL.Query().Get("mode"), r.URL.Query().Get("status")) + r.URL.Query().Get("kind"), r.URL.Query().Get("mode"), r.URL.Query().Get("status"), r.URL.Query().Get("workflowVersionId")) if err != nil { writeError(w, http.StatusInternalServerError, err) return @@ -1225,7 +1154,8 @@ func (h *Handler) ListExecutions(w http.ResponseWriter, r *http.Request) { items = append(items, map[string]any{"run": run}) } writeJSON(w, http.StatusOK, map[string]any{"items": items, "page": result.Page, - "pageSize": result.PageSize, "total": result.Total, "hasNext": result.HasNext}) + "pageSize": result.PageSize, "total": result.Total, "hasNext": result.HasNext, + "workflowVersionId": r.URL.Query().Get("workflowVersionId")}) return } runs, err := h.executions.ListRuns(r.Context()) @@ -1349,43 +1279,53 @@ func (h *Handler) GetExecution(w http.ResponseWriter, r *http.Request) { if h.cloudOperations != nil { response["infrastructureRuns"] = infrastructureRuns } + billing, billingErr := h.executionCloudBilling(r.Context(), activityAttempts, operations) + if billingErr != nil { + writeError(w, http.StatusInternalServerError, billingErr) + return + } + response["cloudBilling"] = billing response["timeline"] = applicationexecution.BuildTimeline(*run, tasks, transfers, operations) - if h.data != nil { - if operationEvents, ok := h.data.(WorkflowOperationEventQuery); ok { - events, eventsErr := operationEvents.ListWorkflowOperationEvents(r.Context(), run.ID) - if eventsErr != nil { - writeError(w, http.StatusInternalServerError, eventsErr) - return - } - response["operationEvents"] = events - } - instances, dataErr := h.data.ListInstances(r.Context(), run.ID) - if dataErr != nil { - writeError(w, http.StatusInternalServerError, dataErr) - return - } - locations, locationErr := h.data.ListLocations(r.Context(), run.ID) - if locationErr != nil { - writeError(w, http.StatusInternalServerError, locationErr) - return - } - response["dataObjects"], response["dataLocations"] = instances, locations - materializations, materializationErr := h.data.ListArtifactMaterializations(r.Context(), run.ID) - if materializationErr != nil { - writeError(w, http.StatusInternalServerError, materializationErr) - return - } - response["artifactMaterializations"] = materializations - transferRuns, transferErr := h.data.ListArtifactTransferRuns(r.Context(), run.ID) - if transferErr != nil { - writeError(w, http.StatusInternalServerError, transferErr) - return - } - response["artifactTransferRuns"] = transferRuns + if err := h.addExecutionData(r.Context(), run.ID, response); err != nil { + writeError(w, http.StatusInternalServerError, err) + return } writeJSON(w, http.StatusOK, response) } +func (h *Handler) addExecutionData(ctx context.Context, runID string, response map[string]any) error { + if h.data == nil { + return nil + } + if operationEvents, ok := h.data.(WorkflowOperationEventQuery); ok { + events, err := operationEvents.ListWorkflowOperationEvents(ctx, runID) + if err != nil { + return err + } + response["operationEvents"] = events + } + instances, err := h.data.ListInstances(ctx, runID) + if err != nil { + return err + } + locations, err := h.data.ListLocations(ctx, runID) + if err != nil { + return err + } + response["dataObjects"], response["dataLocations"] = instances, locations + materializations, err := h.data.ListArtifactMaterializations(ctx, runID) + if err != nil { + return err + } + response["artifactMaterializations"] = materializations + transfers, err := h.data.ListArtifactTransferRuns(ctx, runID) + if err != nil { + return err + } + response["artifactTransferRuns"] = transfers + return nil +} + func latestTaskAttempts(tasks []domain.TaskExecution) []domain.TaskExecution { latest := make(map[string]domain.TaskExecution, len(tasks)) for _, task := range tasks { @@ -1637,7 +1577,19 @@ func (h *Handler) RegisterDockerArtifact(w http.ResponseWriter, r *http.Request) } sum := sha256.Sum256([]byte("docker://" + image)) digest := "sha256:" + hex.EncodeToString(sum[:]) - build := domain.ArtifactBuild{ID: fmt.Sprintf("build-%d", now), ArtifactVersionID: version.ID, SourceType: "docker-image", ContextDigest: digest, RecipePath: image, RecipeDigest: digest, TargetFormat: "sif", TargetOS: "linux", TargetArchitecture: input.Architecture, BuildArguments: "{}", CacheKey: "docker-image:" + digest + ":" + version.ID} + build := domain.ArtifactBuild{ + ID: fmt.Sprintf("build-%d", now), + ArtifactVersionID: version.ID, + SourceType: "docker-image", + ContextDigest: digest, + RecipePath: image, + RecipeDigest: digest, + TargetFormat: "sif", + TargetOS: "linux", + TargetArchitecture: input.Architecture, + BuildArguments: "{}", + CacheKey: "docker-image:" + digest + ":" + version.ID, + } if err := h.data.SaveArtifactBuild(r.Context(), build); err != nil { writeError(w, http.StatusUnprocessableEntity, fmt.Errorf("create Docker artifact build: %w", err)) return @@ -1932,32 +1884,9 @@ func (h *Handler) CreateCloudCapacityTarget(w http.ResponseWriter, r *http.Reque if strings.TrimSpace(value.ID) == "" { value.ID = "cloud-capacity-" + uuid.NewString() } - if value.Provider == "gcp" && h.cloudCatalog != nil { - catalog, err := h.cloudCatalog.Cached(r.Context(), value.EnvironmentID) - if err != nil { - writeError(w, http.StatusUnprocessableEntity, fmt.Errorf("load cloud pricing catalog: %w", err)) - return - } - if catalog != nil { - if value.Configuration == nil { - value.Configuration = make(map[string]any) - } - for _, machine := range catalog.Machines { - if machine.ProviderTypeID == value.ProviderMachineType && machine.PricePerHour > 0 { - value.Configuration["pricePerHour"] = machine.PricePerHour - value.Configuration["priceCurrency"] = machine.PriceCurrency - value.Configuration["priceSource"] = machine.PriceSource - break - } - } - diskType, _ := value.Configuration["diskType"].(string) - for _, disk := range catalog.Disks { - if disk.ProviderTypeID == diskType && disk.PricePerGiBMonth > 0 { - value.Configuration["diskPricePerGiBMonth"] = disk.PricePerGiBMonth - break - } - } - } + if err := h.enrichCapacityTargetPricing(r.Context(), &value); err != nil { + writeError(w, http.StatusUnprocessableEntity, err) + return } value.Enabled = true if err := h.cloud.CreateCapacityTarget(r.Context(), value); err != nil { @@ -1995,8 +1924,14 @@ func (h *Handler) CreateCloudCapacityTarget(w http.ResponseWriter, r *http.Reque "capacityTargetId": value.ID, "providerMachineType": value.ProviderMachineType, "maximumInstances": value.MaximumInstances, "lifecyclePolicy": value.LifecyclePolicy, "provisioningMode": value.ProvisioningMode, "diskPricePerGiBMonth": configurationFloat64(value.Configuration, "diskPricePerGiBMonth"), + "priceCurrency": value.Configuration["priceCurrency"], + "priceSource": value.Configuration["priceSource"], }, } + resource.Metadata["pricingStatus"] = "estimated" + if resource.PricePerSecond <= 0 || (resource.StorageBytes > 0 && configurationFloat64(value.Configuration, "diskPricePerGiBMonth") <= 0) { + resource.Metadata["pricingStatus"] = "incomplete" + } if err := h.resources.Upsert(r.Context(), resource); err != nil { writeError(w, http.StatusUnprocessableEntity, err) return @@ -2010,6 +1945,38 @@ func (h *Handler) CreateCloudCapacityTarget(w http.ResponseWriter, r *http.Reque writeJSON(w, http.StatusCreated, value) } +func (h *Handler) enrichCapacityTargetPricing(ctx context.Context, target *domain.CloudCapacityTarget) error { + if target.Provider != "gcp" || h.cloudCatalog == nil { + return nil + } + catalog, err := h.cloudCatalog.Cached(ctx, target.EnvironmentID) + if err != nil { + return fmt.Errorf("load cloud pricing catalog: %w", err) + } + if catalog == nil { + return nil + } + if target.Configuration == nil { + target.Configuration = make(map[string]any) + } + for _, machine := range catalog.Machines { + if machine.ProviderTypeID == target.ProviderMachineType && machine.PricePerHour > 0 { + target.Configuration["pricePerHour"] = machine.PricePerHour + target.Configuration["priceCurrency"] = machine.PriceCurrency + target.Configuration["priceSource"] = machine.PriceSource + break + } + } + diskType, _ := target.Configuration["diskType"].(string) + for _, disk := range catalog.Disks { + if disk.ProviderTypeID == diskType && disk.PricePerGiBMonth > 0 { + target.Configuration["diskPricePerGiBMonth"] = disk.PricePerGiBMonth + break + } + } + return nil +} + func (h *Handler) DeleteCloudCapacityTarget(w http.ResponseWriter, r *http.Request) { if h.cloud == nil { writeError(w, http.StatusServiceUnavailable, fmt.Errorf("cloud configuration is unavailable")) @@ -2180,6 +2147,19 @@ func (h *Handler) enqueueExistingCloudInstanceOperation(w http.ResponseWriter, r } func (h *Handler) enqueueCloudOperation(w http.ResponseWriter, r *http.Request, kind, environmentID, instanceID, targetID string, request domain.CloudProvisionRequest) { + if kind == "provision" && strings.TrimSpace(request.InstanceID) != "" { + requestedID := strings.TrimSpace(request.InstanceID) + if instanceID != "" && instanceID != requestedID { + writeError(w, http.StatusUnprocessableEntity, fmt.Errorf("conflicting cloud instance identities")) + return + } + instance, err := h.cloud.FindProvisionedInstance(r.Context(), requestedID) + if err != nil || instance == nil || instance.EnvironmentID != environmentID || instance.CapacityTargetID != targetID { + writeError(w, http.StatusUnprocessableEntity, fmt.Errorf("requested cloud instance does not belong to target and environment")) + return + } + instanceID = requestedID + } operations, err := h.cloudOperations.ListCloudOperations(r.Context()) if err != nil { writeError(w, http.StatusInternalServerError, err) @@ -2202,8 +2182,8 @@ func (h *Handler) enqueueCloudOperation(w http.ResponseWriter, r *http.Request, writeError(w, http.StatusConflict, fmt.Errorf("cloud resource already has an active %s run", existing.Kind)) return } - if kind == "provision" && instanceID == "" { - instanceID = "cloud-instance-" + uuid.NewString() + if kind == "provision" { + instanceID = provisionOperationInstanceID(instanceID, request.InstanceID) request.InstanceID = instanceID } operation := domain.CloudOperationRun{ID: "cloud-run-" + uuid.NewString(), Kind: kind, @@ -2232,11 +2212,57 @@ func (h *Handler) enqueueCloudOperation(w http.ResponseWriter, r *http.Request, writeJSON(w, http.StatusAccepted, operation) } +func provisionOperationInstanceID(instanceID, requestedID string) string { + if instanceID != "" { + return instanceID + } + if requestedID = strings.TrimSpace(requestedID); requestedID != "" { + return requestedID + } + return "cloud-instance-" + uuid.NewString() +} + func (h *Handler) ListCloudOperations(w http.ResponseWriter, r *http.Request) { values, err := h.cloudOperations.ListCloudOperations(r.Context()) writeList(w, values, err) } +// ReconcileOrphanCloudOperation never cancels a queued or leased job. It only +// terminalizes an unpublished operation whose owning workflow has stopped. +func (h *Handler) ReconcileOrphanCloudOperation(w http.ResponseWriter, r *http.Request) { + q, ok := h.queue.(ExecutionRecoveryQueueStore) + if !ok || h.cloudOperations == nil || h.executions == nil { + writeError(w, http.StatusServiceUnavailable, fmt.Errorf("orphan reconciliation unavailable")) + return + } + op, err := h.cloudOperations.FindCloudOperation(r.Context(), r.PathValue("operationId")) + if err != nil || op == nil { + writeError(w, http.StatusNotFound, fmt.Errorf("cloud operation not found")) + return + } + if op.Status != "queued" || op.StartedAt != nil || op.ExecutionRunID == "" { + writeError(w, http.StatusConflict, fmt.Errorf("operation is not an unstarted workflow operation")) + return + } + job, err := q.FindLatestByAggregate(r.Context(), "cloud_operation", op.ID, eventloop.CloudOperationEventType(op.Kind)) + if err != nil || job != nil { + writeError(w, http.StatusConflict, fmt.Errorf("operation has a queue job or queue lookup failed")) + return + } + run, err := h.executions.FindRun(r.Context(), op.ExecutionRunID) + if err != nil || run == nil || (run.Status != domain.ExecutionRunFailed && run.Status != domain.ExecutionRunCancelled && run.Status != domain.ExecutionRunCompleted) { + writeError(w, http.StatusConflict, fmt.Errorf("owning workflow is not terminal")) + return + } + now := time.Now().UTC() + op.Status, op.Phase, op.FailureReason, op.FinishedAt = "cancelled", "orphan-reconciled", "Unpublished operation reconciled after terminal workflow", &now + if err := h.cloudOperations.UpdateCloudOperation(r.Context(), *op); err != nil { + writeError(w, http.StatusInternalServerError, err) + return + } + writeJSON(w, http.StatusOK, op) +} + func (h *Handler) GetCloudOperation(w http.ResponseWriter, r *http.Request) { value, err := h.cloudOperations.FindCloudOperation(r.Context(), r.PathValue("operationId")) if err != nil { @@ -2464,6 +2490,45 @@ func (h *Handler) CreatePlanningSession(w http.ResponseWriter, r *http.Request) if !decode(w, r, &session) { return } + if observe, _ := session.Configuration["observeWorkspaceStorageBudgets"].(bool); observe { + if h.resources == nil || h.scopes == nil { + writeError(w, http.StatusServiceUnavailable, fmt.Errorf("resource inventory unavailable for quota observations")) + return + } + resources, err := h.resources.List(r.Context()) + if err != nil { + writeError(w, http.StatusUnprocessableEntity, err) + return + } + scope, err := h.scopes.FindScope(r.Context(), session.ExecutionScopeID) + if err != nil || scope == nil { + writeError(w, http.StatusUnprocessableEntity, fmt.Errorf("execution scope unavailable for quota observations")) + return + } + allowed := map[string]bool{} + for _, id := range scope.EnvironmentVersionIDs { + allowed[id] = true + } + selected := []domain.Resource{} + for _, resource := range resources { + if allowed[resource.EnvironmentVersionID] { + selected = append(selected, resource) + } + } + resources = selected + request := ports.ExecutionRequest{Resources: resources, Workflow: domain.WorkflowVersion{Activities: []domain.Activity{{Metadata: map[string]any{"workspaceProjection": true}}}}} + if err := h.resolveWorkspaceHomes(r.Context(), &request); err != nil { + writeError(w, http.StatusUnprocessableEntity, err) + return + } + snapshots := map[string]any{} + for _, resource := range request.Resources { + if budget := resource.Metadata["workspaceStorageBudget"]; budget != nil { + snapshots[resource.ID] = budget + } + } + session.Configuration["workspaceStorageBudgetSnapshot"] = snapshots + } created, err := h.planning.Create(r.Context(), session) if err != nil { writeError(w, http.StatusUnprocessableEntity, err) @@ -2576,6 +2641,23 @@ func (h *Handler) CreateExecution(w http.ResponseWriter, r *http.Request) { if !decode(w, r, &request) { return } + if err := h.resolveWorkspaceHomes(r.Context(), &request); err != nil { + writeError(w, http.StatusUnprocessableEntity, err) + return + } + storageResources, overrideErr := preflightStorageResources(request) + if overrideErr != nil { + writeError(w, http.StatusUnprocessableEntity, overrideErr) + return + } + storageTimings := []storagebudget.Timing{} + for _, assignment := range request.Plan.Assignments { + storageTimings = append(storageTimings, storagebudget.Timing{ActivityID: assignment.ActivityID, ResourceID: assignment.ResourceID, Start: assignment.PredictedStartAt, Finish: assignment.PredictedFinishAt}) + } + if _, _, err := storagebudget.CheckScheduled(request.Workflow, storageResources, storageTimings); err != nil { + writeError(w, http.StatusUnprocessableEntity, err) + return + } if err := h.resolveBuildPreparations(r.Context(), &request); err != nil { writeError(w, http.StatusUnprocessableEntity, err) return @@ -2603,6 +2685,7 @@ func (h *Handler) CreateExecution(w http.ResponseWriter, r *http.Request) { return } job.AggregateType, job.AggregateID = "execution_run", request.Run.ID + configureExecutionJobRetries(&job, request.Run.Mode) job.IdempotencyKey = "execution-run:" + request.Run.ID stored, err := h.events.Publish(r.Context(), job) if err != nil { @@ -2717,6 +2800,18 @@ func (h *Handler) executionRecoveryPreview(ctx context.Context, runID string) (e return executionRecoveryPreview{}, err } preview := executionRecoveryPreview{RunID: runID, Recoverable: run.Status == domain.ExecutionRunFailed || run.Status == domain.ExecutionRunCancelled} + // A terminal task is not proof that an earlier external job has stopped. + // Inspect every handle, including superseded attempts, before recovery. + handles, err := h.executions.ListHandles(ctx, runID) + if err != nil { + return executionRecoveryPreview{}, err + } + for _, handle := range handles { + if handle.Status == domain.HandleStarting || handle.Status == domain.HandleRunning { + preview.Recoverable = false + preview.CleanupPending = append(preview.CleanupPending, handle.ActivityID) + } + } latest := latestTaskAttempts(tasks) latestByActivity := make(map[string]domain.TaskExecution, len(latest)) reusable := make(map[string]bool, len(latest)) @@ -2746,37 +2841,8 @@ func (h *Handler) executionRecoveryPreview(ctx context.Context, runID string) (e } } } - if recoveryQueue, ok := h.queue.(ExecutionRecoveryQueueStore); ok { - original, requestErr := recoveryQueue.FindLatestByAggregate(ctx, "execution_run", runID, eventloop.EventExecutionRunRequested) - if requestErr != nil { - return executionRecoveryPreview{}, requestErr - } - if original != nil { - var request ports.ExecutionRequest - if json.Unmarshal(original.Payload, &request) == nil { - for _, activity := range request.Workflow.Activities { - if _, observed := latestByActivity[activity.ID]; !observed { - interrupted[activity.ID] = true - } - } - changed := true - for changed { - changed = false - for _, activity := range request.Workflow.Activities { - if reusable[activity.ID] { - continue - } - for _, producerID := range recoveryProducerIDs(request.Workflow, activity.ID) { - if reusable[producerID] && !available[producerID] { - delete(reusable, producerID) - interrupted[producerID] = true - changed = true - } - } - } - } - } - } + if err := h.reconcileRecoveryPreview(ctx, runID, latestByActivity, reusable, available, interrupted); err != nil { + return executionRecoveryPreview{}, err } for activityID := range reusable { preview.Reusable = append(preview.Reusable, activityID) @@ -2860,6 +2926,10 @@ func (h *Handler) RecoverExecution(w http.ResponseWriter, r *http.Request) { return } request.Recovery = &ports.ExecutionRecovery{Mode: input.Mode, ReuseActivityIDs: preview.Reusable} + if err := h.resolveWorkspaceHomes(r.Context(), &request); err != nil { + writeError(w, http.StatusUnprocessableEntity, err) + return + } request.Recovery.RetryActivityIDs = append(append([]string{}, preview.Failed...), preview.Interrupted...) payload, err := json.Marshal(request) if err != nil { @@ -2876,6 +2946,7 @@ func (h *Handler) RecoverExecution(w http.ResponseWriter, r *http.Request) { return } job.AggregateType, job.AggregateID = "execution_run", runID + configureExecutionJobRetries(&job, request.Run.Mode) job.IdempotencyKey = fmt.Sprintf("execution-recovery:%s:%d", runID, time.Now().UTC().UnixNano()) stored, err := h.events.Publish(r.Context(), job) if err != nil { @@ -3155,3 +3226,165 @@ func writeError(w http.ResponseWriter, status int, err error) { } writeJSON(w, status, map[string]string{"error": message}) } + +func (h *Handler) searchExecutionRecords(w http.ResponseWriter, r *http.Request, types map[string]bool, add func(SearchResult, ...string)) bool { + if types["workflow"] { + items, err := h.workflows.List(r.Context()) + if err != nil { + writeError(w, http.StatusInternalServerError, err) + return false + } + for _, item := range items { + add(SearchResult{Type: "workflow", ID: item.ID, Title: item.Name, Subtitle: item.Namespace, Path: "/workflows/" + url.PathEscape(item.ID)}, item.ID, item.ExternalID, item.Name, item.Namespace) + } + } + if types["execution"] { + items, err := h.executions.ListRuns(r.Context()) + if err != nil { + writeError(w, http.StatusInternalServerError, err) + return false + } + for _, item := range items { + add(SearchResult{ + Type: "execution", + ID: item.ID, + Title: firstNonEmpty(item.Title, item.ID), + Subtitle: firstNonEmpty(item.FailureReason, item.ResourceID, item.RuntimeID), + Status: string(item.Status), + Path: "/executions/" + url.PathEscape(item.ID), + }, item.ID, item.Title, item.ResourceID, item.RuntimeID, item.FailureReason, string(item.Status)) + } + } + if types["artifact"] { + items, err := h.data.ListArtifacts(r.Context(), false) + if err != nil { + writeError(w, http.StatusInternalServerError, err) + return false + } + for _, item := range items { + add(SearchResult{Type: "artifact", ID: item.ID, Title: firstNonEmpty(item.Name, item.ID), Subtitle: item.Version, Path: "/artifacts/" + url.PathEscape(item.ID)}, item.ID, item.Name, item.Version) + } + } + return true +} + +func (h *Handler) searchInfrastructureRecords(w http.ResponseWriter, r *http.Request, types map[string]bool, add func(SearchResult, ...string)) bool { + if types["environment"] { + items, err := h.environments.List(r.Context()) + if err != nil { + writeError(w, http.StatusInternalServerError, err) + return false + } + for _, item := range items { + value := item.Environment + add(SearchResult{ + Type: "environment", + ID: value.ID, + Title: value.Name, + Subtitle: value.Description, + Status: string(value.Status), + Path: "/environments/" + url.PathEscape(value.ID), + }, value.ID, value.Name, value.Description, string(value.Status)) + } + } + if types["resource"] { + items, err := h.resources.List(r.Context()) + if err != nil { + writeError(w, http.StatusInternalServerError, err) + return false + } + for _, item := range items { + add(SearchResult{ + Type: "resource", + ID: item.ID, + Title: item.Name, + Subtitle: string(item.Type), + Path: "/resources/" + url.PathEscape(item.ID), + }, item.ID, item.Name, item.ProviderID, item.Region, item.Zone, string(item.Type), item.EnvironmentVersionID) + } + } + return true +} + +func (h *Handler) searchPlanningRecords(w http.ResponseWriter, r *http.Request, types map[string]bool, add func(SearchResult, ...string)) bool { + if types["plan"] { + items, err := h.plans.List(r.Context()) + if err != nil { + writeError(w, http.StatusInternalServerError, err) + return false + } + for _, item := range items { + add(SearchResult{Type: "plan", ID: item.ID, Title: item.ID, Subtitle: item.Algorithm, Path: "/plans/" + url.PathEscape(item.ID)}, item.ID, item.Algorithm, item.Objective, item.WorkflowVersionID, item.ExecutionScopeID) + } + } + if types["scope"] { + items, err := h.scopes.ListScopes(r.Context()) + if err != nil { + writeError(w, http.StatusInternalServerError, err) + return false + } + for _, item := range items { + add(SearchResult{ + Type: "scope", + ID: item.ID, + Title: item.Name, + Subtitle: item.NetworkTopologyID, + Path: "/execution-scopes/" + url.PathEscape(item.ID), + }, item.ID, item.Name, item.NetworkTopologyID, strings.Join(item.EnvironmentVersionIDs, " ")) + } + } + if types["materialization"] { + items, err := h.data.ListArtifactMaterializations(r.Context(), "") + if err != nil { + writeError(w, http.StatusInternalServerError, err) + return false + } + for _, item := range items { + add(SearchResult{ + Type: "materialization", + ID: item.ID, + Title: item.VariantID, + Subtitle: firstNonEmpty(item.DestinationPath, item.ResourceID), + Status: string(item.Status), + Path: "/materializations?runId=" + url.QueryEscape(item.RunID), + }, item.ID, item.VariantID, item.Digest, item.ResourceID, item.RunID, item.ActivityID, item.DestinationPath, string(item.Status)) + } + } + return true +} + +func (h *Handler) reconcileRecoveryPreview(ctx context.Context, runID string, latestByActivity map[string]domain.TaskExecution, reusable, available, interrupted map[string]bool) error { + if recoveryQueue, ok := h.queue.(ExecutionRecoveryQueueStore); ok { + original, requestErr := recoveryQueue.FindLatestByAggregate(ctx, "execution_run", runID, eventloop.EventExecutionRunRequested) + if requestErr != nil { + return requestErr + } + if original != nil { + var request ports.ExecutionRequest + if json.Unmarshal(original.Payload, &request) == nil { + for _, activity := range request.Workflow.Activities { + if _, observed := latestByActivity[activity.ID]; !observed { + interrupted[activity.ID] = true + } + } + changed := true + for changed { + changed = false + for _, activity := range request.Workflow.Activities { + if reusable[activity.ID] { + continue + } + for _, producerID := range recoveryProducerIDs(request.Workflow, activity.ID) { + if reusable[producerID] && !available[producerID] { + delete(reusable, producerID) + interrupted[producerID] = true + changed = true + } + } + } + } + } + } + } + return nil +} diff --git a/internal/api/handlers/workflow_engine_api_handler/workflow_engine_api_handler_test.go b/internal/api/handlers/workflow_engine_api_handler/workflow_engine_api_handler_test.go index f3eaf47f..cbe81a2c 100644 --- a/internal/api/handlers/workflow_engine_api_handler/workflow_engine_api_handler_test.go +++ b/internal/api/handlers/workflow_engine_api_handler/workflow_engine_api_handler_test.go @@ -3,6 +3,7 @@ package workflow_engine_api_handler import ( "bytes" "context" + "encoding/json" "errors" "net/http" "net/http/httptest" @@ -93,11 +94,12 @@ func (s validatorStub) Validate(domain.SchedulePlan, domain.WorkflowVersion, []d } type executionQueryStub struct { - run *domain.ExecutionRun - tasks []domain.TaskExecution - transfers []domain.DataTransfer - handles []domain.ActivityHandle - events []domainevents.Event + pageWorkflowVersion *string + run *domain.ExecutionRun + tasks []domain.TaskExecution + transfers []domain.DataTransfer + handles []domain.ActivityHandle + events []domainevents.Event } type executionCommandStub struct{} @@ -143,10 +145,28 @@ func (s executionQueryStub) ListRuns(context.Context) ([]domain.ExecutionRun, er } return []domain.ExecutionRun{*s.run}, nil } -func (s executionQueryStub) ListRunsPage(_ context.Context, page, pageSize int, _, _, _ string) (domain.ExecutionRunPage, error) { +func (s executionQueryStub) ListRunsPage(_ context.Context, page, pageSize int, _, _, _ string, versions ...string) (domain.ExecutionRunPage, error) { + if s.pageWorkflowVersion != nil && len(versions) > 0 { + *s.pageWorkflowVersion = versions[0] + } items, _ := s.ListRuns(context.Background()) return domain.ExecutionRunPage{Items: items, Page: page, PageSize: pageSize, Total: len(items)}, nil } + +func TestListExecutionsPassesWorkflowFilterToPagedQuery(t *testing.T) { + version := "" + handler := newTestHandler() + handler.executions = executionQueryStub{pageWorkflowVersion: &version} + recorder := httptest.NewRecorder() + handler.ListExecutions(recorder, httptest.NewRequest(http.MethodGet, + "/execution-runs/?workflowVersionId=workflow-v1&page=2&pageSize=20", nil)) + require.Equal(t, http.StatusOK, recorder.Code) + require.Equal(t, "workflow-v1", version) + var response map[string]any + require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &response)) + require.Equal(t, "workflow-v1", response["workflowVersionId"]) + require.Equal(t, float64(2), response["page"]) +} func (s executionQueryStub) ListTasks(context.Context, string) ([]domain.TaskExecution, error) { return s.tasks, nil } diff --git a/internal/api/handlers/workflow_engine_api_handler/workspace_budget.go b/internal/api/handlers/workflow_engine_api_handler/workspace_budget.go new file mode 100644 index 00000000..72efc767 --- /dev/null +++ b/internal/api/handlers/workflow_engine_api_handler/workspace_budget.go @@ -0,0 +1,25 @@ +package workflow_engine_api_handler + +import ( + "context" + "fmt" + "net/http" + + "github.com/UFFeScience/akoflow/internal/domain/storagebudget" +) + +func (h *Handler) WorkspaceStorageBudget(w http.ResponseWriter, r *http.Request) { + observer, ok := h.storage.(interface { + WorkspaceBudget(context.Context, string, string) (storagebudget.Snapshot, error) + }) + if !ok { + writeError(w, http.StatusServiceUnavailable, fmt.Errorf("workspace quota observations are unavailable")) + return + } + snapshot, err := observer.WorkspaceBudget(r.Context(), r.PathValue("storageId"), r.URL.Query().Get("path")) + if err != nil { + writeError(w, http.StatusUnprocessableEntity, err) + return + } + writeJSON(w, http.StatusOK, snapshot) +} diff --git a/internal/api/handlers/workflow_engine_api_handler/workspace_home.go b/internal/api/handlers/workflow_engine_api_handler/workspace_home.go new file mode 100644 index 00000000..970284d7 --- /dev/null +++ b/internal/api/handlers/workflow_engine_api_handler/workspace_home.go @@ -0,0 +1,106 @@ +package workflow_engine_api_handler + +import ( + "context" + "fmt" + "path" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/storagebudget" +) + +// Resolve execution paths from the discovered catalog, without changing the +// frozen placement or assuming that compute-node discovery includes login facts. +func (h *Handler) resolveWorkspaceHomes(ctx context.Context, request *ports.ExecutionRequest) error { + connections, ok := h.environments.(ports.ConnectionStore) + if !ok { + return nil + } + needsBudget := false + for _, activity := range request.Workflow.Activities { + needsBudget = needsBudget || activity.Metadata["workspaceProjection"] != nil + } + observer, canObserve := h.storage.(interface { + WorkspaceBudget(context.Context, string, string) (storagebudget.Snapshot, error) + }) + budgets := map[string]storagebudget.Snapshot{} + for i := range request.Resources { + resource := &request.Resources[i] + connectionID, _ := resource.Metadata["connectionId"].(string) + if connectionID == "" { + continue + } + connection, err := connections.FindConnection(ctx, connectionID) + if err != nil { + return err + } + if connection == nil { + continue + } + definition, err := h.environments.Find(ctx, connection.EnvironmentID) + if err != nil { + return err + } + if definition == nil || definition.Version.ID != resource.EnvironmentVersionID { + continue + } + home, homeStorageID, err := discoveredWorkspaceHome(definition, connectionID) + if err != nil { + return err + } + if home != "" { + metadata := make(map[string]any, len(resource.Metadata)+1) + for key, value := range resource.Metadata { + metadata[key] = value + } + metadata["homeDirectory"] = home + if needsBudget && canObserve { + budget, exists := budgets[homeStorageID] + if !exists { + budget, err = observer.WorkspaceBudget(ctx, homeStorageID, "") + if err != nil { + return fmt.Errorf("discover workspace quota: %w", err) + } + budgets[homeStorageID] = budget + } + metadata["workspaceStorageBudget"] = budget + } + resource.Metadata = metadata + } + } + return nil +} + +// A home is eligible only when the same catalog connection discovers a Slurm +// runtime and an absolute home browse root matching its storage endpoint. +func discoveredWorkspaceHome(definition *domain.EnvironmentDefinition, connectionID string) (string, string, error) { + isSlurm := false + for _, runtime := range definition.Runtimes { + isSlurm = isSlurm || runtime.Driver == domain.RuntimeDriverSlurm + } + if !isSlurm { + return "", "", nil + } + home, storageID := "", "" + for _, storage := range definition.Storages { + id, _ := storage.Metadata["connectionId"].(string) + if id != connectionID { + continue + } + roots, _ := storage.Configuration["browseRoots"].([]any) + for _, raw := range roots { + root, _ := raw.(map[string]any) + label, _ := root["label"].(string) + candidate, _ := root["path"].(string) + if label != "home" || !path.IsAbs(candidate) || candidate == "/" || candidate != storage.Endpoint { + continue + } + if home != "" && home != candidate { + return "", "", fmt.Errorf("ambiguous discovered home for connection %q", connectionID) + } + home, storageID = candidate, storage.ID + } + } + return home, storageID, nil +} diff --git a/internal/api/handlers/workflow_engine_api_handler/workspace_home_test.go b/internal/api/handlers/workflow_engine_api_handler/workspace_home_test.go new file mode 100644 index 00000000..4cf05232 --- /dev/null +++ b/internal/api/handlers/workflow_engine_api_handler/workspace_home_test.go @@ -0,0 +1,73 @@ +package workflow_engine_api_handler + +import ( + "context" + "testing" + "time" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/storagebudget" + "github.com/stretchr/testify/require" +) + +func TestResolveWorkspaceHomesUsesExactDiscoveredConnection(t *testing.T) { + for _, connectionID := range []string{"ssh", "other"} { + t.Run(connectionID, func(t *testing.T) { + definition := domain.EnvironmentDefinition{ + Version: domain.EnvironmentVersion{ID: "v1"}, + Runtimes: []domain.EnvironmentRuntime{{Driver: domain.RuntimeDriverSlurm}}, + Storages: []domain.StorageResource{{Endpoint: "/home/scientist", Metadata: map[string]any{"connectionId": connectionID}, Configuration: map[string]any{"browseRoots": []any{map[string]any{"label": "home", "path": "/home/scientist"}}}}}, + } + h := Handler{environments: &environmentCatalogRichStub{found: &definition, connection: &domain.EnvironmentConnection{ID: "ssh", EnvironmentID: "hpc"}}} + original := map[string]any{"connectionId": "ssh"} + request := ports.ExecutionRequest{Resources: []domain.Resource{{ID: "node", EnvironmentVersionID: "v1", Metadata: original}}} + if err := h.resolveWorkspaceHomes(context.Background(), &request); err != nil { + t.Fatal(err) + } + home, _ := request.Resources[0].Metadata["homeDirectory"].(string) + if connectionID == "ssh" && home != "/home/scientist" { + t.Fatalf("home=%q", home) + } + if connectionID == "other" && home != "" { + t.Fatalf("foreign connection home=%q", home) + } + if _, ok := original["homeDirectory"]; ok { + t.Fatal("mutated frozen input metadata") + } + }) + } +} + +type budgetNavigatorStub struct { + storageNavigatorStub + calls int +} + +func (s *budgetNavigatorStub) WorkspaceBudget(_ context.Context, id, path string) (storagebudget.Snapshot, error) { + s.calls++ + return storagebudget.Snapshot{StorageID: "shared-user-quota", AvailableBytes: 180 * 1024, Source: "live-posix-filesystem-and-user-quota-v1", ObservedAt: time.Now().UTC()}, nil +} + +func TestResolveWorkspaceBudgetRefreshesAndSharesExactObservation(t *testing.T) { + definition := domain.EnvironmentDefinition{ + Version: domain.EnvironmentVersion{ID: "v1"}, + Runtimes: []domain.EnvironmentRuntime{{Driver: domain.RuntimeDriverSlurm}}, + Storages: []domain.StorageResource{ + {ID: "home", Endpoint: "/home/scientist", Metadata: map[string]any{"connectionId": "ssh"}, Configuration: map[string]any{"browseRoots": []any{map[string]any{"label": "home", "path": "/home/scientist"}}}}, + }, + } + navigator := &budgetNavigatorStub{} + h := Handler{storage: navigator, environments: &environmentCatalogRichStub{found: &definition, connection: &domain.EnvironmentConnection{ID: "ssh", EnvironmentID: "hpc"}}} + request := ports.ExecutionRequest{ + Workflow: domain.WorkflowVersion{Activities: []domain.Activity{{Metadata: map[string]any{"workspaceProjection": true}}}}, + Resources: []domain.Resource{ + {ID: "node1", EnvironmentVersionID: "v1", Metadata: map[string]any{"connectionId": "ssh", "workspaceStorageBudget": "stale"}}, + {ID: "node2", EnvironmentVersionID: "v1", Metadata: map[string]any{"connectionId": "ssh"}}, + }, + } + require.NoError(t, h.resolveWorkspaceHomes(context.Background(), &request)) + require.Equal(t, 1, navigator.calls) + require.Equal(t, request.Resources[0].Metadata["workspaceStorageBudget"], request.Resources[1].Metadata["workspaceStorageBudget"]) + require.Equal(t, int64(180*1024), request.Resources[0].Metadata["workspaceStorageBudget"].(storagebudget.Snapshot).AvailableBytes) +} diff --git a/internal/api/handlers/workflow_engine_api_handler/workspace_preservation.go b/internal/api/handlers/workflow_engine_api_handler/workspace_preservation.go new file mode 100644 index 00000000..e6a15722 --- /dev/null +++ b/internal/api/handlers/workflow_engine_api_handler/workspace_preservation.go @@ -0,0 +1,255 @@ +package workflow_engine_api_handler + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "strings" + + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/google/uuid" +) + +type preservedWorkspaceStore interface { + ReleasePreservedWorkspaces(context.Context, string, string, []string, string) error +} + +// PreserveCancelledCloudWorkspaces copies an exact-generation archive to the +// relay before releasing consumer leases. It never removes source files or VMs. +func (h *Handler) PreserveCancelledCloudWorkspaces(w http.ResponseWriter, r *http.Request) { + store, ok := h.executions.(preservedWorkspaceStore) + query, queryOK := h.executions.(WorkspaceQuery) + if h.readOnly || !ok || !queryOK || h.storage == nil { + writeError(w, http.StatusForbidden, fmt.Errorf("workspace preservation is unavailable")) + return + } + var input struct { + DownloadID string `json:"downloadId"` + InstanceID string `json:"instanceId"` + SHA256 string `json:"sha256"` + } + if !decode(w, r, &input) { + return + } + runID := r.PathValue("runId") + if path.Base(runID) != runID || runID == "." || !strings.HasPrefix(input.InstanceID, "cloud-instance-") { + writeError(w, http.StatusBadRequest, fmt.Errorf("explicit run and instance identities required")) + return + } + want := strings.TrimPrefix(input.SHA256, "sha256:") + decoded, err := hex.DecodeString(want) + if err != nil || len(decoded) != sha256.Size { + writeError(w, http.StatusBadRequest, fmt.Errorf("SHA-256 required")) + return + } + run, err := h.executions.FindRun(r.Context(), runID) + if err != nil || run == nil || run.Status != domain.ExecutionRunCancelled { + writeError(w, http.StatusConflict, fmt.Errorf("only cancelled runs can release preserved workspaces")) + return + } + handles, err := h.executions.ListHandles(r.Context(), runID) + if err != nil { + writeError(w, http.StatusInternalServerError, err) + return + } + if !preservationHandlesTerminal(handles) { + writeError(w, http.StatusConflict, fmt.Errorf("execution handles are still active")) + return + } + download, err := h.storage.Download(r.Context(), input.DownloadID) + archivePath := "runs/" + runID + ".tar.gz" + if err != nil || download == nil || string(download.Status) != "ready" || download.Strategy != "archive" || download.StorageID != "cloud-workspace:"+input.InstanceID || download.Path != archivePath { + writeError(w, http.StatusConflict, fmt.Errorf("archive does not match exact run and cloud generation")) + return + } + remoteDigest, err := h.storage.Checksum(r.Context(), download.StorageID, download.Path) + if err != nil || strings.TrimPrefix(remoteDigest, "sha256:") != want { + writeError(w, http.StatusConflict, fmt.Errorf("source archive checksum differs")) + return + } + workspaces, err := query.ListWorkspaces(r.Context(), runID) + if err != nil { + writeError(w, http.StatusInternalServerError, err) + return + } + selected, err := releasablePreservedWorkspaces(workspaces, input.InstanceID) + if err != nil { + writeError(w, http.StatusConflict, err) + return + } + reader, _, err := h.storage.OpenDownload(r.Context(), download.ID) + if err != nil { + writeError(w, http.StatusUnprocessableEntity, err) + return + } + defer reader.Close() + relayPath, err := preserveArchive(reader, "storage/preserved-workspaces", want, runID, selected) + if err != nil { + writeError(w, http.StatusUnprocessableEntity, err) + return + } + ids := preservedWorkspaceIDs(selected) + if err := store.ReleasePreservedWorkspaces(r.Context(), runID, input.InstanceID, ids, relayPath+" sha256:"+want); err != nil { + writeError(w, http.StatusConflict, err) + return + } + writeJSON(w, http.StatusOK, map[string]any{"status": "preserved", "archive": relayPath, "sha256": want, "workspaceIds": ids, "instanceId": input.InstanceID}) +} + +func preservedWorkspaceIDs(workspaces []domain.ActivityWorkspace) []string { + ids := make([]string, 0, len(workspaces)) + for _, workspace := range workspaces { + ids = append(ids, workspace.ID) + } + return ids +} + +func preservationHandlesTerminal(handles []domain.ActivityHandle) bool { + for _, handle := range handles { + if handle.Status != domain.HandleCompleted && handle.Status != domain.HandleFailed && handle.Status != domain.HandleStopped { + return false + } + } + return true +} + +func releasablePreservedWorkspaces(workspaces []domain.ActivityWorkspace, instanceID string) ([]domain.ActivityWorkspace, error) { + selected := []domain.ActivityWorkspace{} + for _, workspace := range workspaces { + location, parseErr := url.Parse(workspace.URI) + if parseErr == nil && location.Query().Get("cloudInstanceId") == instanceID && workspace.State != domain.WorkspaceReleased { + if workspace.Pinned || path.Base(workspace.ActivityID) != workspace.ActivityID { + return nil, fmt.Errorf("workspace cannot be released") + } + selected = append(selected, workspace) + } + } + if len(selected) == 0 { + return nil, fmt.Errorf("no retained workspaces match this generation") + } + return selected, nil +} + +func preserveArchive(reader io.Reader, directory, digest, runID string, workspaces []domain.ActivityWorkspace) (string, error) { + if err := os.MkdirAll(directory, 0700); err != nil { + return "", err + } + root, err := os.OpenRoot(directory) + if err != nil { + return "", err + } + defer root.Close() + temporary := ".pending-" + uuid.NewString() + file, err := root.OpenFile(temporary, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + return "", err + } + defer file.Close() + defer root.Remove(temporary) + hash := sha256.New() + const maxArchive = int64(50 << 30) + size, err := io.Copy(io.MultiWriter(file, hash), io.LimitReader(reader, maxArchive+1)) + if err != nil || size == 0 || size > maxArchive { + return "", fmt.Errorf("archive copy is empty, oversized or failed: %v", err) + } + if hex.EncodeToString(hash.Sum(nil)) != digest { + return "", fmt.Errorf("relay archive checksum differs") + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return "", err + } + if err := validatePreservedArchive(file, runID, workspaces); err != nil { + return "", err + } + if err := file.Sync(); err != nil { + return "", err + } + if err := file.Close(); err != nil { + return "", err + } + name := digest + ".tar.gz" + if err := root.Rename(temporary, name); err != nil { + return "", err + } + dir, err := root.Open(".") + if err != nil { + return "", err + } + defer dir.Close() + if err := dir.Sync(); err != nil { + return "", err + } + return path.Join(directory, name), nil +} + +func validatePreservedArchive(reader io.Reader, runID string, workspaces []domain.ActivityWorkspace) error { + gz, err := gzip.NewReader(reader) + if err != nil { + return err + } + defer gz.Close() + tarReader := tar.NewReader(gz) + prefix := "runs/" + runID + seen := map[string]bool{} + expected := map[string]domain.WorkspaceEntry{} + for _, workspace := range workspaces { + for _, entry := range workspace.Manifest.Final { + expected[prefix+"/"+workspace.ActivityID+"/"+entry.Path] = entry + } + } + files := 0 + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + name := strings.TrimSuffix(header.Name, "/") + if path.Clean(name) != name || (name != prefix && !strings.HasPrefix(name, prefix+"/")) || (header.Typeflag != tar.TypeDir && header.Typeflag != tar.TypeReg) { + return fmt.Errorf("archive contains unsafe or unrelated entry") + } + seen[name] = true + if header.Typeflag == tar.TypeReg { + files++ + } + if entry, ok := expected[name]; ok && header.Typeflag == tar.TypeReg { + digest := sha256.New() + size, err := io.Copy(digest, tarReader) + if err != nil { + return err + } + if size != entry.SizeBytes || strings.TrimPrefix(entry.Digest, "sha256:") != hex.EncodeToString(digest.Sum(nil)) { + return fmt.Errorf("archive differs from sealed workspace manifest") + } + } else if _, err := io.Copy(io.Discard, tarReader); err != nil { + return err + } + } + if _, err := io.Copy(io.Discard, gz); err != nil { + return err + } // validate gzip footer + if files == 0 { + return fmt.Errorf("archive contains no files") + } + for _, workspace := range workspaces { + if !seen[prefix+"/"+workspace.ActivityID] { + return fmt.Errorf("archive omits retained workspace %s", workspace.ActivityID) + } + for _, entry := range workspace.Manifest.Final { + if !seen[prefix+"/"+workspace.ActivityID+"/"+entry.Path] { + return fmt.Errorf("archive omits workspace manifest entry") + } + } + } + return nil +} diff --git a/internal/api/handlers/workflow_engine_api_handler/workspace_preservation_test.go b/internal/api/handlers/workflow_engine_api_handler/workspace_preservation_test.go new file mode 100644 index 00000000..d691a812 --- /dev/null +++ b/internal/api/handlers/workflow_engine_api_handler/workspace_preservation_test.go @@ -0,0 +1,85 @@ +package workflow_engine_api_handler + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "os" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func preservationFixture(t *testing.T, name, content string, kind byte) []byte { + t.Helper() + var output bytes.Buffer + gz := gzip.NewWriter(&output) + archive := tar.NewWriter(gz) + if err := archive.WriteHeader(&tar.Header{Name: "runs/run/producer/", Typeflag: tar.TypeDir, Mode: 0700}); err != nil { + t.Fatal(err) + } + header := &tar.Header{Name: name, Typeflag: kind, Mode: 0600, Size: int64(len(content))} + if kind != tar.TypeReg { + header.Size = 0 + header.Linkname = "outside" + } + if err := archive.WriteHeader(header); err != nil { + t.Fatal(err) + } + if kind == tar.TypeReg { + if _, err := archive.Write([]byte(content)); err != nil { + t.Fatal(err) + } + } + if err := archive.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return output.Bytes() +} + +func TestPreservationValidatesManifestAndDurableCopy(t *testing.T) { + data := preservationFixture(t, "runs/run/producer/output.txt", "science", tar.TypeReg) + digest := sha256.Sum256([]byte("science")) + workspace := domain.ActivityWorkspace{ActivityID: "producer", Manifest: domain.WorkspaceManifest{Final: []domain.WorkspaceEntry{{Path: "output.txt", SizeBytes: 7, Digest: "sha256:" + hex.EncodeToString(digest[:])}}}} + archiveDigest := sha256.Sum256(data) + file, err := preserveArchive(bytes.NewReader(data), t.TempDir(), hex.EncodeToString(archiveDigest[:]), "run", []domain.ActivityWorkspace{workspace}) + if err != nil { + t.Fatal(err) + } + preserved, err := os.ReadFile(file) + if err != nil || !bytes.Equal(preserved, data) { + t.Fatalf("durable archive changed: %v", err) + } + workspace.Manifest.Final[0].Digest = "sha256:wrong" + if err := validatePreservedArchive(bytes.NewReader(data), "run", []domain.ActivityWorkspace{workspace}); err == nil { + t.Fatal("changed manifest accepted") + } + if _, err := preserveArchive(bytes.NewReader(data), t.TempDir(), "wrong", "run", nil); err == nil { + t.Fatal("wrong archive hash accepted") + } +} + +func TestPreservationRejectsLinksTraversalMissingWorkspaceAndCorruption(t *testing.T) { + for _, fixture := range []struct { + name string + kind byte + }{{"runs/run/producer/link", tar.TypeSymlink}, {"runs/run/producer/link", tar.TypeLink}, {"runs/run/../escape", tar.TypeReg}, {"runs/another/file", tar.TypeReg}} { + data := preservationFixture(t, fixture.name, "science", fixture.kind) + if err := validatePreservedArchive(bytes.NewReader(data), "run", nil); err == nil { + t.Fatalf("unsafe archive accepted: %+v", fixture) + } + } + data := preservationFixture(t, "runs/run/producer/output.txt", "science", tar.TypeReg) + if err := validatePreservedArchive(bytes.NewReader(data), "run", []domain.ActivityWorkspace{{ActivityID: "missing"}}); err == nil { + t.Fatal("missing workspace accepted") + } + data[len(data)-5] ^= 0xff + if err := validatePreservedArchive(bytes.NewReader(data), "run", nil); err == nil { + t.Fatal("corrupt gzip accepted") + } +} diff --git a/internal/api/httpserver/httpserver.go b/internal/api/httpserver/httpserver.go index 00864ee5..eab28b89 100755 --- a/internal/api/httpserver/httpserver.go +++ b/internal/api/httpserver/httpserver.go @@ -110,6 +110,7 @@ func NewMux(workflowEngine *workflow_engine_api_handler.Handler) *http.ServeMux mux.HandleFunc("POST /akoflow-api/cloud-instances/{instanceId}/validate/", http_config.KernelHandler(workflowEngine.ValidateCloudInstance)) mux.HandleFunc("GET /akoflow-api/cloud-operations/", http_config.KernelHandler(workflowEngine.ListCloudOperations)) mux.HandleFunc("GET /akoflow-api/cloud-operations/{operationId}/", http_config.KernelHandler(workflowEngine.GetCloudOperation)) + mux.HandleFunc("POST /akoflow-api/cloud-operations/{operationId}/reconcile-orphan/", http_config.KernelHandler(workflowEngine.ReconcileOrphanCloudOperation)) mux.HandleFunc("GET /akoflow-api/cloud-operations/{operationId}/events/", http_config.KernelHandler(workflowEngine.ListCloudOperationEvents)) mux.HandleFunc("GET /akoflow-api/machine-configurations/", http_config.KernelHandler(workflowEngine.ListMachineConfigurations)) mux.HandleFunc("POST /akoflow-api/machine-configurations/", http_config.KernelHandler(workflowEngine.CreateMachineConfiguration)) @@ -124,6 +125,7 @@ func NewMux(workflowEngine *workflow_engine_api_handler.Handler) *http.ServeMux mux.HandleFunc("DELETE /akoflow-api/environments/{environmentId}/", http_config.KernelHandler(workflowEngine.DeleteEnvironment)) mux.HandleFunc("GET /akoflow-api/environments/{environmentId}/storages/", http_config.KernelHandler(workflowEngine.ListStorages)) mux.HandleFunc("GET /akoflow-api/storages/{storageId}/roots/", http_config.KernelHandler(workflowEngine.StorageRoots)) + mux.HandleFunc("GET /akoflow-api/storages/{storageId}/workspace-budget/", http_config.KernelHandler(workflowEngine.WorkspaceStorageBudget)) mux.HandleFunc("GET /akoflow-api/storages/{storageId}/entries/", http_config.KernelHandler(workflowEngine.BrowseStorage)) mux.HandleFunc("DELETE /akoflow-api/storages/{storageId}/entries/", http_config.KernelHandler(workflowEngine.DeleteStorageEntry)) mux.HandleFunc("GET /akoflow-api/storages/{storageId}/entry/", http_config.KernelHandler(workflowEngine.StatStorageEntry)) @@ -196,6 +198,7 @@ func registerExecutionRoutes(mux *http.ServeMux, workflowEngine *workflow_engine mux.HandleFunc("POST /akoflow-api/execution-runs/{runId}/cancel/", http_config.KernelHandler(workflowEngine.CancelExecution)) mux.HandleFunc("GET /akoflow-api/execution-runs/{runId}/recovery/", http_config.KernelHandler(workflowEngine.GetExecutionRecovery)) mux.HandleFunc("POST /akoflow-api/execution-runs/{runId}/recovery/", http_config.KernelHandler(workflowEngine.RecoverExecution)) + mux.HandleFunc("POST /akoflow-api/execution-runs/{runId}/preserve-cloud-workspaces/", http_config.KernelHandler(workflowEngine.PreserveCancelledCloudWorkspaces)) mux.HandleFunc("POST /akoflow-api/execution-runs/{runId}/activities/{activityId}/interrupt/", http_config.KernelHandler(workflowEngine.InterruptExecutionActivity)) mux.HandleFunc("GET /akoflow-api/execution-runs/{runId}/activity-metrics/", http_config.KernelHandler(workflowEngine.ListActivityMetricSummaries)) mux.HandleFunc("GET /akoflow-api/execution-runs/{runId}/activities/{activityId}/metrics/", http_config.KernelHandler(workflowEngine.ListActivityMetricSamples)) diff --git a/internal/api/requests/workflow.go b/internal/api/requests/workflow.go index efb1aba4..d76eb50a 100644 --- a/internal/api/requests/workflow.go +++ b/internal/api/requests/workflow.go @@ -46,6 +46,14 @@ func FromDomain(definition domain.WorkflowDefinition) Workflow { DependsOn: predecessors, Simulation: activity.Simulation, } + if activity.ActivityTypeID != definition.ID+"-activity" { + item.ActivityType = strings.TrimPrefix(activity.ActivityTypeID, definition.ID+"-") + } + item.Metadata = map[string]any{} + for key, value := range metadata { + item.Metadata[key] = value + } + delete(item.Metadata, "workspaceProjectionVolumes") // derived for each placement if metadata != nil { item.Runtime, _ = metadata["runtime"].(string) item.ResourceSelector, _ = metadata["resourceSelector"].(string) @@ -104,6 +112,8 @@ type StoragePolicy struct { type WorkflowActivity struct { Name string `json:"name"` + ActivityType string `json:"activityType,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` Command domain.ActivityCommand `json:"command"` Image string `json:"image,omitempty"` Runtime string `json:"runtime,omitempty"` @@ -143,7 +153,25 @@ func (request Workflow) Domain() (domain.WorkflowDefinition, error) { names[value.Name] = workflowID + "-" + id } for index, value := range request.Spec.Activities { - activity, err := activityDomain(value, names[value.Name], request.Spec.Image, versionID, typeID, index) + activityTypeID := typeID + if value.ActivityType != "" { + family := identifier(value.ActivityType) + if family == "" { + return domain.WorkflowDefinition{}, fmt.Errorf("activity %q has an empty activityType", value.Name) + } + activityTypeID = workflowID + "-" + family + found := false + for _, existing := range definition.Types { + if existing.ID == activityTypeID { + found = true + break + } + } + if !found { + definition.Types = append(definition.Types, domain.ActivityType{ID: activityTypeID, Name: value.ActivityType, DefaultImage: request.Spec.Image}) + } + } + activity, err := activityDomain(value, names[value.Name], request.Spec.Image, versionID, activityTypeID, index) if err != nil { return domain.WorkflowDefinition{}, err } @@ -190,7 +218,7 @@ func activityDomain(value WorkflowActivity, activityID, defaultImage, versionID, command.Arguments = []string{"-c", value.Run} } simulationActivity := value.Simulation != nil - if command.Executable == nil && !simulationActivity { + if command.Executable == nil { image := value.Image if image == "" { image = defaultImage @@ -210,10 +238,21 @@ func activityDomain(value WorkflowActivity, activityID, defaultImage, versionID, return domain.Activity{}, fmt.Errorf("activity %q: %w", value.Name, err) } } - capabilities := []domain.ActivityCapability{domain.ActivityCapabilityReal} + capabilities := make([]domain.ActivityCapability, 0, 2) + if command.Entrypoint != "" && command.Executable != nil { + capabilities = append(capabilities, domain.ActivityCapabilityReal) + } if simulationActivity { - capabilities = []domain.ActivityCapability{domain.ActivityCapabilitySimulation} + capabilities = append(capabilities, domain.ActivityCapabilitySimulation) + } + metadata := map[string]any{} + for key, value := range value.Metadata { + metadata[key] = value } + delete(metadata, "workspaceProjectionVolumes") // executor recomputes from assignment + metadata["runtime"], metadata["resourceSelector"] = value.Runtime, value.ResourceSelector + metadata["keepDisk"], metadata["mountPath"] = value.KeepDisk, value.MountPath + metadata["workspaceSeedPath"], metadata["expectedOutputs"] = value.WorkspaceSeedPath, value.ExpectedOutputs return domain.Activity{ ID: activityID, WorkflowVersionID: versionID, ActivityTypeID: typeID, ExternalID: identifier(value.Name), Name: value.Name, Kind: domain.ActivityKindTask, @@ -222,9 +261,7 @@ func activityDomain(value WorkflowActivity, activityID, defaultImage, versionID, Resources: domain.ActivityResources{CPU: cpu, MemoryBytes: memory}, Simulation: value.Simulation, Policy: domain.ActivityPolicy{TimeoutSeconds: 3600, MaxAttempts: 1}, Priority: len(value.DependsOn) + index, - Metadata: map[string]any{"runtime": value.Runtime, "resourceSelector": value.ResourceSelector, - "keepDisk": value.KeepDisk, "mountPath": value.MountPath, "workspaceSeedPath": value.WorkspaceSeedPath, - "expectedOutputs": value.ExpectedOutputs}, + Metadata: metadata, }, nil } diff --git a/internal/api/requests/workflow_calibration_test.go b/internal/api/requests/workflow_calibration_test.go new file mode 100644 index 00000000..8fdeccf4 --- /dev/null +++ b/internal/api/requests/workflow_calibration_test.go @@ -0,0 +1,32 @@ +package requests + +import ( + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/stretchr/testify/require" + "testing" +) + +func TestWorkflowCalibrationFamiliesAndProjectionSurvivePortableRoundTrip(t *testing.T) { + request := Workflow{Name: "Measured Montage", Spec: WorkflowSpec{Namespace: "experiment", Activities: []WorkflowActivity{ + { + Name: "project", + ActivityType: "mproject", + Simulation: &domain.ActivitySimulation{DurationSeconds: 100}, + Metadata: map[string]any{"workspaceProjection": map[string]any{"contract": "whole-workspace-v1"}, "workspaceProjectionVolumes": "stale", "workspaceSeedPath": "untrusted"}, + WorkspaceSeedPath: "verified", + }, + {Name: "diff", ActivityType: "mdifffit", Simulation: &domain.ActivitySimulation{DurationSeconds: 10}, DependsOn: []string{"project"}}, + }}} + definition, err := request.Domain() + require.NoError(t, err) + require.Equal(t, "measured-montage-mproject", definition.Version.Activities[0].ActivityTypeID) + require.Equal(t, "measured-montage-mdifffit", definition.Version.Activities[1].ActivityTypeID) + require.Contains(t, definition.Version.Activities[0].Metadata, "workspaceProjection") + require.NotContains(t, definition.Version.Activities[0].Metadata, "workspaceProjectionVolumes") + require.Equal(t, "verified", definition.Version.Activities[0].Metadata["workspaceSeedPath"]) + reimported, err := FromDomain(definition).Domain() + require.NoError(t, err) + require.Equal(t, definition.Version.Activities[0].ActivityTypeID, reimported.Version.Activities[0].ActivityTypeID) + require.Equal(t, definition.Version.Activities[0].Metadata["workspaceProjection"], reimported.Version.Activities[0].Metadata["workspaceProjection"]) + require.Len(t, reimported.Version.Dependencies, 1) +} diff --git a/internal/api/requests/workflow_test.go b/internal/api/requests/workflow_test.go index c971c001..a77b7b6c 100644 --- a/internal/api/requests/workflow_test.go +++ b/internal/api/requests/workflow_test.go @@ -111,3 +111,22 @@ func TestSimulationWorkflowPreservesDurationsAndDataDependencies(t *testing.T) { exported := FromDomain(definition) require.Equal(t, int64(10_000_000_000), exported.Spec.DataDependencies[0].SizeBytes) } + +func TestWorkflowWithExecutableAndSimulationSupportsRealAndSimulatedRuns(t *testing.T) { + definition, err := (Workflow{Name: "calibrated-real-flow", Spec: WorkflowSpec{ + Namespace: "science", Image: "docker.io/library/busybox:1.36", + Activities: []WorkflowActivity{{ + Name: "compute", Run: "echo ready", + Simulation: &domain.ActivitySimulation{Model: "deterministic", DurationSeconds: 12}, + }}, + }}).Domain() + require.NoError(t, err) + activity := definition.Version.Activities[0] + require.Equal(t, []domain.ActivityCapability{ + domain.ActivityCapabilityReal, + domain.ActivityCapabilitySimulation, + }, activity.Capabilities) + require.NotNil(t, activity.Command.Executable) + require.Equal(t, "docker.io/library/busybox:1.36", activity.Command.Executable.Source.Reference) + require.Equal(t, 12.0, activity.Simulation.DurationSeconds) +} diff --git a/internal/application/cloudprovision/billing_test.go b/internal/application/cloudprovision/billing_test.go index b37acfdf..6a680193 100644 --- a/internal/application/cloudprovision/billing_test.go +++ b/internal/application/cloudprovision/billing_test.go @@ -39,3 +39,17 @@ func TestInstanceBillingPauseAndResume(t *testing.T) { t.Fatalf("cost = %v, want %v", instance.Billing.CurrentCost, want) } } + +func TestBillingDistinguishesMissingPricesFromFreeCompute(t *testing.T) { + missing := billingForTarget(domain.CloudCapacityTarget{Configuration: map[string]any{"diskSizeGiB": 30}}, time.Now()) + if missing.PricingStatus != "incomplete" || len(missing.MissingPrices) != 2 { + t.Fatalf("missing tariffs presented as calibrated: %+v", missing) + } + priced := billingForTarget(domain.CloudCapacityTarget{Configuration: map[string]any{ + "pricePerHour": 0.032, "diskSizeGiB": 30, "diskPricePerGiBMonth": 0.04, + "priceCurrency": "USD", "priceSource": "gcp-public-catalog", + }}, time.Now()) + if priced.PricingStatus != "estimated" || len(priced.MissingPrices) != 0 || priced.PriceCurrency != "USD" || priced.PriceSource != "gcp-public-catalog" { + t.Fatalf("tariff provenance lost: %+v", priced) + } +} diff --git a/internal/application/cloudprovision/failed_stop_test.go b/internal/application/cloudprovision/failed_stop_test.go new file mode 100644 index 00000000..2faf8d96 --- /dev/null +++ b/internal/application/cloudprovision/failed_stop_test.go @@ -0,0 +1,29 @@ +package cloudprovision + +import ( + "context" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestStopFailedConfigurationPreservesDiskAndFailureEvidence(t *testing.T) { + instance := domain.CloudProvisionedInstance{ID: "vm", Status: "failed", ProviderID: "123", FailureReason: "host key unavailable", PublicAddress: "203.0.113.10", Disk: map[string]any{"name": "retained-disk"}} + store := &capacityStoreStub{instances: []domain.CloudProvisionedInstance{instance}} + terraform := &terraformStub{} + service := &Provisioner{store: store, terraform: terraform} + stopped, err := service.Stop(context.Background(), "vm") + if err != nil || stopped.Status != "stopped" || stopped.Disk["name"] != "retained-disk" || stopped.FailureReason != instance.FailureReason || len(terraform.stopped) != 1 || len(terraform.destroyed) != 0 { + t.Fatalf("unsafe stop: %+v %v", stopped, err) + } +} + +func TestStopRejectsActiveOrUnmaterializedFailedInstance(t *testing.T) { + for _, status := range []string{"provisioning", "configuring", "failed"} { + store := &capacityStoreStub{instances: []domain.CloudProvisionedInstance{{ID: "vm", Status: status}}} + terraform := &terraformStub{} + if _, err := (&Provisioner{store: store, terraform: terraform}).Stop(context.Background(), "vm"); err == nil || len(terraform.stopped) != 0 { + t.Fatalf("accepted %s", status) + } + } +} diff --git a/internal/application/cloudprovision/host_trust_test.go b/internal/application/cloudprovision/host_trust_test.go new file mode 100644 index 00000000..0ae9ec64 --- /dev/null +++ b/internal/application/cloudprovision/host_trust_test.go @@ -0,0 +1,51 @@ +package cloudprovision + +import ( + "context" + "errors" + "testing" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" +) + +type hostVerifierStub struct { + err error + calls int +} + +func (v *hostVerifierStub) Verify(_ context.Context, _ domain.EnvironmentConnection, _ []byte, instance domain.CloudProvisionedInstance) (ports.CloudHostTrust, error) { + v.calls++ + if instance.ProviderID != "123" { + return ports.CloudHostTrust{}, errors.New("wrong generation supplied to verifier") + } + return ports.CloudHostTrust{Alias: "generation-123", KnownHostsFile: "verified-hosts"}, v.err +} + +func TestProvisionRequiresHostVerificationBeforeReady(t *testing.T) { + for _, reject := range []bool{false, true} { + store := &capacityStoreStub{target: &domain.CloudCapacityTarget{ID: "target", EnvironmentID: "environment", Provider: "gcp"}} + environment := &environmentCatalogStub{definition: domain.EnvironmentDefinition{Connections: []domain.EnvironmentConnection{{Type: domain.ConnectionCloud, CredentialRef: "credential"}}}} + verifier := &hostVerifierStub{} + if reject { + verifier.err = errors.New("unverified host key") + } + service := New(store, environment, credentialStub{}, sshKeyStub{}, &terraformStub{result: ports.TerraformResult{ProviderID: "123", PublicAddress: "203.0.113.10"}}, configuratorStub{}).WithHostTrustVerifier(verifier) + instance, err := service.Provision(context.Background(), "environment", domain.CloudProvisionRequest{CapacityTargetID: "target"}) + if verifier.calls != 1 { + t.Fatalf("verification calls=%d", verifier.calls) + } + if reject { + if err == nil || instance.Status != "failed" { + t.Fatalf("unverified instance was accepted: %s %v", instance.Status, err) + } + } else { + if err != nil || instance.Status != "ready" { + t.Fatalf("verified provision: %s %v", instance.Status, err) + } + if _, _, err := instance.VerifiedSSHTrust(); err != nil { + t.Fatal(err) + } + } + } +} diff --git a/internal/application/cloudprovision/release_guard_test.go b/internal/application/cloudprovision/release_guard_test.go new file mode 100644 index 00000000..5c67a41a --- /dev/null +++ b/internal/application/cloudprovision/release_guard_test.go @@ -0,0 +1,42 @@ +package cloudprovision + +import ( + "context" + "errors" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +type guardedReleaseStore struct { + capacityStoreStub + blocked error + checked []string +} + +func (s *guardedReleaseStore) CheckCloudLifecycle(_ context.Context, id, excluded string) error { + s.checked = append(s.checked, id) + if excluded != "" { + return errors.New("direct release must not exclude an operation") + } + return s.blocked +} + +func TestDirectReleaseHonorsLifecycleGuard(t *testing.T) { + for _, policy := range []string{"destroy-after-run", "destroy-after-execution", "stop-when-idle"} { + t.Run(policy, func(t *testing.T) { + store := &guardedReleaseStore{capacityStoreStub: capacityStoreStub{ + target: &domain.CloudCapacityTarget{ID: "target", EnvironmentID: "environment", LifecyclePolicy: policy}, + instances: []domain.CloudProvisionedInstance{{ID: "producer", CapacityTargetID: "target", Status: "ready"}}, + }, blocked: errors.New("producer still has consumers")} + terraform := &terraformStub{} + service := &Provisioner{store: store, terraform: terraform} + if err := service.Release(context.Background(), []string{"target"}); err == nil { + t.Fatal("unsafe release was accepted") + } + if len(store.checked) != 1 || len(terraform.destroyed) != 0 || len(terraform.stopped) != 0 || len(store.updated) != 0 { + t.Fatal("release changed infrastructure before safety validation") + } + }) + } +} diff --git a/internal/application/cloudprovision/service.go b/internal/application/cloudprovision/service.go index 445fd939..3700dccd 100644 --- a/internal/application/cloudprovision/service.go +++ b/internal/application/cloudprovision/service.go @@ -21,6 +21,44 @@ type Provisioner struct { sshKeys ports.CloudSSHKeyManager terraform ports.TerraformRunner configurator ports.MachineConfigurator + hostVerifier ports.CloudHostTrustVerifier +} + +func (s *Provisioner) WithHostTrustVerifier(verifier ports.CloudHostTrustVerifier) *Provisioner { + s.hostVerifier = verifier + return s +} + +func (s *Provisioner) verifyHostTrust(ctx context.Context, instance *domain.CloudProvisionedInstance) error { + if s.hostVerifier == nil { + return nil + } + definition, err := s.environments.Find(ctx, instance.EnvironmentID) + if err != nil || definition == nil { + return fmt.Errorf("load environment for host verification: %w", err) + } + connection := cloudConnection(definition.Connections) + if connection == nil { + return fmt.Errorf("cloud connection unavailable for host verification") + } + credential, err := s.credentials.Resolve(connection.CredentialRef) + if err != nil { + return err + } + trust, err := s.hostVerifier.Verify(ctx, *connection, credential, *instance) + if err != nil { + return fmt.Errorf("verify cloud SSH identity: %w", err) + } + if trust.Alias == "" || trust.KnownHostsFile == "" { + return fmt.Errorf("cloud SSH identity verifier returned incomplete trust") + } + if instance.TerraformOutput == nil { + instance.TerraformOutput = map[string]any{} + } + instance.TerraformOutput["verifiedHostKeyAlias"] = trust.Alias + instance.TerraformOutput["verifiedHostGeneration"] = instance.ProviderID + instance.TerraformOutput["verifiedKnownHostsFile"] = trust.KnownHostsFile + return nil } func New( @@ -236,6 +274,9 @@ func (s *Provisioner) configure( configurations []domain.CloudTargetConfiguration, ) error { fingerprint := configurationFingerprint(configurations) + if err := s.verifyHostTrust(ctx, instance); err != nil { + return err + } if current, _ := instance.TerraformOutput["configurationFingerprint"].(string); current != "" && current == fingerprint { if err := s.validate(ctx, instance, target, configurations); err == nil { return nil @@ -260,6 +301,7 @@ func (s *Provisioner) configure( variables[key] = value } err = s.configurator.Configure(ctx, ports.MachineConfigurationSpec{ + HostKeyAlias: hostTrustString(instance, "verifiedHostKeyAlias"), KnownHostsFile: hostTrustString(instance, "verifiedKnownHostsFile"), InstanceID: instance.ID, Address: instance.PublicAddress, SSHUser: instance.SSHUsername, CredentialRef: instance.SSHCredentialRef, PlaybookYAML: version.PlaybookYAML, Variables: variables, @@ -306,13 +348,26 @@ func (s *Provisioner) validate(ctx context.Context, instance *domain.CloudProvis if err := validateCompatibility(*version, target, assignment.Required); err != nil { return err } - if err := validator.Validate(ctx, ports.MachineConfigurationSpec{InstanceID: instance.ID, Address: instance.PublicAddress, SSHUser: instance.SSHUsername, CredentialRef: instance.SSHCredentialRef, Checks: version.ValidationChecks}); err != nil && assignment.Required { + if err := validator.Validate(ctx, ports.MachineConfigurationSpec{ + HostKeyAlias: hostTrustString(instance, "verifiedHostKeyAlias"), + KnownHostsFile: hostTrustString(instance, "verifiedKnownHostsFile"), + InstanceID: instance.ID, + Address: instance.PublicAddress, + SSHUser: instance.SSHUsername, + CredentialRef: instance.SSHCredentialRef, + Checks: version.ValidationChecks, + }); err != nil && assignment.Required { return err } } return nil } +func hostTrustString(instance *domain.CloudProvisionedInstance, key string) string { + value, _ := instance.TerraformOutput[key].(string) + return value +} + func (s *Provisioner) Configure(ctx context.Context, instanceID string) (domain.CloudProvisionedInstance, error) { instance, err := s.store.FindProvisionedInstance(ctx, instanceID) if err != nil || instance == nil { @@ -443,7 +498,11 @@ func (s *Provisioner) Stop(ctx context.Context, instanceID string) (domain.Cloud if instance.Status == "stopped" { return *instance, nil } - if instance.Status != "ready" { + // A required configuration can fail after a real VM has been created. + // Suspension preserves its disk; the operation handler still enforces + // concurrent-operation and activity/lease lifecycle guards before calling us. + failedConfiguration := instance.Status == "failed" && strings.TrimSpace(instance.ProviderID) != "" + if instance.Status != "ready" && !failedConfiguration { return *instance, fmt.Errorf("cloud instance %q cannot stop from status %q", instanceID, instance.Status) } if err := s.terraform.Stop(ctx, instanceID); err != nil { @@ -459,6 +518,9 @@ func (s *Provisioner) Validate(ctx context.Context, instanceID string) (domain.C if err != nil || instance == nil { return domain.CloudProvisionedInstance{}, fmt.Errorf("cloud instance %q was not found", instanceID) } + if err := s.verifyHostTrust(ctx, instance); err != nil { + return *instance, err + } target, err := s.store.FindCapacityTarget(ctx, instance.CapacityTargetID) if err != nil || target == nil { return *instance, targetError(instance.CapacityTargetID, err) @@ -466,6 +528,9 @@ func (s *Provisioner) Validate(ctx context.Context, instanceID string) (domain.C if err := s.validate(ctx, instance, *target, target.MachineConfigurations); err != nil { return *instance, err } + if err := s.store.UpdateProvisionedInstance(ctx, *instance); err != nil { + return *instance, err + } return *instance, nil } @@ -488,6 +553,13 @@ func (s *Provisioner) Release(ctx context.Context, capacityTargetIDs []string) e if instance.CapacityTargetID != targetID || instance.Status != "ready" { continue } + // Direct releases (including console closure) must honor the same + // lifecycle barrier as queued infrastructure operations. + if guard, ok := s.store.(ports.CloudLifecycleGuard); ok { + if err := guard.CheckCloudLifecycle(ctx, instance.ID, ""); err != nil { + return fmt.Errorf("release cloud capacity: %w", err) + } + } switch target.LifecyclePolicy { case "destroy-after-run", "destroy-after-execution": if _, err := s.Destroy(ctx, instance.ID); err != nil { @@ -512,11 +584,30 @@ func billingForTarget(target domain.CloudCapacityTarget, now time.Time) *domain. computeHourly := configurationNumber(target.Configuration, "pricePerHour") diskGiB := configurationNumber(target.Configuration, "diskSizeGiB") diskMonthly := configurationNumber(target.Configuration, "diskPricePerGiBMonth") - return &domain.InstanceBilling{ + billing := &domain.InstanceBilling{ StartedAt: now, ComputeStartedAt: &now, ComputePricePerSecond: max(0, computeHourly/3600), DiskPricePerSecond: max(0, diskGiB*diskMonthly/(730*3600)), + PricingStatus: "estimated", PriceCurrency: configurationString(target.Configuration, "priceCurrency"), + PriceSource: configurationString(target.Configuration, "priceSource"), + PriceObservedAt: configurationString(target.Configuration, "priceObservedAt"), + PriceRegion: target.Region, PriceMachineType: target.ProviderMachineType, + } + if computeHourly <= 0 { + billing.MissingPrices = append(billing.MissingPrices, "compute") } + if diskGiB > 0 && diskMonthly <= 0 { + billing.MissingPrices = append(billing.MissingPrices, "disk") + } + if len(billing.MissingPrices) > 0 { + billing.PricingStatus = "incomplete" + } + return billing +} + +func configurationString(values map[string]any, key string) string { + value, _ := values[key].(string) + return value } func configurationNumber(values map[string]any, key string) float64 { diff --git a/internal/application/console/service.go b/internal/application/console/service.go index 4fac8add..c193fbee 100644 --- a/internal/application/console/service.go +++ b/internal/application/console/service.go @@ -160,6 +160,10 @@ func (s *CommandController) resolveCloudConnection( if !connectable || !targetIDs[instance.CapacityTargetID] || address == "" { continue } + alias, knownHosts, err := instance.VerifiedSSHTrust() + if err != nil { + return connection, err + } return domain.EnvironmentConnection{ ID: connection.ID + "-" + instance.ID, EnvironmentID: connection.EnvironmentID, Name: instance.Name, Type: domain.ConnectionSSH, Endpoint: address, @@ -169,7 +173,7 @@ func (s *CommandController) resolveCloudConnection( // Public cloud addresses are routinely reused by a different VM. Key the // trust record by Akoflow's immutable instance id instead of that address, // so SSH still verifies the host without colliding with a former tenant. - "hostKeyAlias": instance.ID, + "hostKeyAlias": alias, "knownHostsFile": knownHosts, "strictHostKeyChecking": true, }, }, nil } diff --git a/internal/application/console/service_test.go b/internal/application/console/service_test.go index 5f94f2cb..31710581 100644 --- a/internal/application/console/service_test.go +++ b/internal/application/console/service_test.go @@ -149,6 +149,7 @@ func TestCommandControllerFailureAndValidation(t *testing.T) { func TestCloudConnectionIsAvailableWhileAnsibleConfigures(t *testing.T) { controller := &CommandController{cloud: cloudConfigurationStoreStub{instances: []domain.CloudProvisionedInstance{{ ID: "instance", CapacityTargetID: "capacity", Status: "configuring", + ProviderID: "123", TerraformOutput: map[string]any{"verifiedHostGeneration": "123", "verifiedHostKeyAlias": "verified-generation", "verifiedKnownHostsFile": "verified-hosts"}, PublicAddress: "203.0.113.10", SSHUsername: "akoflow", SSHCredentialRef: "file:key", }}}} connection, err := controller.resolveCloudConnection(context.Background(), domain.EnvironmentConnection{ @@ -160,7 +161,7 @@ func TestCloudConnectionIsAvailableWhileAnsibleConfigures(t *testing.T) { if connection.Type != domain.ConnectionSSH || connection.Endpoint != "203.0.113.10" { t.Fatalf("unexpected SSH connection %#v", connection) } - if connection.Configuration["hostKeyAlias"] != "instance" { + if connection.Configuration["hostKeyAlias"] != "verified-generation" || connection.Configuration["knownHostsFile"] != "verified-hosts" || connection.Configuration["strictHostKeyChecking"] != true { t.Fatalf("cloud host key must be scoped to the provisioned instance: %#v", connection.Configuration) } } diff --git a/internal/application/execution/cancellation_coordination_test.go b/internal/application/execution/cancellation_coordination_test.go new file mode 100644 index 00000000..02710a59 --- /dev/null +++ b/internal/application/execution/cancellation_coordination_test.go @@ -0,0 +1,62 @@ +package execution + +import ( + "context" + "errors" + "github.com/UFFeScience/akoflow/internal/domain" + "testing" +) + +func TestCancelledInspectionDoesNotPersistComputationFailure(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + runtime := &runtimeFake{inspectErr: context.Canceled} + handles := &handlesFake{handle: &domain.ActivityHandle{ID: "h", RuntimeID: "runtime", Status: domain.HandleRunning}} + h, err := New(resolverFake{adapter: runtime}, handles).Inspect(ctx, "h", domain.ExecutionModeReal) + if !errors.Is(err, context.Canceled) || h.Status != domain.HandleRunning || handles.handle.Status != domain.HandleRunning { + t.Fatalf("handle=%+v err=%v", h, err) + } +} + +type stoppingRuntime struct { + runtimeFake + entered chan struct{} + release chan struct{} + inspections int +} + +func (r *stoppingRuntime) Stop(context.Context, domain.ActivityHandle) error { + close(r.entered) + <-r.release + return nil +} +func (r *stoppingRuntime) Inspect(context.Context, domain.ActivityHandle) (domain.ActivityHandle, error) { + r.inspections++ + return domain.ActivityHandle{}, nil +} + +func TestInspectDoesNotQueryRemovedRuntimeDuringStop(t *testing.T) { + runtime := &stoppingRuntime{entered: make(chan struct{}), release: make(chan struct{})} + handles := &handlesFake{handle: &domain.ActivityHandle{ID: "h", RuntimeID: "runtime", Status: domain.HandleRunning}} + controller := New(resolverFake{adapter: runtime}, handles) + stopped := make(chan error, 1) + go func() { stopped <- controller.Stop(context.Background(), "h", domain.ExecutionModeReal) }() + <-runtime.entered + inspected := make(chan *domain.ActivityHandle, 1) + go func() { + h, err := controller.Inspect(context.Background(), "h", domain.ExecutionModeReal) + if err != nil { + inspected <- nil + return + } + inspected <- h + }() + close(runtime.release) + if err := <-stopped; err != nil { + t.Fatal(err) + } + h := <-inspected + if h == nil || h.Status != domain.HandleStopped || runtime.inspections != 0 { + t.Fatalf("handle=%+v runtime inspections=%d", h, runtime.inspections) + } +} diff --git a/internal/application/execution/controller.go b/internal/application/execution/controller.go index 71fb10e6..f7ebca7f 100644 --- a/internal/application/execution/controller.go +++ b/internal/application/execution/controller.go @@ -3,8 +3,10 @@ package execution import ( "context" "fmt" + "hash/fnv" "net/url" "strings" + "sync" "time" "github.com/UFFeScience/akoflow/internal/application/ports" @@ -15,6 +17,17 @@ type Controller struct { runtimes ports.RuntimeResolver handles ports.ActivityExecutionStore data ports.DataCatalog + // Fixed stripes serialize lifecycle operations on the same handle without + // retaining an unbounded lock entry for every scientific activity. + handleLocks [256]sync.Mutex +} + +func (s *Controller) lockHandle(id string) func() { + h := fnv.New32a() + _, _ = h.Write([]byte(id)) + lock := &s.handleLocks[h.Sum32()%uint32(len(s.handleLocks))] + lock.Lock() + return lock.Unlock } func New(runtimes ports.RuntimeResolver, handles ports.ActivityExecutionStore, catalogs ...ports.DataCatalog) *Controller { @@ -32,22 +45,8 @@ func (s *Controller) Start(ctx context.Context, execution domain.ActivityExecuti // This is intentionally enforced at the final execution boundary as well as // by the supervisor. A caller cannot bypass a failed materialization by // constructing an ActivityExecutionContext directly. - if execution.Preparation != nil { - if err := execution.Preparation.Ready(); err != nil { - return domain.ActivityHandle{}, fmt.Errorf("activity preparation is not ready: %w", err) - } - if executable := execution.Preparation.Executable; executable != nil { - execution.Activity.Command.ResolvedExecutable = &domain.ResolvedExecutable{ - VariantID: executable.VariantID, Digest: executable.Digest, LocalPath: executable.DestinationPath, - EnvironmentID: executable.EnvironmentID, ResourceID: executable.ResourceID, - MaterializationID: executable.ID, MaterializationDone: executable.Committed(), - } - } - if workspace := execution.Preparation.Workspace; workspace != nil && execution.Activity.Command.WorkingDirectory == "" { - if destination, parseErr := url.Parse(workspace.Destination.URI); parseErr == nil && destination.Path != "" { - execution.Activity.Command.WorkingDirectory = destination.Path - } - } + if err := prepareActivityCommand(&execution); err != nil { + return domain.ActivityHandle{}, err } capability, err := capabilityFor(execution.Run.Mode) if err != nil { @@ -56,12 +55,17 @@ func (s *Controller) Start(ctx context.Context, execution domain.ActivityExecuti if !execution.Activity.Supports(capability) { return domain.ActivityHandle{}, fmt.Errorf("activity %q does not support %q execution", execution.Activity.ID, execution.Run.Mode) } - handleID := execution.Run.ID + ":" + execution.Activity.ID + handleID := execution.HandleID() + unlock := s.lockHandle(handleID) + defer unlock() existing, err := s.handles.Find(ctx, handleID) if err != nil { return domain.ActivityHandle{}, fmt.Errorf("find activity handle: %w", err) } if existing != nil { + if existing.Metadata[ports.RuntimeReconciliationPending] == true { + return *existing, ports.NewRuntimeReconciliationError(fmt.Errorf("runtime cleanup is not confirmed"), existing.Metadata) + } return *existing, nil } if execution.RuntimeID == "" { @@ -73,7 +77,7 @@ func (s *Controller) Start(ctx context.Context, execution domain.ActivityExecuti } handle, err := runtime.Start(ctx, execution) if err != nil { - return domain.ActivityHandle{}, err + return s.persistUncertainStart(ctx, execution, err) } if handle.ActivityID != execution.Activity.ID || handle.RunID != execution.Run.ID { return domain.ActivityHandle{}, fmt.Errorf("runtime returned an invalid activity handle") @@ -99,17 +103,26 @@ func (s *Controller) Start(ctx context.Context, execution domain.ActivityExecuti } } if err := s.handles.Save(ctx, handle); err != nil { - _ = runtime.Stop(ctx, handle) + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + if stopErr := runtime.Stop(cleanupCtx, handle); stopErr != nil { + return domain.ActivityHandle{}, fmt.Errorf("persist activity handle: %w; stop submitted runtime %q: %v", err, handle.ExternalID, stopErr) + } return domain.ActivityHandle{}, fmt.Errorf("persist activity handle: %w", err) } return handle, nil } func (s *Controller) Inspect(ctx context.Context, handleID string, mode domain.ExecutionMode) (*domain.ActivityHandle, error) { + unlock := s.lockHandle(handleID) + defer unlock() handle, err := s.handles.Find(ctx, handleID) if err != nil || handle == nil { return handle, err } + if handle.Status == domain.HandleStopped || handle.Status == domain.HandleCompleted || handle.Status == domain.HandleFailed { + return handle, nil + } runtime, err := s.runtimes.Resolve(mode, handle.RuntimeID) if err != nil { return s.persistInspectionFailure(ctx, handle, err) @@ -119,27 +132,7 @@ func (s *Controller) Inspect(ctx context.Context, handleID string, mode domain.E return s.persistInspectionFailure(ctx, handle, err) } updated.Log = mergeLogs(handle.Log, updated.Log) - if len(updated.Metrics) > 0 { - if metrics, ok := s.handles.(interface { - SaveActivityMetrics(context.Context, []domain.ActivityMetricSample) error - }); ok { - if metricErr := metrics.SaveActivityMetrics(ctx, updated.Metrics); metricErr != nil { - if updated.Metadata == nil { - updated.Metadata = make(map[string]any) - } - // Keep the previous cursor so the next inspection retries the - // same file rows instead of silently losing observations. - if previous, exists := handle.Metadata["metricsSampleCount"]; exists { - updated.Metadata["metricsSampleCount"] = previous - } else { - delete(updated.Metadata, "metricsSampleCount") - } - updated.Metadata["metricCollectionError"] = metricErr.Error() - } else { - delete(updated.Metadata, "metricCollectionError") - } - } - } + s.saveInspectionMetrics(ctx, handle, &updated) if updated.Status != handle.Status { switch updated.Status { case domain.HandleCompleted: @@ -184,7 +177,57 @@ func (s *Controller) Inspect(ctx context.Context, handleID string, mode domain.E return &updated, nil } +func prepareActivityCommand(execution *domain.ActivityExecutionContext) error { + if execution.Preparation == nil { + return nil + } + if err := execution.Preparation.Ready(); err != nil { + return fmt.Errorf("activity preparation is not ready: %w", err) + } + if executable := execution.Preparation.Executable; executable != nil { + execution.Activity.Command.ResolvedExecutable = &domain.ResolvedExecutable{ + VariantID: executable.VariantID, Digest: executable.Digest, + LocalPath: executable.DestinationPath, EnvironmentID: executable.EnvironmentID, + ResourceID: executable.ResourceID, MaterializationID: executable.ID, + MaterializationDone: executable.Committed(), + } + } + if workspace := execution.Preparation.Workspace; workspace != nil && execution.Activity.Command.WorkingDirectory == "" { + if destination, err := url.Parse(workspace.Destination.URI); err == nil && destination.Path != "" { + execution.Activity.Command.WorkingDirectory = destination.Path + } + } + return nil +} + +func (s *Controller) saveInspectionMetrics(ctx context.Context, previous *domain.ActivityHandle, updated *domain.ActivityHandle) { + metrics, ok := s.handles.(interface { + SaveActivityMetrics(context.Context, []domain.ActivityMetricSample) error + }) + if !ok || len(updated.Metrics) == 0 { + return + } + if err := metrics.SaveActivityMetrics(ctx, updated.Metrics); err != nil { + if updated.Metadata == nil { + updated.Metadata = make(map[string]any) + } + // Preserve the cursor so the next inspection retries the same rows. + if cursor, exists := previous.Metadata["metricsSampleCount"]; exists { + updated.Metadata["metricsSampleCount"] = cursor + } else { + delete(updated.Metadata, "metricsSampleCount") + } + updated.Metadata["metricCollectionError"] = err.Error() + } else { + delete(updated.Metadata, "metricCollectionError") + } +} + func (s *Controller) persistInspectionFailure(ctx context.Context, handle *domain.ActivityHandle, err error) (*domain.ActivityHandle, error) { + if ctx.Err() != nil { + // Observation cancellation is not evidence that computation failed. + return handle, ctx.Err() + } handle.Status = domain.HandleFailed handle.FinishedAt = float64(time.Now().UnixNano()) / float64(time.Second) handle.Failure = "inspect activity: " + err.Error() @@ -250,6 +293,8 @@ func failureMessage(failure string) string { } func (s *Controller) Stop(ctx context.Context, handleID string, mode domain.ExecutionMode) error { + unlock := s.lockHandle(handleID) + defer unlock() handle, err := s.handles.Find(ctx, handleID) if err != nil { return err @@ -257,6 +302,9 @@ func (s *Controller) Stop(ctx context.Context, handleID string, mode domain.Exec if handle == nil { return fmt.Errorf("activity handle %q not found", handleID) } + if handle.Status == domain.HandleStopped || handle.Status == domain.HandleCompleted { + return nil + } runtime, err := s.runtimes.Resolve(mode, handle.RuntimeID) if err != nil { return err @@ -264,8 +312,15 @@ func (s *Controller) Stop(ctx context.Context, handleID string, mode domain.Exec if err := runtime.Stop(ctx, *handle); err != nil { return err } - handle.Status = domain.HandleStopped - return s.handles.Save(ctx, *handle) + stopped := *handle + stopped.Status = domain.HandleStopped + stopped.Metadata = make(map[string]any, len(handle.Metadata)) + for key, value := range handle.Metadata { + if key != ports.RuntimeReconciliationPending { + stopped.Metadata[key] = value + } + } + return s.handles.Save(ctx, stopped) } func capabilityFor(mode domain.ExecutionMode) (domain.ActivityCapability, error) { diff --git a/internal/application/execution/controller_test.go b/internal/application/execution/controller_test.go index 9601203f..9edbf009 100644 --- a/internal/application/execution/controller_test.go +++ b/internal/application/execution/controller_test.go @@ -20,12 +20,13 @@ func (f resolverFake) Resolve(domain.ExecutionMode, string) (ports.RuntimeAdapte } type runtimeFake struct { - handle domain.ActivityHandle - stopped bool - starts int - startErr error - inspectErr error - stopErr error + handle domain.ActivityHandle + stopped bool + starts int + startErr error + inspectErr error + stopErr error + stopContextErr error } func (f *runtimeFake) Modes() []domain.ExecutionMode { @@ -42,8 +43,9 @@ func (f *runtimeFake) Inspect(context.Context, domain.ActivityHandle) (domain.Ac f.handle.Status = domain.HandleCompleted return f.handle, nil } -func (f *runtimeFake) Stop(context.Context, domain.ActivityHandle) error { +func (f *runtimeFake) Stop(ctx context.Context, _ domain.ActivityHandle) error { f.stopped = true + f.stopContextErr = ctx.Err() return f.stopErr } @@ -130,6 +132,24 @@ func TestStartStopsRuntimeWhenHandleCannotBePersisted(t *testing.T) { } } +func TestStartCleanupSurvivesCancelledPersistenceContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + runtime := &runtimeFake{handle: domain.ActivityHandle{RunID: "run", ActivityID: "activity", ExternalID: "123"}} + _, err := New(resolverFake{adapter: runtime}, &handlesFake{saveErr: context.Canceled}).Start(ctx, executionFixture()) + if err == nil || !runtime.stopped || runtime.stopContextErr != nil { + t.Fatalf("cleanup did not detach cancellation: %v %+v", err, runtime) + } +} + +func TestStartReportsCleanupFailure(t *testing.T) { + runtime := &runtimeFake{handle: domain.ActivityHandle{RunID: "run", ActivityID: "activity", ExternalID: "123"}, stopErr: errors.New("SSH unavailable")} + _, err := New(resolverFake{adapter: runtime}, &handlesFake{saveErr: context.Canceled}).Start(context.Background(), executionFixture()) + if err == nil || !strings.Contains(err.Error(), "SSH unavailable") || !strings.Contains(err.Error(), "123") { + t.Fatal(err) + } +} + func TestStartRejectsUnsupportedMode(t *testing.T) { execution := executionFixture() execution.Run.Mode = domain.ExecutionModeInteractive @@ -147,8 +167,8 @@ func TestInspectAndStop(t *testing.T) { if err != nil || updated.Status != domain.HandleCompleted { t.Fatal("inspect failed") } - if err := service.Stop(context.Background(), "handle", domain.ExecutionModeReal); err != nil || !runtime.stopped || handles.handle.Status != domain.HandleStopped { - t.Fatal("stop failed") + if err := service.Stop(context.Background(), "handle", domain.ExecutionModeReal); err != nil || runtime.stopped || handles.handle.Status != domain.HandleCompleted { + t.Fatal("stop must preserve completed scientific result") } } diff --git a/internal/application/execution/interruption.go b/internal/application/execution/interruption.go index 1cb07cec..14cbb813 100644 --- a/internal/application/execution/interruption.go +++ b/internal/application/execution/interruption.go @@ -51,28 +51,37 @@ func (s Interrupter) Interrupt(ctx context.Context, runID, activityID string) (* if task == nil { return nil, fmt.Errorf("activity %q has no execution record in run %q", activityID, runID) } - if task.Status == domain.TaskCancelled { - return task, nil - } - if task.Status == domain.TaskCompleted || task.Status == domain.TaskFailed { - return nil, fmt.Errorf("activity %q is already %s", activityID, task.Status) - } handles, err := s.Store.ListHandles(ctx, runID) if err != nil { return nil, err } var handle *domain.ActivityHandle + stopped := false for index := range handles { if handles[index].ActivityID == activityID { handle = &handles[index] - break + if handle.Status == domain.HandleStarting || handle.Status == domain.HandleRunning { + if err := s.Stopper.Stop(ctx, handle.ID, run.Mode); err != nil { + return nil, fmt.Errorf("stop activity %q handle %q on its runtime: %w", activityID, handle.ID, err) + } + stopped = true + } } } + if task.Status == domain.TaskCompleted || task.Status == domain.TaskFailed { + if stopped { + return task, nil + } + return nil, fmt.Errorf("activity %q is already %s", activityID, task.Status) + } if handle == nil { + if task.Status == domain.TaskCancelled { + return task, nil + } if run.Status == domain.ExecutionRunRunning { return nil, fmt.Errorf("activity %q has no runtime handle while the run is active; wait for the scheduler before interrupting", activityID) } - } else { + } else if !stopped { switch handle.Status { case domain.HandleStarting, domain.HandleRunning: if err := s.Stopper.Stop(ctx, handle.ID, run.Mode); err != nil { @@ -81,6 +90,9 @@ func (s Interrupter) Interrupt(ctx context.Context, runID, activityID string) (* case domain.HandleStopped: // A prior request stopped the runtime; reconcile the task record. default: + if task.Status == domain.TaskCancelled && (handle.Status == domain.HandleCompleted || handle.Status == domain.HandleFailed) { + return task, nil + } return nil, fmt.Errorf("activity %q has a %s runtime handle; inspect it before interrupting", activityID, handle.Status) } } diff --git a/internal/application/execution/interruption_test.go b/internal/application/execution/interruption_test.go index b4f388d5..5ce782ab 100644 --- a/internal/application/execution/interruption_test.go +++ b/internal/application/execution/interruption_test.go @@ -33,13 +33,33 @@ func (f *interruptionStoreFake) SaveTask(_ context.Context, task domain.TaskExec type interruptionStopperFake struct { called bool err error + ids []string } -func (f *interruptionStopperFake) Stop(context.Context, string, domain.ExecutionMode) error { +func (f *interruptionStopperFake) Stop(_ context.Context, id string, _ domain.ExecutionMode) error { f.called = true + f.ids = append(f.ids, id) return f.err } +func TestInterruptStopsEveryActiveAttemptIgnoringCompletedHistory(t *testing.T) { + store := &interruptionStoreFake{ + run: &domain.ExecutionRun{ID: "run", Status: domain.ExecutionRunFailed, Mode: domain.ExecutionModeReal}, + task: domain.TaskExecution{ActivityID: "activity", Status: domain.TaskCancelled}, + handles: []domain.ActivityHandle{ + {ID: "old", ActivityID: "activity", Status: domain.HandleCompleted}, + {ID: "attempt-2", ActivityID: "activity", Status: domain.HandleRunning}, + {ID: "attempt-3", ActivityID: "activity", Status: domain.HandleStarting}, + {ID: "other", ActivityID: "other", Status: domain.HandleRunning}, + }, + } + stopper := &interruptionStopperFake{} + _, err := (Interrupter{Store: store, Stopper: stopper}).Interrupt(context.Background(), "run", "activity") + if err != nil || len(stopper.ids) != 2 || stopper.ids[0] != "attempt-2" || stopper.ids[1] != "attempt-3" { + t.Fatalf("stopped=%v err=%v", stopper.ids, err) + } +} + func TestInterruptStopsRuntimeBeforeReleasingTask(t *testing.T) { store := &interruptionStoreFake{ run: &domain.ExecutionRun{ID: "run", Status: domain.ExecutionRunFailed, Mode: domain.ExecutionModeReal}, @@ -66,6 +86,19 @@ func TestInterruptKeepsTaskActiveWhenRuntimeStopFails(t *testing.T) { } } +func TestInterruptStopsHandleEvenWhenTaskWasAlreadyCancelled(t *testing.T) { + store := &interruptionStoreFake{ + run: &domain.ExecutionRun{ID: "run", Status: domain.ExecutionRunFailed, Mode: domain.ExecutionModeReal}, + task: domain.TaskExecution{ID: "task", ActivityID: "activity", Status: domain.TaskCancelled}, + handles: []domain.ActivityHandle{{ID: "run:activity", ActivityID: "activity", Status: domain.HandleStarting}}, + } + stopper := &interruptionStopperFake{} + _, err := (Interrupter{Store: store, Stopper: stopper}).Interrupt(context.Background(), "run", "activity") + if err != nil || !stopper.called { + t.Fatalf("stop called=%v err=%v", stopper.called, err) + } +} + func TestInterruptRequiresHandleWhileRunIsActive(t *testing.T) { store := &interruptionStoreFake{ run: &domain.ExecutionRun{ID: "run", Status: domain.ExecutionRunRunning}, diff --git a/internal/application/execution/recovery_handle_test.go b/internal/application/execution/recovery_handle_test.go new file mode 100644 index 00000000..a22d49b1 --- /dev/null +++ b/internal/application/execution/recovery_handle_test.go @@ -0,0 +1,43 @@ +package execution + +import ( + "context" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +type attemptHandleStore struct { + handles map[string]domain.ActivityHandle +} + +func (s *attemptHandleStore) Find(_ context.Context, id string) (*domain.ActivityHandle, error) { + h, ok := s.handles[id] + if !ok { + return nil, nil + } + return &h, nil +} + +func (s *attemptHandleStore) Save(_ context.Context, h domain.ActivityHandle) error { + s.handles[h.ID] = h + return nil +} + +func TestRecoveryStartsNewHandleAndKeepsSameAttemptIdempotent(t *testing.T) { + prior := domain.ActivityHandle{ID: "run:activity", RunID: "run", ActivityID: "activity", Status: domain.HandleFailed, Failure: "old transfer error"} + store := &attemptHandleStore{handles: map[string]domain.ActivityHandle{prior.ID: prior}} + runtime := &runtimeFake{handle: domain.ActivityHandle{RunID: "run", ActivityID: "activity", RuntimeID: "local", Status: domain.HandleRunning}} + controller := New(resolverFake{adapter: runtime}, store) + execution := executionFixture() + execution.Attempt = 2 + for i := 0; i < 2; i++ { + h, err := controller.Start(context.Background(), execution) + if err != nil || h.ID != "run:activity:attempt-2" || h.Status != domain.HandleRunning { + t.Fatalf("recovery returned stale handle: %+v, %v", h, err) + } + } + if runtime.starts != 1 || store.handles[prior.ID].Failure != prior.Failure { + t.Fatalf("lost history or repeated command: starts=%d handles=%+v", runtime.starts, store.handles) + } +} diff --git a/internal/application/execution/uncertain_start.go b/internal/application/execution/uncertain_start.go new file mode 100644 index 00000000..4c4d470a --- /dev/null +++ b/internal/application/execution/uncertain_start.go @@ -0,0 +1,31 @@ +package execution + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" +) + +func (s *Controller) persistUncertainStart(ctx context.Context, execution domain.ActivityExecutionContext, cause error) (domain.ActivityHandle, error) { + var uncertain ports.RuntimeReconciliationFailure + if !errors.As(cause, &uncertain) { + return domain.ActivityHandle{}, cause + } + handle := domain.ActivityHandle{ + ID: execution.HandleID(), RunID: execution.Run.ID, ActivityID: execution.Activity.ID, + ResourceID: execution.Resource.ID, RuntimeID: execution.RuntimeID, + Status: domain.HandleStarting, StartedAt: float64(time.Now().UnixNano()) / float64(time.Second), + Failure: cause.Error(), Metadata: uncertain.RuntimeReconciliationMetadata(), + } + handle.Metadata[ports.RuntimeReconciliationPending] = true + persist, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + if err := s.handles.Save(persist, handle); err != nil { + return handle, fmt.Errorf("persist uncertain runtime start: %v: %w", err, cause) + } + return handle, cause +} diff --git a/internal/application/execution/uncertain_start_test.go b/internal/application/execution/uncertain_start_test.go new file mode 100644 index 00000000..e53e4e90 --- /dev/null +++ b/internal/application/execution/uncertain_start_test.go @@ -0,0 +1,38 @@ +package execution + +import ( + "context" + "errors" + "testing" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestUncertainStartPersistsAndFencesDuplicateAttempt(t *testing.T) { + runtime := &runtimeFake{startErr: ports.NewRuntimeReconciliationError(errors.New("lost reply"), map[string]any{"identity": "attempt"})} + store := &handlesFake{} + controller := New(resolverFake{adapter: runtime}, store) + handle, err := controller.Start(context.Background(), executionFixture()) + if err == nil || handle.Status != domain.HandleStarting || store.handle == nil { + t.Fatalf("uncertain start lost: %+v %v", handle, err) + } + _, err = controller.Start(context.Background(), executionFixture()) + if err == nil || runtime.starts != 1 { + t.Fatal("duplicate attempt must remain fenced") + } + runtime.stopErr = errors.New("cleanup unconfirmed") + if controller.Stop(context.Background(), handle.ID, domain.ExecutionModeReal) == nil { + t.Fatal("unconfirmed cleanup accepted") + } + if store.handle.Status != domain.HandleStarting || store.handle.Metadata[ports.RuntimeReconciliationPending] != true { + t.Fatal("cleanup fence lost") + } + runtime.stopErr = nil + if err := controller.Stop(context.Background(), handle.ID, domain.ExecutionModeReal); err != nil { + t.Fatal(err) + } + if store.handle.Status != domain.HandleStopped || store.handle.Metadata[ports.RuntimeReconciliationPending] == true { + t.Fatal("confirmed cleanup not persisted") + } +} diff --git a/internal/application/planning/estimate.go b/internal/application/planning/estimate.go index 12ce4b05..b7a52be6 100644 --- a/internal/application/planning/estimate.go +++ b/internal/application/planning/estimate.go @@ -49,7 +49,7 @@ func estimateAlgorithmRun( coreEvaluations := 0 for _, resource := range compatible { - cores := resource.CPUCores + cores := resource.SchedulingCPUCores() if cores < 1 || opaquePlanningBatchTarget(resource) { cores = 1 } diff --git a/internal/application/planning/profile_snapshot.go b/internal/application/planning/profile_snapshot.go new file mode 100644 index 00000000..474ea8ed --- /dev/null +++ b/internal/application/planning/profile_snapshot.go @@ -0,0 +1,54 @@ +package planning + +import ( + "encoding/json" + "fmt" + "math" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +// A planning session freezes the measured profile entities independently of +// mutable catalog profiles, just as it freezes interference knowledge. This +// avoids replacing an environment and destroying its execution provenance. +func calibratedProfileSnapshot(raw any, workflow domain.WorkflowVersion, resources []domain.Resource) ([]domain.ActivityResourceProfile, error) { + if raw == nil { + return nil, nil + } + payload, err := json.Marshal(raw) + if err != nil { + return nil, err + } + var profiles []domain.ActivityResourceProfile + if err = json.Unmarshal(payload, &profiles); err != nil { + return nil, fmt.Errorf("decode activity profile snapshot: %w", err) + } + types, ids, pairs := map[string]bool{}, map[string]bool{}, map[string]bool{} + for _, activity := range workflow.Activities { + types[activity.ActivityTypeID] = true + } + for _, resource := range resources { + ids[resource.ID] = true + } + for _, profile := range profiles { + pair := profile.ActivityTypeID + "\x00" + profile.ResourceID + if !types[profile.ActivityTypeID] || !ids[profile.ResourceID] || pairs[pair] || profile.ID == "" || profile.Source == "" || profile.SampleSize < 1 || !profile.UsesWorkUnits() || profile.RuntimeSeconds <= 0 || math.IsInf( + profile.RuntimeSeconds, + 0, + ) || math.IsNaN( + profile.RuntimeSeconds, + ) { + return nil, fmt.Errorf("invalid or duplicate calibrated profile %q", profile.ID) + } + for _, activity := range workflow.Activities { + if activity.ActivityTypeID == profile.ActivityTypeID && profile.RuntimeFor(activity) <= 0 { + return nil, fmt.Errorf("calibrated profile requires valid %s work for activity %s", profile.ModelVersion, activity.ID) + } + } + pairs[pair] = true + } + if len(profiles) == 0 { + return nil, fmt.Errorf("empty calibrated profile snapshot") + } + return profiles, nil +} diff --git a/internal/application/planning/profile_snapshot_test.go b/internal/application/planning/profile_snapshot_test.go new file mode 100644 index 00000000..561467e9 --- /dev/null +++ b/internal/application/planning/profile_snapshot_test.go @@ -0,0 +1,34 @@ +package planning + +import ( + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/stretchr/testify/require" + "testing" +) + +func TestIndependentProfileSnapshotNeedsWorkRatherThanLegacySimulationDuration(t *testing.T) { + workflow := domain.WorkflowVersion{Activities: []domain.Activity{{ID: "a", ActivityTypeID: "family", Metadata: map[string]any{"computeWorkUnits": 6.0}}}} + profile := domain.ActivityResourceProfile{ID: "measured", ActivityTypeID: "family", ResourceID: "hpc", Source: "input-hash", ModelVersion: "compute-work-v1", RuntimeSeconds: 50, SampleSize: 4} + decoded, err := calibratedProfileSnapshot([]domain.ActivityResourceProfile{profile}, workflow, []domain.Resource{{ID: "hpc"}}) + require.NoError(t, err) + require.Equal(t, 300.0, decoded[0].RuntimeFor(workflow.Activities[0])) + delete(workflow.Activities[0].Metadata, "computeWorkUnits") + _, err = calibratedProfileSnapshot([]domain.ActivityResourceProfile{profile}, workflow, []domain.Resource{{ID: "hpc"}}) + require.Error(t, err) +} + +func TestProfileSnapshotRejectsStaleResourcesDuplicatePairsAndLegacyUnits(t *testing.T) { + workflow := domain.WorkflowVersion{Activities: []domain.Activity{{ID: "a", ActivityTypeID: "family", Simulation: &domain.ActivitySimulation{DurationSeconds: 10}}}} + resources := []domain.Resource{{ID: "micro"}} + profile := domain.ActivityResourceProfile{ID: "measured", ActivityTypeID: "family", ResourceID: "micro", Source: "baseline-hash", ModelVersion: "reference-work-v1", RuntimeSeconds: 2, SampleSize: 5} + decoded, err := calibratedProfileSnapshot([]domain.ActivityResourceProfile{profile}, workflow, resources) + require.NoError(t, err) + require.Equal(t, float64(20), decoded[0].RuntimeFor(workflow.Activities[0])) + _, err = calibratedProfileSnapshot([]domain.ActivityResourceProfile{profile, profile}, workflow, resources) + require.Error(t, err) + _, err = calibratedProfileSnapshot([]domain.ActivityResourceProfile{profile}, workflow, nil) + require.Error(t, err) + profile.ModelVersion = "legacy-absolute" + _, err = calibratedProfileSnapshot([]domain.ActivityResourceProfile{profile}, workflow, resources) + require.Error(t, err) +} diff --git a/internal/application/planning/service.go b/internal/application/planning/service.go index a4d9907f..237e6e82 100644 --- a/internal/application/planning/service.go +++ b/internal/application/planning/service.go @@ -15,6 +15,7 @@ import ( "github.com/UFFeScience/akoflow/internal/application/ports" "github.com/UFFeScience/akoflow/internal/domain" domainqueue "github.com/UFFeScience/akoflow/internal/domain/queue" + domainworkflow "github.com/UFFeScience/akoflow/internal/domain/workflow" ) const EventPlanningSessionRequested = "planning.session.requested" @@ -410,6 +411,9 @@ func (s *Coordinator) buildRequest( resources = append(resources, resource) } } + if err := applyWorkspaceBudgetSnapshot(resources, session.Configuration["workspaceStorageBudgetSnapshot"]); err != nil { + return domain.PlanningRequest{}, err + } definitions, err := s.Environments.List(ctx) if err != nil { return domain.PlanningRequest{}, err @@ -430,8 +434,17 @@ func (s *Coordinator) buildRequest( if err != nil { return domain.PlanningRequest{}, fmt.Errorf("decode interference knowledge: %w", err) } + projectedWorkflow, err := domainworkflow.ProjectWorkspaceDependencies(*workflow) + if err != nil { + return domain.PlanningRequest{}, fmt.Errorf("project workspace dependencies: %w", err) + } + if snapshot, snapshotErr := calibratedProfileSnapshot(session.Configuration["activityProfileSnapshot"], projectedWorkflow, resources); snapshotErr != nil { + return domain.PlanningRequest{}, snapshotErr + } else if snapshot != nil { + profiles = snapshot + } return domain.PlanningRequest{ - Workflow: *workflow, + Workflow: projectedWorkflow, ExecutionScope: *scope, Environments: environments, Resources: resources, @@ -444,6 +457,31 @@ func (s *Coordinator) buildRequest( }, nil } +func applyWorkspaceBudgetSnapshot(resources []domain.Resource, raw any) error { + if raw == nil { + return nil + } + payload, err := json.Marshal(raw) + if err != nil { + return err + } + var snapshots map[string]any + if err := json.Unmarshal(payload, &snapshots); err != nil { + return fmt.Errorf("decode workspace storage budgets: %w", err) + } + for i := range resources { + if budget, exists := snapshots[resources[i].ID]; exists { + metadata := map[string]any{} + for key, value := range resources[i].Metadata { + metadata[key] = value + } + metadata["workspaceStorageBudget"] = budget + resources[i].Metadata = metadata + } + } + return nil +} + func (s *Coordinator) Select( ctx context.Context, sessionID string, diff --git a/internal/application/ports/cloud.go b/internal/application/ports/cloud.go index d8ad3197..5f47452c 100644 --- a/internal/application/ports/cloud.go +++ b/internal/application/ports/cloud.go @@ -104,18 +104,29 @@ type CloudSSHKey struct { PublicKey string } +type CloudHostTrust struct { + Alias string + KnownHostsFile string +} + +type CloudHostTrustVerifier interface { + Verify(context.Context, domain.EnvironmentConnection, []byte, domain.CloudProvisionedInstance) (CloudHostTrust, error) +} + type CloudSSHKeyManager interface { Ensure(string, string) (CloudSSHKey, error) } type MachineConfigurationSpec struct { - InstanceID string - Address string - SSHUser string - CredentialRef string - PlaybookYAML string - Variables map[string]any - Checks []domain.MachineConfigurationValidationCheck + HostKeyAlias string + KnownHostsFile string + InstanceID string + Address string + SSHUser string + CredentialRef string + PlaybookYAML string + Variables map[string]any + Checks []domain.MachineConfigurationValidationCheck } type MachineConfigurator interface { diff --git a/internal/application/ports/runtime_reconciliation.go b/internal/application/ports/runtime_reconciliation.go new file mode 100644 index 00000000..6aa32791 --- /dev/null +++ b/internal/application/ports/runtime_reconciliation.go @@ -0,0 +1,41 @@ +package ports + +// RuntimeReconciliationPending marks a startup whose external outcome is not +// yet proven terminal. Such a handle must retain runtime cleanup ownership. +const RuntimeReconciliationPending = "runtimeReconciliationPending" + +// RuntimeReconciliationFailure carries bounded, non-sensitive correlation +// metadata when a runtime cannot establish the outcome of a startup request. +type RuntimeReconciliationFailure interface { + error + RuntimeReconciliationMetadata() map[string]any +} + +type runtimeReconciliationError struct { + cause error + metadata map[string]any +} + +func NewRuntimeReconciliationError(cause error, metadata map[string]any) error { + copy := make(map[string]any, len(metadata)+1) + for key, value := range metadata { + copy[key] = value + } + copy[RuntimeReconciliationPending] = true + return &runtimeReconciliationError{cause: cause, metadata: copy} +} + +func (e *runtimeReconciliationError) Error() string { + if e.cause == nil { + return "runtime outcome requires reconciliation" + } + return e.cause.Error() +} +func (e *runtimeReconciliationError) Unwrap() error { return e.cause } +func (e *runtimeReconciliationError) RuntimeReconciliationMetadata() map[string]any { + copy := make(map[string]any, len(e.metadata)) + for key, value := range e.metadata { + copy[key] = value + } + return copy +} diff --git a/internal/application/storage/cloud_catalog.go b/internal/application/storage/cloud_catalog.go new file mode 100644 index 00000000..6097ea1c --- /dev/null +++ b/internal/application/storage/cloud_catalog.go @@ -0,0 +1,48 @@ +package storage + +import ( + "context" + "fmt" + "strings" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" +) + +// CloudWorkspaceCatalog resolves a concrete generation, never a logical target +// or an old discovered local endpoint. The root excludes SSH credentials. +type CloudWorkspaceCatalog struct { + Catalog + Cloud ports.CloudConfigurationStore +} + +func (c CloudWorkspaceCatalog) FindStorage(ctx context.Context, id string) (*domain.StorageResource, error) { + const prefix = "cloud-workspace:" + if !strings.HasPrefix(id, prefix) { + return c.Catalog.FindStorage(ctx, id) + } + instanceID := strings.TrimPrefix(id, prefix) + if c.Cloud == nil || instanceID == "" { + return nil, fmt.Errorf("cloud workspace identity resolver is unavailable") + } + instance, err := c.Cloud.FindProvisionedInstance(ctx, instanceID) + if err != nil || instance == nil || instance.ID != instanceID || instance.Status != "ready" { + return nil, fmt.Errorf("concrete cloud workspace instance is unavailable") + } + if _, _, err := instance.VerifiedSSHTrust(); err != nil { + return nil, fmt.Errorf("cloud workspace trust: %w", err) + } + storage := &domain.StorageResource{ + ID: id, Type: domain.StorageSSH, Endpoint: "/akoflow/workspace", + BrowseRoots: []domain.StorageBrowseRoot{{Path: "/akoflow/workspace"}}, + Metadata: map[string]any{"cloudInstanceId": instance.ID}, + } + if registrar, ok := c.Catalog.(interface { + EnsureCloudWorkspaceStorage(context.Context, string, domain.StorageResource) error + }); ok { + if err := registrar.EnsureCloudWorkspaceStorage(ctx, instance.EnvironmentID, *storage); err != nil { + return nil, fmt.Errorf("register concrete cloud storage: %w", err) + } + } + return storage, nil +} diff --git a/internal/application/storage/cloud_catalog_test.go b/internal/application/storage/cloud_catalog_test.go new file mode 100644 index 00000000..74ac37a8 --- /dev/null +++ b/internal/application/storage/cloud_catalog_test.go @@ -0,0 +1,41 @@ +package storage + +import ( + "context" + "testing" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" +) + +type concreteCloudStore struct { + ports.CloudConfigurationStore + instance *domain.CloudProvisionedInstance + requested string +} + +func (s *concreteCloudStore) FindProvisionedInstance(_ context.Context, id string) (*domain.CloudProvisionedInstance, error) { + s.requested = id + return s.instance, nil +} + +func TestCloudWorkspaceCatalogRequiresExactReadyTrustedGeneration(t *testing.T) { + instance := &domain.CloudProvisionedInstance{ID: "vm", ProviderID: "generation", Status: "ready", + TerraformOutput: map[string]any{"verifiedHostKeyAlias": "alias", "verifiedKnownHostsFile": "known", "verifiedHostGeneration": "generation"}} + store := &concreteCloudStore{instance: instance} + catalog := CloudWorkspaceCatalog{Cloud: store} + storage, err := catalog.FindStorage(context.Background(), "cloud-workspace:vm") + if err != nil || store.requested != "vm" || storage.Endpoint != "/akoflow/workspace" || len(storage.BrowseRoots) != 1 { + t.Fatalf("storage=%v err=%v", storage, err) + } + for _, state := range []string{"destroyed", "stopped", "configuring"} { + instance.Status = state + if _, err := catalog.FindStorage(context.Background(), "cloud-workspace:vm"); err == nil { + t.Fatalf("accepted %s instance", state) + } + } + instance.Status, instance.ProviderID = "ready", "replacement" + if _, err := catalog.FindStorage(context.Background(), "cloud-workspace:vm"); err == nil { + t.Fatal("accepted trust from a different provider generation") + } +} diff --git a/internal/application/storage/service.go b/internal/application/storage/service.go index 4c9f96c9..328dc10f 100644 --- a/internal/application/storage/service.go +++ b/internal/application/storage/service.go @@ -16,6 +16,7 @@ import ( "github.com/UFFeScience/akoflow/internal/application/ports" "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/storagebudget" ) type Catalog interface { @@ -218,6 +219,24 @@ func (s *BrowserCoordinator) Roots(ctx context.Context, id string) ([]domain.Sto return v.BrowseRoots, nil } +func (s *BrowserCoordinator) WorkspaceBudget(ctx context.Context, id, path string) (storagebudget.Snapshot, error) { + value, err := s.catalog.FindStorage(ctx, id) + if err != nil || value == nil { + return storagebudget.Snapshot{}, notFound(err) + } + browser, err := s.usable(value, false) + if err != nil { + return storagebudget.Snapshot{}, err + } + observer, ok := browser.(interface { + WorkspaceBudget(context.Context, domain.StorageResource, string) (storagebudget.Snapshot, error) + }) + if !ok { + return storagebudget.Snapshot{}, fmt.Errorf("storage driver does not expose an effective workspace quota observer") + } + return observer.WorkspaceBudget(ctx, *value, path) +} + func (s *BrowserCoordinator) usable(v *domain.StorageResource, write bool) (ports.StorageBrowser, error) { if v == nil { return nil, fmt.Errorf("storage resource not found") @@ -433,7 +452,11 @@ func (s *BrowserCoordinator) archive(ctx context.Context, run domain.DownloadRun return } b, err := s.usable(&storage, true) - if err == nil { + if native, ok := b.(interface { + ArchiveDirectory(context.Context, domain.StorageResource, string) (string, error) + }); err == nil && ok { + run.Path, err = native.ArchiveDirectory(ctx, storage, run.Path) + } else if err == nil { reader, writer := io.Pipe() done := make(chan error, 1) go func() { done <- s.writeArchive(ctx, b, storage, run.Path, writer); _ = writer.Close() }() diff --git a/internal/application/transfer/coordinator_workspace_test.go b/internal/application/transfer/coordinator_workspace_test.go index 66461fa5..32f200f5 100644 --- a/internal/application/transfer/coordinator_workspace_test.go +++ b/internal/application/transfer/coordinator_workspace_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/UFFeScience/akoflow/internal/domain" + executiondomain "github.com/UFFeScience/akoflow/internal/domain/execution" ) type workspaceSyncStub struct { @@ -32,3 +33,35 @@ func TestWorkspacePreparationBlocksActivityOnFailedPredecessorSync(t *testing.T) t.Fatalf("transfer failure was not persisted: %+v", catalog.runs) } } + +func TestMixedDependencyRoutesPersistEvidenceBeforeRelease(t *testing.T) { + plans := []domain.DataTransferPlan{ + {ID: "hpc-edge", ExecutionRunID: "run", ProducerActivityID: "hpc-producer", ConsumerActivityID: "consumer", SyncWorkspace: true}, + {ID: "fog-edge", ExecutionRunID: "run", ProducerActivityID: "fog-producer", ConsumerActivityID: "consumer", SyncWorkspace: true}, + } + for _, test := range []struct { + name string + secondStatus executiondomain.TransferStatus + wantReady bool + }{ + {"all verified", domain.TransferCompleted, true}, + {"network still running", domain.TransferRunning, false}, + {"network failed", domain.TransferFailed, false}, + } { + t.Run(test.name, func(t *testing.T) { + runs := []domain.DataTransferRun{ + {ID: "hpc-edge", Status: domain.TransferCompleted, NetworkBytes: 0, Route: domain.TransferRoute{SharedFilesystem: "lustre", SharedFilesystemID: "verified-id", ProducerActivityID: "hpc-producer"}}, + {ID: "fog-edge", Status: test.secondStatus, NetworkBytes: 100, Route: domain.TransferRoute{ProducerActivityID: "fog-producer"}}, + } + catalog := &materializationCatalogStub{} + requirement := domain.PreparationRequirement{Workspace: &domain.WorkspaceMaterialization{ID: "workspace", Status: domain.MaterializationPlanned}, WorkspaceTransfers: plans} + gate, err := (Coordinator{Catalog: catalog, WorkspaceSyncer: &workspaceSyncStub{runs: runs}}).Prepare(context.Background(), "consumer", requirement) + if (err == nil && gate != nil) != test.wantReady { + t.Fatalf("mixed-route barrier: gate=%+v err=%v", gate, err) + } + if len(catalog.runs) != 4 || catalog.runs[2].Route.SharedFilesystemID != "verified-id" || catalog.runs[2].NetworkBytes != 0 || catalog.runs[3].NetworkBytes != 100 { + t.Fatalf("route evidence not persisted: %+v", catalog.runs) + } + }) + } +} diff --git a/internal/application/transfer/service.go b/internal/application/transfer/service.go index 6e7a0a76..1196daf6 100644 --- a/internal/application/transfer/service.go +++ b/internal/application/transfer/service.go @@ -89,15 +89,7 @@ func (m Materializer) Materialize(ctx context.Context, plan domain.DataTransferP if err != nil { return failed(target, run, err) } - strategy := plan.Strategy - route := plan.Route - if strategy == "" { - route = m.Strategies.Resolve(source, destination) - strategy = route.Strategy - } else if route.Strategy == "" { - route = domain.TransferRoute{Strategy: strategy, Fallback: domain.TransferGateway, Reason: "strategy explicitly selected by the transfer plan"} - } - route = routeWithEndpoints(route, plan) + strategy, route := m.materializationRoute(plan, source, destination) // Destination pull requires an agent on the destination. if strategy == domain.TransferDestinationPull { run.Strategy, run.Route = strategy, route @@ -156,7 +148,11 @@ func (m Materializer) Materialize(ctx context.Context, plan domain.DataTransferP }, func() error { // Re-check after taking ownership: another request may have committed // the same immutable object while this caller was waiting. - if ok, verifyErr := m.verify(ctx, dc, destination, final, blob.Digest); verifyErr == nil && ok { + ok, verifyErr := m.verify(ctx, dc, destination, final, blob.Digest) + if verifyErr != nil { + return fmt.Errorf("verify existing artifact: %w", verifyErr) + } + if ok { usedExisting = true return nil } @@ -182,7 +178,7 @@ func (m Materializer) Materialize(ctx context.Context, plan domain.DataTransferP if routed && sizeBytes > offset { run.TransferredBytes += sizeBytes - offset } - ok, verifyErr := m.verify(ctx, dc, destination, partial, blob.Digest) + ok, verifyErr = m.verify(ctx, dc, destination, partial, blob.Digest) if verifyErr != nil { return verifyErr } @@ -224,6 +220,17 @@ func (m Materializer) Materialize(ctx context.Context, plan domain.DataTransferP return target, run, nil } +func (m Materializer) materializationRoute(plan domain.DataTransferPlan, source, destination domain.TransferEndpoint) (domain.TransferStrategy, domain.TransferRoute) { + strategy, route := plan.Strategy, plan.Route + if strategy == "" { + route = m.Strategies.Resolve(source, destination) + strategy = route.Strategy + } else if route.Strategy == "" { + route = domain.TransferRoute{Strategy: strategy, Fallback: domain.TransferGateway, Reason: "strategy explicitly selected by the transfer plan"} + } + return strategy, routeWithEndpoints(route, plan) +} + func shortArtifactDigest(digest string) string { const visible = 12 if len(digest) <= visible { @@ -474,6 +481,11 @@ func (m Materializer) verify(ctx context.Context, c ports.TransferConnector, end if err != nil || !exists { return false, err } + if verifier, ok := c.(interface { + VerifyDigest(context.Context, domain.TransferEndpoint, string, string) (bool, error) + }); ok { + return verifier.VerifyDigest(ctx, endpoint, name, expected) + } r, err := c.Open(ctx, endpoint, name, 0) if err != nil { return false, err diff --git a/internal/application/transfer/service_test.go b/internal/application/transfer/service_test.go index 4677ad3d..98260ce4 100644 --- a/internal/application/transfer/service_test.go +++ b/internal/application/transfer/service_test.go @@ -21,6 +21,27 @@ type materializationCatalogStub struct { err error } +type checksumConnectorStub struct { + ports.TransferConnector + called bool +} + +func (*checksumConnectorStub) Exists(context.Context, domain.TransferEndpoint, string) (bool, error) { + return true, nil +} +func (s *checksumConnectorStub) VerifyDigest(context.Context, domain.TransferEndpoint, string, string) (bool, error) { + s.called = true + return true, nil +} + +func TestVerifyUsesRemoteChecksumWithoutOpeningContent(t *testing.T) { + connector := &checksumConnectorStub{} + ok, err := (Materializer{}).verify(context.Background(), connector, domain.TransferEndpoint{}, "artifact", "sha256:expected") + if err != nil || !ok || !connector.called { + t.Fatalf("ok=%v called=%v err=%v", ok, connector.called, err) + } +} + type sessionFilesystem struct { infra.LocalFilesystem begins int diff --git a/internal/application/transfer/verification_cache.go b/internal/application/transfer/verification_cache.go index 2fecb829..773ac9c4 100644 --- a/internal/application/transfer/verification_cache.go +++ b/internal/application/transfer/verification_cache.go @@ -114,6 +114,12 @@ func (c *VerifiedArtifactCache) DoObserved( } func verificationCacheKey(endpoint domain.TransferEndpoint, name, digest string) string { + // SSH artifacts are addressed on the connection host, not on the + // scheduler node. Different HPC nodes can share the same destination file. + if strings.HasPrefix(endpoint.URI, "ssh://") { + endpoint.ResourceID = "" + endpoint.RuntimeID = "" + } return strings.Join([]string{ endpoint.URI, endpoint.ResourceID, diff --git a/internal/application/transfer/verification_cache_test.go b/internal/application/transfer/verification_cache_test.go index a747bd81..2b6798c3 100644 --- a/internal/application/transfer/verification_cache_test.go +++ b/internal/application/transfer/verification_cache_test.go @@ -3,6 +3,7 @@ package transfer import ( "context" "errors" + "fmt" "sync" "sync/atomic" "testing" @@ -21,11 +22,13 @@ func TestVerifiedArtifactCacheCoalescesConcurrentMaterializations(t *testing.T) results := make(chan bool, 8) errorsFound := make(chan error, 8) var group sync.WaitGroup - for range 8 { + for node := range 8 { group.Add(1) go func() { defer group.Done() - shared, err := cache.Do(context.Background(), endpoint, "artifact.sif", "sha256:digest", func() error { + nodeEndpoint := endpoint + nodeEndpoint.ResourceID = fmt.Sprintf("scheduler-node-%d", node) + shared, err := cache.Do(context.Background(), nodeEndpoint, "artifact.sif", "sha256:digest", func() error { if calls.Add(1) == 1 { close(started) } @@ -84,6 +87,24 @@ func TestVerifiedArtifactCacheReleasesFailedMaterialization(t *testing.T) { } } +func TestVerifiedArtifactCacheSharesSSHArtifactAcrossSchedulerNodes(t *testing.T) { + cache := &VerifiedArtifactCache{} + first := domain.TransferEndpoint{URI: "ssh://cluster/artifacts", ConnectionID: "cluster", ResourceID: "diablo03", RuntimeID: "slurm"} + second := first + second.ResourceID = "bora001" + cache.Remember(first, "artifact.sif", "sha256:digest") + shared, err := cache.Do(context.Background(), second, "artifact.sif", "sha256:digest", func() error { + return errors.New("must not write the shared partial file again") + }) + if err != nil || !shared { + t.Fatalf("shared=%v err=%v", shared, err) + } + second.CloudInstanceID = "different-vm" + if cache.Has(second, "artifact.sif", "sha256:digest") { + t.Fatal("different VMs must not share verification") + } +} + func TestVerifiedArtifactCacheReportsInFlightOwnerBeforeWaiting(t *testing.T) { cache := &VerifiedArtifactCache{} endpoint := domain.TransferEndpoint{URI: "ssh://cluster/artifacts"} diff --git a/internal/controlplane/eventloop/eventloop.go b/internal/controlplane/eventloop/eventloop.go index f2df6779..a7261519 100644 --- a/internal/controlplane/eventloop/eventloop.go +++ b/internal/controlplane/eventloop/eventloop.go @@ -2,6 +2,7 @@ package eventloop import ( "context" + "errors" "fmt" "sync" "time" @@ -130,12 +131,15 @@ func splitCategories(configured []string) ([]string, []string) { } func (l *Loop) handle(ctx context.Context, job domainqueue.Job) { - jobCtx, cancel := context.WithCancel(ctx) - defer cancel() + jobCtx, cancel := context.WithCancelCause(ctx) + defer cancel(nil) leaseDone := make(chan struct{}) go l.keepLeaseAlive(jobCtx, job, cancel, leaseDone) err := l.dispatcher.Dispatch(jobCtx, job) - cancel() + if errors.Is(err, context.Canceled) && context.Cause(jobCtx) != nil { + err = fmt.Errorf("queue handler cancelled: %w", context.Cause(jobCtx)) + } + cancel(nil) <-leaseDone now := time.Now().UTC() if err == nil { @@ -146,7 +150,7 @@ func (l *Loop) handle(ctx context.Context, job domainqueue.Job) { _ = l.queue.Retry(ctx, job.ID, l.config.Owner, err, now.Add(backoff)) } -func (l *Loop) keepLeaseAlive(ctx context.Context, job domainqueue.Job, cancel context.CancelFunc, done chan<- struct{}) { +func (l *Loop) keepLeaseAlive(ctx context.Context, job domainqueue.Job, cancel context.CancelCauseFunc, done chan<- struct{}) { defer close(done) interval := l.config.LeaseDuration / 3 if interval < 10*time.Millisecond { @@ -158,9 +162,10 @@ func (l *Loop) keepLeaseAlive(ctx context.Context, job domainqueue.Job, cancel c select { case <-ctx.Done(): return - case now := <-ticker.C: - if err := l.queue.RenewLease(ctx, job.ID, l.config.Owner, now.UTC().Add(l.config.LeaseDuration)); err != nil { - cancel() + case <-ticker.C: + // A buffered tick can predate a stalled worker; never renew from that stale time. + if err := l.queue.RenewLease(ctx, job.ID, l.config.Owner, time.Now().UTC().Add(l.config.LeaseDuration)); err != nil { + cancel(fmt.Errorf("renew lease for queue job %q: %w", job.ID, err)) return } } diff --git a/internal/controlplane/eventloop/lease_cancellation_test.go b/internal/controlplane/eventloop/lease_cancellation_test.go new file mode 100644 index 00000000..1c0ff7df --- /dev/null +++ b/internal/controlplane/eventloop/lease_cancellation_test.go @@ -0,0 +1,34 @@ +package eventloop + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/UFFeScience/akoflow/internal/application/ports" + domainqueue "github.com/UFFeScience/akoflow/internal/domain/queue" +) + +type failedLeaseQueue struct{ ports.QueueStore } + +func (failedLeaseQueue) RenewLease(context.Context, string, string, time.Time) error { + return errors.New("lease ownership lost") +} + +func TestLeaseRenewalCancellationPreservesCause(t *testing.T) { + loop := &Loop{queue: failedLeaseQueue{}, config: Config{Owner: "worker", LeaseDuration: 30 * time.Millisecond}} + ctx, cancel := context.WithCancelCause(context.Background()) + defer cancel(nil) + done := make(chan struct{}) + go loop.keepLeaseAlive(ctx, domainqueue.Job{ID: "execution-command"}, cancel, done) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("lease renewal did not cancel handler") + } + if cause := context.Cause(ctx); cause == nil || !strings.Contains(cause.Error(), "execution-command") || !strings.Contains(cause.Error(), "lease ownership lost") { + t.Fatalf("cancellation cause lost: %v", cause) + } +} diff --git a/internal/controlplane/execution/admission.go b/internal/controlplane/execution/admission.go index a2a7678d..9e517ffb 100644 --- a/internal/controlplane/execution/admission.go +++ b/internal/controlplane/execution/admission.go @@ -35,3 +35,28 @@ func plannedReadyActivities( } return eligible } + +// Preserve intentional overcommit for unrestricted legacy resources. Cloud VMs +// honor explicit capacity even when it equals the physical core count; other +// resources honor an explicit reduction of the physical lane capacity. +func restrictedCapacityReady(ready []string, assignments map[string]domain.PlanAssignment, resources map[string]domain.Resource, tasks map[string]domain.TaskExecution, pending map[string]bool) []string { + occupied := make(map[string]int) + for id, task := range tasks { + if task.Status == domain.TaskPreparing || task.Status == domain.TaskRunning || pending[id] { + occupied[assignments[id].ResourceID]++ + } + } + eligible := make([]string, 0, len(ready)) + for _, id := range ready { + resourceID := assignments[id].ResourceID + r := resources[resourceID] + if r.CPUCapacity > 0 && (r.Type == domain.ResourceCloudVM || r.CPUCapacity < float64(r.CPUCores)) { + if occupied[resourceID] >= r.SchedulingCPUCores() { + continue + } + occupied[resourceID]++ + } + eligible = append(eligible, id) + } + return eligible +} diff --git a/internal/controlplane/execution/admission_capacity_test.go b/internal/controlplane/execution/admission_capacity_test.go new file mode 100644 index 00000000..79f8321a --- /dev/null +++ b/internal/controlplane/execution/admission_capacity_test.go @@ -0,0 +1,75 @@ +package execution + +import ( + "github.com/UFFeScience/akoflow/internal/domain" + "testing" +) + +func TestRestrictedCapacityReservesOneMicroLane(t *testing.T) { + assignments := map[string]domain.PlanAssignment{"a": {ResourceID: "micro"}, "b": {ResourceID: "micro"}} + resources := map[string]domain.Resource{"micro": {CPUCores: 2, CPUCapacity: 1}} + if got := restrictedCapacityReady([]string{"a", "b"}, assignments, resources, nil, nil); len(got) != 1 || got[0] != "a" { + t.Fatalf("admission=%v", got) + } + for _, status := range []domain.TaskExecutionStatus{domain.TaskPreparing, domain.TaskRunning} { + tasks := map[string]domain.TaskExecution{"a": {Status: status}} + if got := restrictedCapacityReady([]string{"b"}, assignments, resources, tasks, nil); len(got) != 0 { + t.Fatalf("busy admission=%v", got) + } + } + tasks := map[string]domain.TaskExecution{"a": {Status: domain.TaskQueued}} + if got := restrictedCapacityReady([]string{"b"}, assignments, resources, tasks, map[string]bool{"a": true}); len(got) != 0 { + t.Fatalf("allocation admission=%v", got) + } + tasks["a"] = domain.TaskExecution{Status: domain.TaskCompleted} + if got := restrictedCapacityReady([]string{"b"}, assignments, resources, tasks, nil); len(got) != 1 { + t.Fatalf("released admission=%v", got) + } +} + +func TestRequestRejectsPlanExceedingExplicitMicroCapacity(t *testing.T) { + request := requestFixture(domain.ExecutionModeReal) + request.Resources[0].CPUCores = 2 + request.Resources[0].CPUCapacity = 1 + request.Plan.Assignments[0].CoreID = "r-core-1" + request.Plan.Assignments[1].CoreID = "r-core-2" + if err := validateRequest(request); err == nil { + t.Fatal("oversubscribed micro plan accepted") + } + request.Plan.Assignments[1].CoreID = "r-core-1" + if err := validateRequest(request); err != nil { + t.Fatal(err) + } +} + +func TestRestrictedCapacityReservesOneBatchLane(t *testing.T) { + assignments := map[string]domain.PlanAssignment{"a": {ResourceID: "hpc"}, "b": {ResourceID: "hpc"}} + resources := map[string]domain.Resource{"hpc": {CPUCores: 64, CPUCapacity: 1, ExecutionTarget: domain.ExecutionTargetBatch}} + if got := restrictedCapacityReady([]string{"a", "b"}, assignments, resources, nil, nil); len(got) != 1 { + t.Fatalf("batch admission exceeded explicit capacity: %v", got) + } + for _, status := range []domain.TaskExecutionStatus{domain.TaskPreparing, domain.TaskRunning} { + if got := restrictedCapacityReady([]string{"b"}, assignments, resources, map[string]domain.TaskExecution{"a": {Status: status}}, nil); len(got) != 0 { + t.Fatalf("occupied batch lane admitted another activity: %v", got) + } + } + if got := restrictedCapacityReady([]string{"b"}, assignments, resources, map[string]domain.TaskExecution{"a": {Status: domain.TaskCompleted}}, nil); len(got) != 1 { + t.Fatalf("completed batch lane was not released: %v", got) + } +} + +func TestCloudCapacityEqualToPhysicalCoresStillLimitsAdmission(t *testing.T) { + assignments := map[string]domain.PlanAssignment{"a": {ResourceID: "vm"}, "b": {ResourceID: "vm"}, "c": {ResourceID: "vm"}} + resources := map[string]domain.Resource{"vm": {Type: domain.ResourceCloudVM, CPUCores: 2, CPUCapacity: 2}} + if got := restrictedCapacityReady([]string{"a", "b", "c"}, assignments, resources, nil, nil); len(got) != 2 { + t.Fatalf("cloud admission=%v, want two slots", got) + } + tasks := map[string]domain.TaskExecution{"a": {Status: domain.TaskPreparing}, "b": {Status: domain.TaskRunning}} + if got := restrictedCapacityReady([]string{"c"}, assignments, resources, tasks, nil); len(got) != 0 { + t.Fatalf("busy cloud admitted=%v", got) + } + tasks["b"] = domain.TaskExecution{Status: domain.TaskCompleted} + if got := restrictedCapacityReady([]string{"c"}, assignments, resources, tasks, nil); len(got) != 1 { + t.Fatalf("released cloud slot admission=%v", got) + } +} diff --git a/internal/controlplane/execution/cloud_allocator.go b/internal/controlplane/execution/cloud_allocator.go index a5790b90..618ab0ad 100644 --- a/internal/controlplane/execution/cloud_allocator.go +++ b/internal/controlplane/execution/cloud_allocator.go @@ -27,6 +27,42 @@ type CloudStopper interface { Stop(context.Context, string, map[string]domain.RuntimeAllocation) error } +type CloudRecoveryReadier interface { + ReadyRecoveryInstance(context.Context, string, string, string) error +} + +// Recovery must not replace a stopped producer with another generation: its +// completed outputs belong to this exact disk and instance. +func (a QueuedCloudAllocator) ReadyRecoveryInstance(ctx context.Context, runID, activityID, instanceID string) error { + instance, err := a.Cloud.FindProvisionedInstance(ctx, instanceID) + if err != nil { + return err + } + if instance == nil || (instance.Status != "ready" && instance.Status != "stopped") { + return fmt.Errorf("recovery producer instance %q is unavailable", instanceID) + } + if instance.Status == "ready" { + return nil + } + target, err := a.Cloud.FindCapacityTarget(ctx, instance.CapacityTargetID) + if err != nil { + return err + } + if target == nil || target.EnvironmentID != instance.EnvironmentID { + return fmt.Errorf("recovery producer instance %q has invalid capacity target", instanceID) + } + for _, kind := range []string{"start", "validate"} { + op, err := a.findOrCreate(ctx, runID, activityID, *target, kind, instanceID) + if err != nil { + return err + } + if _, err := a.wait(ctx, op); err != nil { + return err + } + } + return nil +} + // QueuedCloudAllocator is the execution-to-infrastructure boundary. It never // invokes Terraform itself: it persists an operation, publishes a durable // message, then waits for the infrastructure worker to settle it. @@ -105,8 +141,14 @@ func (a QueuedCloudAllocator) wait(ctx context.Context, operation domain.CloudOp return domain.CloudProvisionedInstance{}, err } instance, err := a.Cloud.FindProvisionedInstance(ctx, stored.InstanceID) - if err != nil || instance == nil || instance.Status != "ready" { - return domain.CloudProvisionedInstance{}, fmt.Errorf("cloud allocation %q completed without a ready instance: %w", stored.ID, err) + if err != nil { + return domain.CloudProvisionedInstance{}, fmt.Errorf("inspect cloud allocation %q: %w", stored.ID, err) + } + if instance == nil { + return domain.CloudProvisionedInstance{}, fmt.Errorf("cloud allocation %q completed but instance %q is missing", stored.ID, stored.InstanceID) + } + if instance.Status != "ready" { + return domain.CloudProvisionedInstance{}, fmt.Errorf("cloud allocation %q completed but instance %q has status %q", stored.ID, instance.ID, instance.Status) } return *instance, nil } @@ -145,17 +187,46 @@ func (a QueuedCloudAllocator) findOrCreate(ctx context.Context, executionRunID, if err != nil { return domain.CloudOperationRun{}, err } + // Start/validation idempotency is scoped to a stop/start cycle, not the + // lifetime of the workflow. A completed start before a later stop is history. + cycle := cloudAllocationCycle(operations, kind, instanceID) // Several ready activities can share one capacity target. Reuse its // infrastructure operation so each activity does not request another VM. for _, operation := range operations { if operation.Kind == kind && operation.ExecutionRunID == executionRunID && operation.CapacityTargetID == target.ID && (instanceID == "" || operation.InstanceID == instanceID) { + if cycle.ID != "" && operation.CreatedAt.Before(cycle.CreatedAt) { + continue + } + if kind == "provision" && (operation.Status == "completed" || operation.Status == "failed" || operation.Status == "cancelled") { + instance, inspectErr := a.Cloud.FindProvisionedInstance(ctx, operation.InstanceID) + if inspectErr != nil { + return domain.CloudOperationRun{}, inspectErr + } + // An old provision command is not capacity after its VM has + // been destroyed. Preserve history and create a new generation. + if instance == nil || instance.Status == "destroyed" { + continue + } + } return operation, nil } } if kind == "provision" && instanceID == "" { instanceID = "cloud-instance-" + uuid.NewString() } - operation := domain.CloudOperationRun{ID: "cloud-run-" + uuid.NewString(), Kind: kind, Status: "queued", Phase: "queued", EnvironmentID: target.EnvironmentID, CapacityTargetID: target.ID, InstanceID: instanceID, ExecutionRunID: executionRunID, ActivityID: activityID, Request: domain.CloudProvisionRequest{CapacityTargetID: target.ID, InstanceID: instanceID}, CreatedAt: time.Now().UTC()} + operation := domain.CloudOperationRun{ + ID: "cloud-run-" + uuid.NewString(), + Kind: kind, + Status: "queued", + Phase: "queued", + EnvironmentID: target.EnvironmentID, + CapacityTargetID: target.ID, + InstanceID: instanceID, + ExecutionRunID: executionRunID, + ActivityID: activityID, + Request: domain.CloudProvisionRequest{CapacityTargetID: target.ID, InstanceID: instanceID}, + CreatedAt: time.Now().UTC(), + } if err := a.Operations.CreateCloudOperation(ctx, operation); err != nil { return domain.CloudOperationRun{}, err } @@ -164,18 +235,39 @@ func (a QueuedCloudAllocator) findOrCreate(ctx context.Context, executionRunID, if err == nil { job.AggregateType, job.AggregateID = "cloud_operation", operation.ID job.IdempotencyKey = "cloud-allocation:" + executionRunID + ":" + target.ID + ":" + instanceID + ":" + kind + if cycle.ID != "" { + job.IdempotencyKey += ":after-stop:" + cycle.ID + } job.Priority, job.MaxAttempts = 100, 3 - _, err = a.Queue.Publish(ctx, job) + var published domainqueue.Job + published, err = a.Queue.Publish(ctx, job) + if err == nil && published.AggregateID != "" && published.AggregateID != operation.ID { + return a.canonicalAllocation(ctx, operation, published.AggregateID) + } } if err != nil { now := time.Now().UTC() operation.Status, operation.Phase, operation.FailureReason, operation.FinishedAt = "failed", "failed", err.Error(), &now - _ = a.Operations.UpdateCloudOperation(ctx, operation) + _ = a.Operations.UpdateCloudOperation(context.WithoutCancel(ctx), operation) return operation, err } return operation, nil } +func cloudAllocationCycle(operations []domain.CloudOperationRun, kind, instanceID string) domain.CloudOperationRun { + var cycle domain.CloudOperationRun + if kind != "start" && kind != "validate" { + return cycle + } + for _, candidate := range operations { + if candidate.Kind == "stop" && candidate.Status == "completed" && candidate.InstanceID == instanceID && + (cycle.ID == "" || candidate.CreatedAt.After(cycle.CreatedAt)) { + cycle = candidate + } + } + return cycle +} + func (a QueuedCloudAllocator) Release(ctx context.Context, executionRunID string, allocations map[string]domain.RuntimeAllocation, failed bool) error { seen := map[string]bool{} for activityID, allocation := range allocations { @@ -216,7 +308,18 @@ func (a QueuedCloudAllocator) Release(ctx context.Context, executionRunID string if kind == "" { continue } - operation := domain.CloudOperationRun{ID: "cloud-run-" + uuid.NewString(), Kind: kind, Status: "queued", Phase: "queued", EnvironmentID: instance.EnvironmentID, CapacityTargetID: instance.CapacityTargetID, InstanceID: instance.ID, ExecutionRunID: executionRunID, ActivityID: activityID, CreatedAt: time.Now().UTC()} + operation := domain.CloudOperationRun{ + ID: "cloud-run-" + uuid.NewString(), + Kind: kind, + Status: "queued", + Phase: "queued", + EnvironmentID: instance.EnvironmentID, + CapacityTargetID: instance.CapacityTargetID, + InstanceID: instance.ID, + ExecutionRunID: executionRunID, + ActivityID: activityID, + CreatedAt: time.Now().UTC(), + } if err := a.Operations.CreateCloudOperation(ctx, operation); err != nil { return err } @@ -287,3 +390,21 @@ func (a QueuedCloudAllocator) Stop(ctx context.Context, executionRunID string, a } return nil } + +func (a QueuedCloudAllocator) canonicalAllocation(ctx context.Context, operation domain.CloudOperationRun, canonicalID string) (domain.CloudOperationRun, error) { + canonical, lookupErr := a.Operations.FindCloudOperation(ctx, canonicalID) + if lookupErr != nil || canonical == nil { + cause := fmt.Errorf("resolve canonical cloud operation %q: %v", canonicalID, lookupErr) + now := time.Now().UTC() + operation.Status, operation.Phase, operation.FailureReason, operation.FinishedAt = "failed", "failed", cause.Error(), &now + _ = a.Operations.UpdateCloudOperation(context.WithoutCancel(ctx), operation) + return operation, cause + } else { + now := time.Now().UTC() + operation.Status, operation.Phase, operation.FailureReason, operation.FinishedAt = "cancelled", "deduplicated", "Superseded by cloud operation "+canonical.ID, &now + if updateErr := a.Operations.UpdateCloudOperation(context.WithoutCancel(ctx), operation); updateErr != nil { + return operation, updateErr + } + return *canonical, nil + } +} diff --git a/internal/controlplane/execution/cloud_allocator_test.go b/internal/controlplane/execution/cloud_allocator_test.go index a4494227..16daf13a 100644 --- a/internal/controlplane/execution/cloud_allocator_test.go +++ b/internal/controlplane/execution/cloud_allocator_test.go @@ -2,6 +2,7 @@ package execution import ( "context" + "strings" "sync" "testing" "time" @@ -83,6 +84,32 @@ type allocatorQueue struct { jobs []domainqueue.Job } +type deduplicatingAllocatorQueue struct { + ports.QueueStore + store *allocatorStore +} + +func (q deduplicatingAllocatorQueue) Publish(_ context.Context, job domainqueue.Job) (domainqueue.Job, error) { + // Simulate another caller winning between List and Publish. + q.store.operations["canonical"] = domain.CloudOperationRun{ID: "canonical", Kind: "start", Status: "completed", InstanceID: "vm", ExecutionRunID: "run", CapacityTargetID: "target"} + job.AggregateID = "canonical" + return job, nil +} + +func TestQueuedCloudAllocatorReturnsCanonicalDeduplicatedOperation(t *testing.T) { + store := &allocatorStore{operations: map[string]domain.CloudOperationRun{}} + a := QueuedCloudAllocator{Cloud: store, Operations: store, Queue: deduplicatingAllocatorQueue{store: store}} + op, err := a.findOrCreate(context.Background(), "run", "activity", domain.CloudCapacityTarget{ID: "target"}, "start", "vm") + if err != nil || op.ID != "canonical" { + t.Fatalf("op=%+v err=%v", op, err) + } + for id, value := range store.operations { + if id != "canonical" && (value.Status != "cancelled" || value.Phase != "deduplicated" || value.FinishedAt == nil) { + t.Fatalf("orphan left live: %+v", value) + } + } +} + func (q *allocatorQueue) Publish(_ context.Context, job domainqueue.Job) (domainqueue.Job, error) { q.jobs = append(q.jobs, job) go func() { @@ -90,6 +117,9 @@ func (q *allocatorQueue) Publish(_ context.Context, job domainqueue.Job) (domain q.store.mu.Lock() defer q.store.mu.Unlock() for id, operation := range q.store.operations { + if operation.Status != "queued" { + continue + } operation.Status, operation.InstanceID = "completed", operation.Request.InstanceID q.store.operations[id] = operation status := "ready" @@ -104,6 +134,31 @@ func (q *allocatorQueue) Publish(_ context.Context, job domainqueue.Job) (domain return job, nil } +func TestQueuedCloudAllocatorReplacesDestroyedProvisionGeneration(t *testing.T) { + for _, status := range []string{"completed", "failed", "cancelled"} { + t.Run(status, func(t *testing.T) { + store := &allocatorStore{ + operations: map[string]domain.CloudOperationRun{"old-operation": {ID: "old-operation", Kind: "provision", Status: status, ExecutionRunID: "run", CapacityTargetID: "target", InstanceID: "old-vm"}}, + instances: map[string]domain.CloudProvisionedInstance{"old-vm": {ID: "old-vm", CapacityTargetID: "target", EnvironmentID: "environment", Status: "destroyed"}}, + } + queue := &allocatorQueue{store: store} + allocator := QueuedCloudAllocator{Cloud: store, Operations: store, Queue: queue, PollInterval: time.Millisecond} + target := domain.CloudCapacityTarget{ID: "target", EnvironmentID: "environment"} + if err := allocator.Prewarm(context.Background(), "run", "activity", target); err != nil { + t.Fatal(err) + } + instance, err := allocator.Allocate(context.Background(), "run", "activity", target) + if err != nil || instance.ID == "old-vm" || instance.Status != "ready" || len(queue.jobs) != 1 { + t.Fatalf("obsolete allocation reused or duplicated: instance=%+v err=%v jobs=%d", instance, err, len(queue.jobs)) + } + old, _ := store.FindProvisionedInstance(context.Background(), "old-vm") + if old.Status != "destroyed" { + t.Fatal("old instance history mutated") + } + }) + } +} + func TestQueuedCloudAllocatorAssociatesDurableOperationWithActivity(t *testing.T) { store := &allocatorStore{operations: map[string]domain.CloudOperationRun{}, instances: map[string]domain.CloudProvisionedInstance{}} queue := &allocatorQueue{store: store} @@ -182,6 +237,33 @@ func TestQueuedCloudAllocatorStartsAndValidatesStoppedInstance(t *testing.T) { } } +func TestRecoveryStartsExactProducerAfterLaterStop(t *testing.T) { + now := time.Now().UTC() + store := &allocatorStore{operations: map[string]domain.CloudOperationRun{ + "old-start": {ID: "old-start", Kind: "start", Status: "completed", InstanceID: "producer", ExecutionRunID: "run", CapacityTargetID: "target", CreatedAt: now.Add(-time.Hour)}, + "stop-cycle": {ID: "stop-cycle", Kind: "stop", Status: "completed", InstanceID: "producer", CreatedAt: now}, + }, instances: map[string]domain.CloudProvisionedInstance{ + "producer": {ID: "producer", CapacityTargetID: "target", EnvironmentID: "environment", Status: "stopped"}, + "other": {ID: "other", CapacityTargetID: "target", EnvironmentID: "environment", Status: "ready"}, + }, targets: map[string]domain.CloudCapacityTarget{"target": {ID: "target", EnvironmentID: "environment"}}} + queue := &allocatorQueue{store: store} + a := QueuedCloudAllocator{Cloud: store, Operations: store, Queue: queue, PollInterval: time.Millisecond} + if err := a.ReadyRecoveryInstance(context.Background(), "run", "activity", "producer"); err != nil { + t.Fatal(err) + } + if len(queue.jobs) != 2 { + t.Fatalf("expected exact start and validation, got %d", len(queue.jobs)) + } + for _, job := range queue.jobs { + if !strings.Contains(job.IdempotencyKey, ":after-stop:stop-cycle") { + t.Fatalf("stale idempotency cycle: %s", job.IdempotencyKey) + } + } + if err := a.ReadyRecoveryInstance(context.Background(), "run", "activity", "missing"); err == nil { + t.Fatal("missing producer replaced") + } +} + func TestQueuedCloudAllocatorStopPreservesInstanceAndIsIdempotent(t *testing.T) { store := &allocatorStore{ operations: map[string]domain.CloudOperationRun{}, diff --git a/internal/controlplane/execution/cloud_stop_gate.go b/internal/controlplane/execution/cloud_stop_gate.go index 1ae8623c..8c9fdbd0 100644 --- a/internal/controlplane/execution/cloud_stop_gate.go +++ b/internal/controlplane/execution/cloud_stop_gate.go @@ -2,33 +2,59 @@ package execution import ( "context" + "sync" "github.com/UFFeScience/akoflow/internal/application/ports" "github.com/UFFeScience/akoflow/internal/domain" ) -func (s *Supervisor) stopIdleCloud( - ctx context.Context, - request ports.ExecutionRequest, - completed map[string]domain.TaskExecution, - tasks map[string]domain.TaskExecution, - stopping map[string]bool, -) { +// Idle teardown must not hold the activity observation loop. Admission remains +// on that loop; workers receive immutable allocation snapshots and never touch +// its maps. Joining before release prevents overlapping cleanup ownership. +type idleCloudStops struct { + ctx context.Context + cancel context.CancelFunc + slots chan struct{} + wg sync.WaitGroup +} + +func newIdleCloudStops(ctx context.Context) *idleCloudStops { + ctx, cancel := context.WithCancel(ctx) + return &idleCloudStops{ctx: ctx, cancel: cancel, slots: make(chan struct{}, 4)} +} + +func (p *idleCloudStops) submit(stop func(context.Context) error) bool { + select { + case p.slots <- struct{}{}: + default: + return false + } + p.wg.Add(1) + go func() { + defer p.wg.Done() + defer func() { <-p.slots }() + // The durable operation records failures; final release verifies/retries + // cleanup without turning successful computation into a failure. + _ = stop(p.ctx) + }() + return true +} + +func (p *idleCloudStops) close() { + p.cancel() + p.wg.Wait() +} + +func (s *Supervisor) scheduleIdleCloudStops(request ports.ExecutionRequest, completed, tasks map[string]domain.TaskExecution, stopping map[string]bool, workers *idleCloudStops) { stopper, ok := s.config.CloudAllocator.(CloudStopper) if !ok { return } - ready := readyCloudStops( - request, completed, tasks, stopping, - s.config.Data != nil && s.config.Preparer != nil, - ) + ready := readyCloudStops(request, completed, tasks, stopping, s.config.Data != nil && s.config.Preparer != nil) for instanceID, allocations := range ready { - if err := stopper.Stop(ctx, request.Run.ID, allocations); err != nil { - // Teardown must not change the scientific run result. The final - // release path will retry if infrastructure submission failed. - continue + if workers.submit(func(workerCtx context.Context) error { return stopper.Stop(workerCtx, request.Run.ID, allocations) }) { + stopping[instanceID] = true } - stopping[instanceID] = true } } diff --git a/internal/controlplane/execution/cloud_stop_gate_test.go b/internal/controlplane/execution/cloud_stop_gate_test.go index 2379ee3e..93010ca1 100644 --- a/internal/controlplane/execution/cloud_stop_gate_test.go +++ b/internal/controlplane/execution/cloud_stop_gate_test.go @@ -3,10 +3,41 @@ package execution import ( "context" "testing" + "time" "github.com/UFFeScience/akoflow/internal/domain" ) +func TestIdleCloudStopsDoNotBlockObservationAndBoundAdmission(t *testing.T) { + workers := newIdleCloudStops(context.Background()) + entered := make(chan struct{}, 4) + exited := make(chan struct{}, 4) + for i := 0; i < 4; i++ { + if !workers.submit(func(ctx context.Context) error { + entered <- struct{}{} + <-ctx.Done() + exited <- struct{}{} + return ctx.Err() + }) { + t.Fatal("worker admission blocked") + } + } + if workers.submit(func(context.Context) error { t.Fatal("exceeded stop concurrency"); return nil }) { + t.Fatal("admitted fifth stop") + } + for i := 0; i < 4; i++ { + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("stop did not start independently") + } + } + workers.close() + if len(exited) != 4 { + t.Fatal("cleanup returned with workers still active") + } +} + type cloudStopperFake struct { stopped map[string]domain.RuntimeAllocation released map[string]domain.RuntimeAllocation diff --git a/internal/controlplane/execution/cloud_workspace_retention_test.go b/internal/controlplane/execution/cloud_workspace_retention_test.go new file mode 100644 index 00000000..69e2156a --- /dev/null +++ b/internal/controlplane/execution/cloud_workspace_retention_test.go @@ -0,0 +1,26 @@ +package execution + +import ( + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestFailedRunRetainsExactProducerGenerationWithPendingConsumers(t *testing.T) { + allocations := map[string]domain.RuntimeAllocation{ + "producer": {CloudInstanceID: "vm-new"}, + "neighbor": {CloudInstanceID: "vm-new"}, + "old": {CloudInstanceID: "vm-old"}, + } + workspaces := []domain.ActivityWorkspace{{ID: "workspace", State: domain.WorkspaceSealed, URI: "cloud://target/path?cloudInstanceId=vm-new"}} + leases := []domain.WorkspaceLease{{WorkspaceID: "workspace"}} + remaining, retained := retainCloudSourcesWithWorkspaceLeases(allocations, workspaces, leases) + if retained != 1 || len(remaining) != 1 || remaining["old"].CloudInstanceID != "vm-old" { + t.Fatalf("retained=%d allocations=%v", retained, remaining) + } + leases[0].Released = true + remaining, retained = retainCloudSourcesWithWorkspaceLeases(allocations, workspaces, leases) + if retained != 0 || len(remaining) != 3 { + t.Fatal("released leases must allow lifecycle cleanup") + } +} diff --git a/internal/controlplane/execution/dependency_completion_test.go b/internal/controlplane/execution/dependency_completion_test.go new file mode 100644 index 00000000..a514a8a4 --- /dev/null +++ b/internal/controlplane/execution/dependency_completion_test.go @@ -0,0 +1,36 @@ +package execution + +import ( + "context" + "fmt" + "sync" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +// Observe the dependency barrier, not unrelated goroutine startup ordering. +type dependencyCompletionController struct { + activityControllerFake + completionMu sync.Mutex + producerCompleted bool +} + +func (f *dependencyCompletionController) Start(ctx context.Context, execution domain.ActivityExecutionContext) (domain.ActivityHandle, error) { + f.completionMu.Lock() + completed := f.producerCompleted + f.completionMu.Unlock() + if execution.Activity.ID == "a" && !completed { + return domain.ActivityHandle{}, fmt.Errorf("consumer started before producer completion") + } + return f.activityControllerFake.Start(ctx, execution) +} + +func (f *dependencyCompletionController) Inspect(ctx context.Context, id string, mode domain.ExecutionMode) (*domain.ActivityHandle, error) { + handle, err := f.activityControllerFake.Inspect(ctx, id, mode) + if err == nil && id == "h-c" && handle != nil && handle.Status == domain.HandleCompleted { + f.completionMu.Lock() + f.producerCompleted = true + f.completionMu.Unlock() + } + return handle, err +} diff --git a/internal/controlplane/execution/ready_allocations.go b/internal/controlplane/execution/ready_allocations.go new file mode 100644 index 00000000..f921f8d6 --- /dev/null +++ b/internal/controlplane/execution/ready_allocations.go @@ -0,0 +1,27 @@ +package execution + +import "github.com/UFFeScience/akoflow/internal/domain" + +type allocationResult struct { + activityID string + allocation domain.RuntimeAllocation + err error +} + +// Drain only available completions; unfinished provision operations never +// hold the scheduling loop or delay unrelated eligible activities. +func drainReadyAllocations(results <-chan allocationResult, consume func(allocationResult) error) error { + for { + select { + case result, ok := <-results: + if !ok { + return nil + } + if err := consume(result); err != nil { + return err + } + default: + return nil + } + } +} diff --git a/internal/controlplane/execution/ready_allocations_test.go b/internal/controlplane/execution/ready_allocations_test.go new file mode 100644 index 00000000..c2ca64b9 --- /dev/null +++ b/internal/controlplane/execution/ready_allocations_test.go @@ -0,0 +1,33 @@ +package execution + +import ( + "errors" + "testing" +) + +func TestDrainReadyAllocationsBatchesAvailableWithoutWaiting(t *testing.T) { + results := make(chan allocationResult, 8) + for i := 0; i < 8; i++ { + results <- allocationResult{activityID: "ready"} + } + consumed := 0 + err := drainReadyAllocations(results, func(allocationResult) error { consumed++; return nil }) + if err != nil || consumed != 8 { + t.Fatalf("consumed=%d err=%v", consumed, err) + } + if err := drainReadyAllocations(results, func(allocationResult) error { t.Fatal("empty channel consumed"); return nil }); err != nil { + t.Fatal(err) + } +} + +func TestDrainReadyAllocationsStopsBeforeLaunchingOnFailure(t *testing.T) { + failure := errors.New("allocation failed") + results := make(chan allocationResult, 2) + results <- allocationResult{err: failure} + results <- allocationResult{activityID: "later"} + consumed := 0 + err := drainReadyAllocations(results, func(r allocationResult) error { consumed++; return r.err }) + if !errors.Is(err, failure) || consumed != 1 || len(results) != 1 { + t.Fatalf("consumed=%d err=%v remaining=%d", consumed, err, len(results)) + } +} diff --git a/internal/controlplane/execution/recovered_allocation.go b/internal/controlplane/execution/recovered_allocation.go new file mode 100644 index 00000000..d5276742 --- /dev/null +++ b/internal/controlplane/execution/recovered_allocation.go @@ -0,0 +1,16 @@ +package execution + +import "github.com/UFFeScience/akoflow/internal/domain" + +// Reused output remains on the exact instance recorded by its completed +// attempt. Never substitute a ready instance with the same capacity target. +func recoveredAllocation(task domain.TaskExecution) domain.RuntimeAllocation { + resourceID := task.AllocatedResourceID + if resourceID == "" { + resourceID = task.PlannedResourceID + } + return domain.RuntimeAllocation{ + ResourceID: resourceID, RuntimeID: task.RuntimeID, + ConnectionID: task.ConnectionID, CloudInstanceID: task.CloudInstanceID, + } +} diff --git a/internal/controlplane/execution/recovered_allocation_test.go b/internal/controlplane/execution/recovered_allocation_test.go new file mode 100644 index 00000000..7ebf58cd --- /dev/null +++ b/internal/controlplane/execution/recovered_allocation_test.go @@ -0,0 +1,18 @@ +package execution + +import ( + "github.com/UFFeScience/akoflow/internal/domain" + "testing" +) + +func TestRecoveredAllocationPreservesExactProducerInstance(t *testing.T) { + task := domain.TaskExecution{AllocatedResourceID: "target", PlannedResourceID: "other", RuntimeID: "cloud", ConnectionID: "connection", CloudInstanceID: "producer-vm"} + allocation := recoveredAllocation(task) + if allocation.ResourceID != "target" || allocation.CloudInstanceID != "producer-vm" || allocation.RuntimeID != "cloud" || allocation.ConnectionID != "connection" { + t.Fatalf("allocation lost exact producer identity: %+v", allocation) + } + task.AllocatedResourceID = "" + if recoveredAllocation(task).ResourceID != "other" { + t.Fatal("planned resource fallback was lost") + } +} diff --git a/internal/controlplane/execution/recovery_workspace_integrity.go b/internal/controlplane/execution/recovery_workspace_integrity.go new file mode 100644 index 00000000..7a868c75 --- /dev/null +++ b/internal/controlplane/execution/recovery_workspace_integrity.go @@ -0,0 +1,34 @@ +package execution + +import ( + "context" + "fmt" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" +) + +// A sealed catalog entry does not prove its files still exist after cleanup or +// an interrupted recovery. Missing data requires recomputing the producer. +func recoveryWorkspacePresent(workspace domain.ActivityWorkspace, usage domain.WorkspaceUsage) bool { + return workspace.State == domain.WorkspaceSealed && usage.FileCount == workspace.FileCount && usage.SizeBytes == workspace.SizeBytes +} + +// A regenerated producer must not be reclaimed using leases released by a +// previous attempt before its pending consumers have received the data again. +func restoreRecoveryWorkspaceLeases(ctx context.Context, store ports.WorkspaceStore, runID string, reusable map[string]bool) error { + leases, err := store.ListWorkspaceLeases(ctx, runID) + if err != nil { + return err + } + for _, lease := range leases { + if reusable[lease.ConsumerActivityID] || !lease.Released { + continue + } + lease.Released, lease.ReleasedAt, lease.ReleaseReason = false, nil, "" + if err := store.SaveWorkspaceLease(ctx, lease); err != nil { + return fmt.Errorf("restore recovery workspace lease %q: %w", lease.ID, err) + } + } + return nil +} diff --git a/internal/controlplane/execution/recovery_workspace_integrity_test.go b/internal/controlplane/execution/recovery_workspace_integrity_test.go new file mode 100644 index 00000000..9fc12b41 --- /dev/null +++ b/internal/controlplane/execution/recovery_workspace_integrity_test.go @@ -0,0 +1,56 @@ +package execution + +import ( + "context" + "testing" + "time" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" +) + +type recoveryLeaseStore struct { + ports.WorkspaceStore + leases []domain.WorkspaceLease + saved []domain.WorkspaceLease +} + +func (s *recoveryLeaseStore) ListWorkspaceLeases(context.Context, string) ([]domain.WorkspaceLease, error) { + return s.leases, nil +} +func (s *recoveryLeaseStore) SaveWorkspaceLease(_ context.Context, lease domain.WorkspaceLease) error { + s.saved = append(s.saved, lease) + return nil +} + +func TestRecoveryRejectsSealedButEmptyWorkspace(t *testing.T) { + w := domain.ActivityWorkspace{State: domain.WorkspaceSealed, FileCount: 47, SizeBytes: 196938502} + if recoveryWorkspacePresent(w, domain.WorkspaceUsage{}) { + t.Fatal("stale manifest reused an empty producer") + } + if !recoveryWorkspacePresent(w, domain.WorkspaceUsage{FileCount: w.FileCount, SizeBytes: w.SizeBytes}) { + t.Fatal("intact workspace rejected") + } +} + +func TestRecoveryRejectsUnsealedEmptyWorkspace(t *testing.T) { + for _, state := range []domain.WorkspaceState{domain.WorkspacePlanned, domain.WorkspaceActive, domain.WorkspaceReleased, domain.WorkspaceFailed} { + if recoveryWorkspacePresent(domain.ActivityWorkspace{State: state}, domain.WorkspaceUsage{}) { + t.Fatalf("unsealed %s workspace reused because zero counts matched", state) + } + } +} + +func TestRecoveryRearmsOnlyPendingConsumerLeases(t *testing.T) { + now := time.Now() + store := &recoveryLeaseStore{leases: []domain.WorkspaceLease{ + {ID: "pending", ConsumerActivityID: "retry", Released: true, ReleasedAt: &now, ReleaseReason: "previous attempt"}, + {ID: "completed", ConsumerActivityID: "done", Released: true}, + }} + if err := restoreRecoveryWorkspaceLeases(context.Background(), store, "run", map[string]bool{"done": true}); err != nil { + t.Fatal(err) + } + if len(store.saved) != 1 || store.saved[0].ID != "pending" || store.saved[0].Released || store.saved[0].ReleasedAt != nil || store.saved[0].ReleaseReason != "" { + t.Fatalf("incorrect retention: %+v", store.saved) + } +} diff --git a/internal/controlplane/execution/remote_observation_timing_test.go b/internal/controlplane/execution/remote_observation_timing_test.go new file mode 100644 index 00000000..593dd018 --- /dev/null +++ b/internal/controlplane/execution/remote_observation_timing_test.go @@ -0,0 +1,30 @@ +package execution + +import ( + "github.com/UFFeScience/akoflow/internal/domain" + "testing" +) + +func TestRemoteObservationEvidenceIsPublishedWithoutDoubleCounting(t *testing.T) { + task := domain.TaskExecution{OverheadSeconds: 2} + handle := domain.ActivityHandle{StartedAt: 10, FinishedAt: 20, Metadata: map[string]any{ + "remoteWorkspaceSnapshotSeconds": 7.0, + "remoteArtifactObservationSeconds": 9.0, + "remoteArtifactsReadyAt": 29.0, + "artifactObservationBefore": "private manifest", + "artifactObservationRoot": "/private/path", + }} + completeTask(&task, handle) + if task.RuntimeSeconds != 10 || task.OverheadSeconds != 2 { + t.Fatal("diagnostics inflated runtime or overhead") + } + if task.Metadata["remoteArtifactsReadyAt"] != 29.0 || task.Metadata["remoteWorkspaceSnapshotSeconds"] != 7.0 { + t.Fatal("timing evidence missing") + } + if _, ok := task.Metadata["artifactObservationBefore"]; ok { + t.Fatal("manifest must not be copied as timing evidence") + } + if _, ok := task.Metadata["artifactObservationRoot"]; ok { + t.Fatal("private path must not be copied as timing evidence") + } +} diff --git a/internal/controlplane/execution/supervisor.go b/internal/controlplane/execution/supervisor.go index 45df8955..f24fa0c1 100644 --- a/internal/controlplane/execution/supervisor.go +++ b/internal/controlplane/execution/supervisor.go @@ -3,6 +3,7 @@ package execution import ( "context" "crypto/sha256" + "errors" "fmt" "net/url" "os" @@ -50,6 +51,7 @@ type Supervisor struct { activities ActivityController simulation ports.PlanExecutor config Config + activeRuns sync.Map } type startResult struct { @@ -82,6 +84,10 @@ func New(executions ports.ExecutionStore, activities ActivityController, simulat } func (s *Supervisor) Execute(ctx context.Context, request ports.ExecutionRequest) (trace domain.ExecutionTrace, err error) { + if _, active := s.activeRuns.LoadOrStore(request.Run.ID, struct{}{}); active { + return domain.ExecutionTrace{}, fmt.Errorf("execution run %q already has an active supervisor; duplicate command rejected", request.Run.ID) + } + defer s.activeRuns.Delete(request.Run.ID) if request.RuntimeAllocations == nil { request.RuntimeAllocations = make(map[string]domain.RuntimeAllocation) } @@ -93,12 +99,12 @@ func (s *Supervisor) Execute(ctx context.Context, request ports.ExecutionRequest if err := s.executions.CreateRun(ctx, request.Run); err != nil { return domain.ExecutionTrace{}, fmt.Errorf("create execution run: %w", err) } - if request.Recovery != nil { - if recoverErr := s.RecoverWorkspaces(ctx); recoverErr != nil { - return domain.ExecutionTrace{}, fmt.Errorf("reconcile workspaces before recovery: %w", recoverErr) - } - } defer func() { + if errors.Is(err, context.Canceled) { + if cause := context.Cause(ctx); cause != nil && !errors.Is(cause, context.Canceled) { + err = fmt.Errorf("%w; cancellation cause: %v", err, cause) + } + } if request.Run.Mode != domain.ExecutionModeSimulation && request.Run.Mode != domain.ExecutionModeInteractive { // Lifecycle cleanup belongs to the infrastructure result, not the // scientific result. Release records its own failed instance status; @@ -136,12 +142,27 @@ func (s *Supervisor) releaseCloud(ctx context.Context, request ports.ExecutionRe if s.config.CloudAllocator != nil { allocations := request.RuntimeAllocations retained := 0 + if failed { + if store := s.workspaceStore(); store != nil { + workspaces, err := store.ListWorkspaces(ctx, request.Run.ID) + if err != nil { + return 0, fmt.Errorf("inspect producer workspaces before cloud cleanup: %w", err) + } + leases, err := store.ListWorkspaceLeases(ctx, request.Run.ID) + if err != nil { + return 0, fmt.Errorf("inspect producer leases before cloud cleanup: %w", err) + } + allocations, retained = retainCloudSourcesWithWorkspaceLeases(allocations, workspaces, leases) + } + } if failed && s.config.Data != nil { locations, err := s.config.Data.ListLocations(ctx, request.Run.ID) if err != nil { return 0, fmt.Errorf("inspect ephemeral outputs before cloud cleanup: %w", err) } - allocations, retained = retainCloudSourcesWithEphemeralOutputs(allocations, locations) + var additional int + allocations, additional = retainCloudSourcesWithEphemeralOutputs(allocations, locations) + retained += additional } if !failed { if stopper, ok := s.config.CloudAllocator.(CloudStopper); ok { @@ -185,6 +206,34 @@ func (s *Supervisor) releaseCloud(ctx context.Context, request ports.ExecutionRe return 0, nil } +func retainCloudSourcesWithWorkspaceLeases(allocations map[string]domain.RuntimeAllocation, workspaces []domain.ActivityWorkspace, leases []domain.WorkspaceLease) (map[string]domain.RuntimeAllocation, int) { + needed := make(map[string]bool) + for _, lease := range leases { + if !lease.Released { + needed[lease.WorkspaceID] = true + } + } + protected := make(map[string]bool) + for _, workspace := range workspaces { + if workspace.State == domain.WorkspaceReleased || !needed[workspace.ID] { + continue + } + uri, err := url.Parse(workspace.URI) + if err == nil { + if id := uri.Query().Get("cloudInstanceId"); id != "" { + protected[id] = true + } + } + } + remaining := make(map[string]domain.RuntimeAllocation) + for activity, allocation := range allocations { + if !protected[allocation.CloudInstanceID] { + remaining[activity] = allocation + } + } + return remaining, len(protected) +} + func retainCloudSourcesWithEphemeralOutputs(allocations map[string]domain.RuntimeAllocation, locations []domain.DataLocation) (map[string]domain.RuntimeAllocation, int) { protectedResources := make(map[string]bool) for _, location := range locations { @@ -207,7 +256,7 @@ func retainCloudSourcesWithEphemeralOutputs(allocations map[string]domain.Runtim return remaining, len(protectedInstances) } -func (s *Supervisor) executeActivities(ctx context.Context, request ports.ExecutionRequest) (domain.ExecutionTrace, error) { +func (s *Supervisor) executeActivities(ctx context.Context, request ports.ExecutionRequest) (trace domain.ExecutionTrace, executionErr error) { if request.RuntimeAllocations == nil { request.RuntimeAllocations = make(map[string]domain.RuntimeAllocation) } @@ -226,59 +275,77 @@ func (s *Supervisor) executeActivities(ctx context.Context, request ports.Execut tasks := make(map[string]domain.TaskExecution) attempts := make(map[string]int) if request.Recovery != nil { - reader, ok := s.executions.(interface { - ListTasks(context.Context, string) ([]domain.TaskExecution, error) - }) - if !ok { - return domain.ExecutionTrace{}, fmt.Errorf("execution repository does not support recovery") - } - previous, loadErr := reader.ListTasks(ctx, request.Run.ID) - if loadErr != nil { - return domain.ExecutionTrace{}, fmt.Errorf("load prior activity attempts: %w", loadErr) - } - reusable := make(map[string]bool, len(request.Recovery.ReuseActivityIDs)) - for _, activityID := range request.Recovery.ReuseActivityIDs { - reusable[activityID] = true - } - latest := make(map[string]domain.TaskExecution) - for _, task := range previous { - if task.Attempt > attempts[task.ActivityID] { - attempts[task.ActivityID] = task.Attempt - latest[task.ActivityID] = task - } - } - if store := s.workspaceStore(); store != nil { - workspaces, workspaceErr := store.ListWorkspaces(ctx, request.Run.ID) - if workspaceErr != nil { - return domain.ExecutionTrace{}, fmt.Errorf("load prior activity workspaces: %w", workspaceErr) - } - available := make(map[string]bool, len(workspaces)) - for _, workspace := range workspaces { - available[workspace.ActivityID] = workspace.State != domain.WorkspaceReleased && workspace.State != domain.WorkspaceFailed - } - reconcileReusableWorkspaces(activities, predecessors, reusable, available) - } - for _, task := range latest { - if task.Status == domain.TaskCompleted && reusable[task.ActivityID] { - tasks[task.ActivityID], completed[task.ActivityID] = task, task - } + if err := s.restoreActivityAttempts(ctx, request, activities, predecessors, completed, tasks, attempts); err != nil { + return domain.ExecutionTrace{}, err } } transfers := make([]domain.DataTransfer, 0) - type allocationResult struct { - activityID string - allocation domain.RuntimeAllocation - err error - } allocationResults := make(chan allocationResult, len(activities)) pendingAllocations := make(map[string]bool) readyAllocations := make(map[string]domain.RuntimeAllocation) stoppingInstances := make(map[string]bool) + idleStops := newIdleCloudStops(ctx) + defer idleStops.close() + startCtx, cancelStarts := context.WithCancel(ctx) + startResults := make(chan startResult, len(activities)) + pendingStarts := make(map[string]bool) + var startWorkers sync.WaitGroup + consumeStart := func(result startResult) error { + delete(pendingStarts, result.activityID) + return s.applyStartResult(ctx, request, result, running, completed, tasks, &transfers) + } + // Join preparation workers before the caller releases cloud allocations. + // A Start that succeeds concurrently with cancellation still belongs to us. + defer func() { + cancelStarts() + startWorkers.Wait() + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 60*time.Second) + defer cancel() + for len(pendingAllocations) > 0 { + result := <-allocationResults + delete(pendingAllocations, result.activityID) + if result.allocation.ResourceID != "" { + request.RuntimeAllocations[result.activityID] = result.allocation + } + } + for len(pendingStarts) > 0 { + result := <-startResults + delete(pendingStarts, result.activityID) + if err := s.applyStartResult(cleanupCtx, request, result, running, completed, tasks, &transfers); executionErr == nil && err != nil { + executionErr = err + } + } + if executionErr != nil { + s.cancelRunningActivities(cleanupCtx, request.Run.Mode, running, tasks, executionErr) + } + }() + consumeAllocation := func(result allocationResult) error { + delete(pendingAllocations, result.activityID) + if result.err != nil { + failure := fmt.Errorf("allocate runtime for activity %q: %w", result.activityID, result.err) + assignment := assignments[result.activityID] + resource := resources[assignment.ResourceID] + _ = s.recordStartFailure(ctx, request.Run.ID, result.activityID, assignment, resource, selectRuntime(request, assignment), tasks[result.activityID].Attempt, failure) + return failure + } + readyAllocations[result.activityID] = result.allocation + return nil + } for len(completed) < len(activities) { if err := ctx.Err(); err != nil { return domain.ExecutionTrace{}, err } + for len(startResults) > 0 { + if err := consumeStart(<-startResults); err != nil { + return domain.ExecutionTrace{}, err + } + } + // Admit all completed allocations before launching a batch. Waiting for + // one runtime Start must not serialize other already-ready cloud hosts. + if err := drainReadyAllocations(allocationResults, consumeAllocation); err != nil { + return domain.ExecutionTrace{}, err + } if err := s.inspectRunning(ctx, request.Run.Mode, request.Workflow, request.Run.ID, running, completed, tasks); err != nil { return domain.ExecutionTrace{}, err } @@ -289,49 +356,11 @@ func (s *Supervisor) executeActivities(ctx context.Context, request ports.Execut admitted := plannedReadyActivities( ready, assignments, pendingAllocations, ) - for _, activityID := range ready { - queueReason := "awaiting assigned runtime" - if queued, exists := tasks[activityID]; exists { - if queued.Metadata["queueReason"] != queueReason { - if queued.Metadata == nil { - queued.Metadata = make(map[string]any) - } - queued.Metadata["queueReason"] = queueReason - if err := s.executions.SaveTask(ctx, queued); err != nil { - return domain.ExecutionTrace{}, fmt.Errorf("update queue for activity %q: %w", activityID, err) - } - tasks[activityID] = queued - } - continue - } - now := unixNow() - assignment := assignments[activityID] - attempt := attempts[activityID] + 1 - queued := domain.TaskExecution{ - ID: taskExecutionID(request.Run.ID, activityID, attempt), ExecutionRunID: request.Run.ID, - PlanAssignmentID: assignment.ID, ActivityID: activityID, - PlannedResourceID: assignment.ResourceID, Attempt: attempt, - Status: domain.TaskQueued, ReadyAt: now, QueuedAt: now, - Metadata: map[string]any{"queueReason": queueReason}, - } - if err := s.executions.SaveTask(ctx, queued); err != nil { - return domain.ExecutionTrace{}, fmt.Errorf("queue activity %q: %w", activityID, err) - } - tasks[activityID] = queued - } - candidates := make([]string, 0) - immediate := make([]string, 0) - for _, activityID := range admitted { - if runtimeDriver(request, activityID) != domain.RuntimeDriverCloud { - immediate = append(immediate, activityID) - continue - } - if _, allocated := readyAllocations[activityID]; allocated { - immediate = append(immediate, activityID) - continue - } - candidates = append(candidates, activityID) + admitted = restrictedCapacityReady(admitted, assignments, resources, tasks, pendingAllocations) + if err := s.queueReadyActivities(ctx, request, ready, assignments, tasks, attempts); err != nil { + return domain.ExecutionTrace{}, err } + candidates, immediate := partitionReadyAllocations(request, admitted, readyAllocations) if err := s.prewarmReadyCloud(ctx, request, candidates, assignments, resources); err != nil { return domain.ExecutionTrace{}, fmt.Errorf("prewarm cloud capacity: %w", err) } @@ -339,38 +368,56 @@ func (s *Supervisor) executeActivities(ctx context.Context, request ports.Execut assignment := assignments[activityID] resource := resources[assignment.ResourceID] pendingAllocations[activityID] = true + allocationRequest := request + allocationRequest.RuntimeAllocations = cloneRuntimeAllocations(request.RuntimeAllocations) + startWorkers.Add(1) go func(activityID string, assignment domain.PlanAssignment, resource domain.Resource) { - allocation, err := s.ensureRuntimeAllocation(ctx, request, activityID, assignment, resource) + defer startWorkers.Done() + allocation, err := s.ensureRuntimeAllocation(startCtx, allocationRequest, activityID, assignment, resource) allocationResults <- allocationResult{activityID: activityID, allocation: allocation, err: err} }(activityID, assignment, resource) } - if err := s.startReadyActivities( - ctx, request, immediate, activities, assignments, resources, - running, completed, tasks, &transfers, readyAllocations, - ); err != nil { - return domain.ExecutionTrace{}, err - } for _, activityID := range immediate { + if len(pendingStarts) >= readyActivityConcurrency { + break + } + task := tasks[activityID] + task.Status = domain.TaskPreparing + if task.Metadata == nil { + task.Metadata = make(map[string]any) + } + task.Metadata["queueReason"] = "synchronizing predecessor workspaces and preparing runtime" + if err := s.executions.SaveTask(ctx, task); err != nil { + return domain.ExecutionTrace{}, fmt.Errorf("mark activity %q preparing: %w", activityID, err) + } + tasks[activityID] = task + startRequest := request + startRequest.RuntimeAllocations = cloneRuntimeAllocations(request.RuntimeAllocations) + allocated := cloneRuntimeAllocations(readyAllocations) + pendingStarts[activityID] = true + startWorkers.Add(1) + go func(activityID string, attempt int) { + defer startWorkers.Done() + startResults <- s.startReadyActivity(startCtx, startRequest, activityID, attempt, activities, assignments, resources, allocated) + }(activityID, task.Attempt) delete(readyAllocations, activityID) } - s.stopIdleCloud(ctx, request, completed, tasks, stoppingInstances) - if len(running) == 0 && len(pendingAllocations) == 0 && len(immediate) == 0 && len(completed) < len(activities) { + s.scheduleIdleCloudStops(request, completed, tasks, stoppingInstances, idleStops) + if len(running) == 0 && len(pendingStarts) == 0 && len(pendingAllocations) == 0 && len(immediate) == 0 && len(completed) < len(activities) { return domain.ExecutionTrace{}, fmt.Errorf("execution cannot progress: dependency cycle or incomplete plan") } - if len(running) > 0 || len(pendingAllocations) > 0 { + if len(running) > 0 || len(pendingStarts) > 0 || len(pendingAllocations) > 0 { select { case <-ctx.Done(): return domain.ExecutionTrace{}, ctx.Err() + case result := <-startResults: + if err := consumeStart(result); err != nil { + return domain.ExecutionTrace{}, err + } case result := <-allocationResults: - delete(pendingAllocations, result.activityID) - if result.err != nil { - failure := fmt.Errorf("allocate runtime for activity %q: %w", result.activityID, result.err) - assignment := assignments[result.activityID] - resource := resources[assignment.ResourceID] - _ = s.recordStartFailure(ctx, request.Run.ID, result.activityID, assignment, resource, selectRuntime(request, assignment), tasks[result.activityID].Attempt, failure) - return domain.ExecutionTrace{}, failure + if err := consumeAllocation(result); err != nil { + return domain.ExecutionTrace{}, err } - readyAllocations[result.activityID] = result.allocation case <-time.After(s.config.PollInterval): } } @@ -378,6 +425,22 @@ func (s *Supervisor) executeActivities(ctx context.Context, request ports.Execut return completedTrace(request, tasks, transfers), nil } +func partitionReadyAllocations(request ports.ExecutionRequest, admitted []string, readyAllocations map[string]domain.RuntimeAllocation) ([]string, []string) { + candidates, immediate := make([]string, 0), make([]string, 0) + for _, activityID := range admitted { + if runtimeDriver(request, activityID) != domain.RuntimeDriverCloud { + immediate = append(immediate, activityID) + continue + } + if _, allocated := readyAllocations[activityID]; allocated { + immediate = append(immediate, activityID) + continue + } + candidates = append(candidates, activityID) + } + return candidates, immediate +} + func reconcileReusableWorkspaces( activities map[string]domain.Activity, predecessors map[string][]string, @@ -573,6 +636,8 @@ func (s *Supervisor) cancelRunningActivities( tasks map[string]domain.TaskExecution, cause error, ) { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 60*time.Second) + defer cancel() type stopResult struct { activityID string err error @@ -586,7 +651,7 @@ func (s *Supervisor) cancelRunningActivities( defer group.Done() semaphore <- struct{}{} defer func() { <-semaphore }() - results <- stopResult{activityID: activityID, err: s.activities.Stop(ctx, handleID, mode)} + results <- stopResult{activityID: activityID, err: s.activities.Stop(cleanupCtx, handleID, mode)} }(activityID, handle.ID) } group.Wait() @@ -649,75 +714,49 @@ func validateExpectedOutputInstances(activityID string, values []string, instanc return nil } -func (s *Supervisor) startReadyActivities( +func (s *Supervisor) applyStartResult( ctx context.Context, request ports.ExecutionRequest, - ready []string, - activities map[string]domain.Activity, - assignments map[string]domain.PlanAssignment, - resources map[string]domain.Resource, + result startResult, running map[string]domain.ActivityHandle, completed map[string]domain.TaskExecution, tasks map[string]domain.TaskExecution, transfers *[]domain.DataTransfer, - readyAllocations map[string]domain.RuntimeAllocation, ) error { - results := make([]startResult, len(ready)) - semaphore := make(chan struct{}, readyActivityConcurrency) - var group sync.WaitGroup - for _, activityID := range ready { - task, exists := tasks[activityID] - if !exists { - continue - } - task.Status = domain.TaskPreparing - if task.Metadata == nil { - task.Metadata = make(map[string]any) - } - task.Metadata["queueReason"] = "synchronizing predecessor workspaces and preparing runtime" - if err := s.executions.SaveTask(ctx, task); err != nil { - return fmt.Errorf("mark activity %q preparing: %w", activityID, err) - } - tasks[activityID] = task - } - for index, activityID := range ready { - group.Add(1) - go func(index int, activityID string) { - defer group.Done() - semaphore <- struct{}{} - defer func() { <-semaphore }() - results[index] = s.startReadyActivity(ctx, request, activityID, activities, assignments, resources, readyAllocations) - }(index, activityID) - } - group.Wait() - for _, result := range results { - activityID := result.activityID - if result.err != nil { - attempt := tasks[activityID].Attempt - _ = s.recordStartFailure(ctx, request.Run.ID, activityID, result.assignment, result.resource, selectRuntime(request, result.assignment), attempt, result.err) - return result.err - } + activityID := result.activityID + if result.allocation.ResourceID != "" { request.RuntimeAllocations[activityID] = result.allocation - *transfers = append(*transfers, result.transfers...) - queued := tasks[activityID] - task := newRunningTask(request.Run.ID, activityID, result.assignment, result.resource, result.allocation, result.handle, queued.Attempt, result.readyAt, result.preparation) - if queued.Attempt > 0 { - task.ReadyAt = queued.ReadyAt - task.QueuedAt = queued.QueuedAt - task.QueueSeconds = maxFloat(0, task.StartedAt-queued.QueuedAt) - } - tasks[activityID], running[activityID] = task, result.handle - if result.handle.Status == domain.HandleCompleted { - completeTask(&task, result.handle) - if err := s.sealWorkspace(ctx, request.Run.ID, activityID, result.handle.Artifacts); err != nil { - return fmt.Errorf("seal workspace for activity %q: %w", activityID, err) - } - tasks[activityID], completed[activityID] = task, task - delete(running, activityID) - } - if err := s.executions.SaveTask(ctx, task); err != nil { - return err + } + if result.err != nil { + attempt := tasks[activityID].Attempt + if result.handle.Metadata[ports.RuntimeReconciliationPending] == true { + running[activityID] = result.handle + } + if err := s.recordStartFailure(ctx, request.Run.ID, activityID, result.assignment, result.resource, selectRuntime(request, result.assignment), attempt, result.err); err != nil { + return fmt.Errorf("record startup failure: %v: %w", err, result.err) + } + return result.err + } + request.RuntimeAllocations[activityID] = result.allocation + *transfers = append(*transfers, result.transfers...) + queued := tasks[activityID] + task := newRunningTask(request.Run.ID, activityID, result.assignment, result.resource, result.allocation, result.handle, queued.Attempt, result.readyAt, result.preparation) + if queued.Attempt > 0 { + task.ReadyAt = queued.ReadyAt + task.QueuedAt = queued.QueuedAt + task.QueueSeconds = maxFloat(0, task.StartedAt-queued.QueuedAt) + } + tasks[activityID], running[activityID] = task, result.handle + if result.handle.Status == domain.HandleCompleted { + completeTask(&task, result.handle) + if err := s.sealWorkspace(ctx, request.Run.ID, activityID, result.handle.Artifacts); err != nil { + return fmt.Errorf("seal workspace for activity %q: %w", activityID, err) } + tasks[activityID], completed[activityID] = task, task + delete(running, activityID) + } + if err := s.executions.SaveTask(ctx, task); err != nil { + return err } return nil } @@ -726,6 +765,7 @@ func (s *Supervisor) startReadyActivity( ctx context.Context, request ports.ExecutionRequest, activityID string, + attempt int, activities map[string]domain.Activity, assignments map[string]domain.PlanAssignment, resources map[string]domain.Resource, @@ -774,14 +814,16 @@ func (s *Supervisor) startReadyActivity( result.err = fmt.Errorf("activity %q has an executable reference but no preparation requirement", activityID) return result } - if err := s.activateWorkspace(ctx, request.Run.ID, activityID); err != nil { + if err := s.activateWorkspace(ctx, request.Run.ID, activityID, allocation); err != nil { result.err = fmt.Errorf("activate workspace for activity %q: %w", activityID, err) return result } handle, err := s.activities.Start(ctx, domain.ActivityExecutionContext{ - Run: request.Run, Workflow: request.Workflow, Activity: activity, Assignment: assignment, + Attempt: attempt, + Run: request.Run, Workflow: request.Workflow, Activity: activity, Assignment: assignment, Resource: resource, RuntimeID: selectRuntime(request, assignment), Allocation: allocation, Preparation: result.preparation, }) + result.handle = handle if err != nil { result.err = err return result @@ -911,6 +953,13 @@ func (s *Supervisor) addWorkspacePreparation( if sourceErr != nil { return sourceErr } + if store := s.workspaceStore(); store != nil { + workspace, err := store.FindWorkspace(ctx, "workspace-"+request.Run.ID+"-"+producerID) + if err != nil { + return fmt.Errorf("read producer snapshot identity: %w", err) + } + source.WorkspaceSnapshotID, source.WorkspaceSnapshotEntries = sealedWorkspaceSnapshot(workspace, source.CloudInstanceID) + } requirement.WorkspaceTransfers = append(requirement.WorkspaceTransfers, domain.DataTransferPlan{ ID: fmt.Sprintf("transfer-workspace-%s-%s-%d", request.Run.ID, activityID, index), ExecutionRunID: request.Run.ID, @@ -1196,10 +1245,7 @@ func transferObservations( // Artifact stores are not execution resources in the current transfer // schema. Attribute ingress to the selected resource until the schema // includes a storage endpoint dimension. - bytes := observation.NetworkBytes - if bytes <= 0 { - bytes = observation.TransferredBytes - } + bytes := observedNetworkBytes(observation) transfers = append(transfers, domain.DataTransfer{ ID: runID + ":" + activityID + ":" + observation.ID, ExecutionRunID: runID, @@ -1207,16 +1253,18 @@ func transferObservations( ConsumerActivityID: activityID, SourceResourceID: sourceResourceID, TargetResourceID: targetResourceID, - Bytes: observation.TransferredBytes, - StartedAt: observation.StartedAt, - FinishedAt: observation.FinishedAt, - DurationSeconds: maxFloat(0, observation.FinishedAt-observation.StartedAt), - Cost: transferPrice(topology, sourceResourceID, targetResourceID) * float64(bytes), - Strategy: observation.Strategy, - Route: observation.Route, - LogicalBytes: observation.LogicalBytes, - NetworkBytes: observation.NetworkBytes, - FilesTransferred: observation.FilesTransferred, + // Comparable with native SimGrid: physical traffic, not the + // uncompressed filesystem copy work retained in LogicalBytes. + Bytes: bytes, + StartedAt: observation.StartedAt, + FinishedAt: observation.FinishedAt, + DurationSeconds: maxFloat(0, observation.FinishedAt-observation.StartedAt), + Cost: transferPrice(topology, sourceResourceID, targetResourceID) * float64(bytes), + Strategy: observation.Strategy, + Route: observation.Route, + LogicalBytes: observation.LogicalBytes, + NetworkBytes: observation.NetworkBytes, + FilesTransferred: observation.FilesTransferred, }) } return transfers @@ -1244,8 +1292,9 @@ func (s *Supervisor) recordStartFailure( ) error { now := float64(time.Now().UnixNano()) / float64(time.Second) message := "start activity: " + err.Error() + handleID := (domain.ActivityExecutionContext{Run: domain.ExecutionRun{ID: runID}, Activity: domain.Activity{ID: activityID}, Attempt: attempt}).HandleID() handle := domain.ActivityHandle{ - ID: runID + ":" + activityID, + ID: handleID, RunID: runID, ActivityID: activityID, ResourceID: resource.ID, @@ -1255,6 +1304,14 @@ func (s *Supervisor) recordStartFailure( FinishedAt: now, Failure: message, Log: "[AKOFLOW ERROR] " + message + "\n", } + var uncertain ports.RuntimeReconciliationFailure + if errors.As(err, &uncertain) { + handle.Status, handle.FinishedAt = domain.HandleStarting, 0 + handle.Metadata = uncertain.RuntimeReconciliationMetadata() + handle.Metadata[ports.RuntimeReconciliationPending] = true + } + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() if saveErr := s.executions.Save(ctx, handle); saveErr != nil { return saveErr } @@ -1316,7 +1373,7 @@ func newRunningTask( var preparationStartedAt, preparationFinishedAt float64 for _, transfer := range preparation.TransferRuns { task.TransferSeconds += maxFloat(0, transfer.FinishedAt-transfer.StartedAt) - task.TransferBytes += transfer.TransferredBytes + task.TransferBytes += observedNetworkBytes(transfer) if transfer.StartedAt > 0 && (preparationStartedAt == 0 || transfer.StartedAt < preparationStartedAt) { preparationStartedAt = transfer.StartedAt } @@ -1358,6 +1415,23 @@ func completeTask(task *domain.TaskExecution, handle domain.ActivityHandle) { } func applyHandleTiming(task *domain.TaskExecution, handle domain.ActivityHandle) { + // Publish bounded timing evidence, not observation paths or file manifests. + // These diagnostics overlap launch wait and must not be added to it again. + for _, key := range []string{ + "remoteWorkspacePrepareSeconds", "remoteWorkspaceSnapshotSeconds", + "remoteWorkspaceSnapshotServiceSeconds", + "remoteContainerLaunchSeconds", "remoteInitialTelemetrySeconds", + "remoteArtifactObservationStartedAt", "remoteArtifactObservationFinishedAt", + "remoteArtifactObservationSeconds", "remoteArtifactsReadyAt", "terminalObservedAt", + "remoteArtifactObservationServiceSeconds", + } { + if value, ok := metadataFloat(handle.Metadata, key); ok && value >= 0 { + if task.Metadata == nil { + task.Metadata = map[string]any{} + } + task.Metadata[key] = value + } + } if submittedAt, ok := metadataFloat(handle.Metadata, domain.TimingSubmittedAt); ok { task.QueuedAt = submittedAt task.QueueSeconds = maxFloat(0, handle.StartedAt-submittedAt) @@ -1408,7 +1482,19 @@ func validateRequest(request ports.ExecutionRequest) error { if len(request.Plan.Assignments) != len(request.Workflow.Activities) { return fmt.Errorf("plan must assign every activity exactly once") } + resources := indexResources(request.Resources) + lanes := make(map[string]map[string]bool) for _, assignment := range request.Plan.Assignments { + resource := resources[assignment.ResourceID] + if resource.ExecutionTarget != domain.ExecutionTargetBatch && resource.CPUCapacity > 0 && resource.CPUCapacity < float64(resource.CPUCores) && assignment.CoreID != "" { + if lanes[resource.ID] == nil { + lanes[resource.ID] = make(map[string]bool) + } + lanes[resource.ID][assignment.CoreID] = true + if len(lanes[resource.ID]) > resource.SchedulingCPUCores() { + return fmt.Errorf("plan exceeds explicit CPU capacity for resource %q: %d lanes, capacity %d", resource.ID, len(lanes[resource.ID]), resource.SchedulingCPUCores()) + } + } if selectRuntime(request, assignment) == "" { return fmt.Errorf("resource %q has no enabled runtime for %q mode", assignment.ResourceID, request.Run.Mode) @@ -1552,3 +1638,107 @@ func maxFloat(a, b float64) float64 { } return b } + +func (s *Supervisor) restoreActivityAttempts( + ctx context.Context, request ports.ExecutionRequest, + activities map[string]domain.Activity, predecessors map[string][]string, + completed, tasks map[string]domain.TaskExecution, attempts map[string]int, +) error { + reader, ok := s.executions.(interface { + ListTasks(context.Context, string) ([]domain.TaskExecution, error) + }) + if !ok { + return fmt.Errorf("execution repository does not support recovery") + } + previous, loadErr := reader.ListTasks(ctx, request.Run.ID) + if loadErr != nil { + return fmt.Errorf("load prior activity attempts: %w", loadErr) + } + reusable := make(map[string]bool, len(request.Recovery.ReuseActivityIDs)) + for _, activityID := range request.Recovery.ReuseActivityIDs { + reusable[activityID] = true + } + latest := make(map[string]domain.TaskExecution) + for _, task := range previous { + if task.Attempt > attempts[task.ActivityID] { + attempts[task.ActivityID] = task.Attempt + latest[task.ActivityID] = task + } + } + if store := s.workspaceStore(); store != nil { + workspaces, workspaceErr := store.ListWorkspaces(ctx, request.Run.ID) + if workspaceErr != nil { + return fmt.Errorf("load prior activity workspaces: %w", workspaceErr) + } + available := make(map[string]bool, len(workspaces)) + for _, workspace := range workspaces { + available[workspace.ActivityID] = workspace.State != domain.WorkspaceReleased && workspace.State != domain.WorkspaceFailed + if reusable[workspace.ActivityID] && available[workspace.ActivityID] && s.config.Workspaces != nil { + allocation := recoveredAllocation(latest[workspace.ActivityID]) + if allocation.CloudInstanceID != "" { + if readier, ok := s.config.CloudAllocator.(CloudRecoveryReadier); ok { + if err := readier.ReadyRecoveryInstance(ctx, request.Run.ID, workspace.ActivityID, allocation.CloudInstanceID); err != nil { + return fmt.Errorf("ready recovery producer: %w", err) + } + } + } + if err := bindWorkspaceAllocation(&workspace, recoveredAllocation(latest[workspace.ActivityID])); err != nil { + return err + } + if err := store.SaveWorkspace(ctx, workspace); err != nil { + return err + } + usage, inspectErr := s.config.Workspaces.Inspect(ctx, workspace) + if inspectErr != nil { + return fmt.Errorf("inspect recovery workspace %q: %w", workspace.ID, inspectErr) + } + available[workspace.ActivityID] = recoveryWorkspacePresent(workspace, usage) + } + } + reconcileReusableWorkspaces(activities, predecessors, reusable, available) + if err := restoreRecoveryWorkspaceLeases(ctx, store, request.Run.ID, reusable); err != nil { + return err + } + } + for _, task := range latest { + if task.Status == domain.TaskCompleted && reusable[task.ActivityID] { + tasks[task.ActivityID], completed[task.ActivityID] = task, task + request.RuntimeAllocations[task.ActivityID] = recoveredAllocation(task) + } + } + return nil +} + +func (s *Supervisor) queueReadyActivities(ctx context.Context, request ports.ExecutionRequest, ready []string, assignments map[string]domain.PlanAssignment, tasks map[string]domain.TaskExecution, attempts map[string]int) error { + for _, activityID := range ready { + queueReason := "awaiting assigned runtime" + if queued, exists := tasks[activityID]; exists { + if queued.Metadata["queueReason"] != queueReason { + if queued.Metadata == nil { + queued.Metadata = make(map[string]any) + } + queued.Metadata["queueReason"] = queueReason + if err := s.executions.SaveTask(ctx, queued); err != nil { + return fmt.Errorf("update queue for activity %q: %w", activityID, err) + } + tasks[activityID] = queued + } + continue + } + now := unixNow() + assignment := assignments[activityID] + attempt := attempts[activityID] + 1 + queued := domain.TaskExecution{ + ID: taskExecutionID(request.Run.ID, activityID, attempt), ExecutionRunID: request.Run.ID, + PlanAssignmentID: assignment.ID, ActivityID: activityID, + PlannedResourceID: assignment.ResourceID, Attempt: attempt, + Status: domain.TaskQueued, ReadyAt: now, QueuedAt: now, + Metadata: map[string]any{"queueReason": queueReason}, + } + if err := s.executions.SaveTask(ctx, queued); err != nil { + return fmt.Errorf("queue activity %q: %w", activityID, err) + } + tasks[activityID] = queued + } + return nil +} diff --git a/internal/controlplane/execution/supervisor_cancellation_test.go b/internal/controlplane/execution/supervisor_cancellation_test.go new file mode 100644 index 00000000..cfd8628c --- /dev/null +++ b/internal/controlplane/execution/supervisor_cancellation_test.go @@ -0,0 +1,58 @@ +package execution + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" +) + +type cancelledExecutionPlan struct{} + +type cleanupContextController struct { + activityControllerFake + err error +} + +func (c *cleanupContextController) Stop(ctx context.Context, _ string, _ domain.ExecutionMode) error { + c.err = ctx.Err() + if _, ok := ctx.Deadline(); !ok { + return errors.New("cleanup has no bounded deadline") + } + return c.err +} + +func TestFailureCleanupUsesBoundedUncancelledContext(t *testing.T) { + store := &executionStoreFake{} + controller := &cleanupContextController{} + s, _ := New(store, controller, &planExecutorFake{}, Config{}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + running := map[string]domain.ActivityHandle{"a": {ID: "h"}} + tasks := map[string]domain.TaskExecution{"a": {ActivityID: "a", Status: domain.TaskRunning}} + s.cancelRunningActivities(ctx, domain.ExecutionModeReal, running, tasks, context.Canceled) + if controller.err != nil || len(running) != 0 || tasks["a"].Status != domain.TaskCancelled { + t.Fatalf("cleanup err=%v running=%v task=%+v", controller.err, running, tasks["a"]) + } +} + +func (cancelledExecutionPlan) Execute(ctx context.Context, _ ports.ExecutionRequest) (domain.ExecutionTrace, error) { + return domain.ExecutionTrace{}, ctx.Err() +} + +func TestSupervisorPersistsLeaseCancellationCause(t *testing.T) { + store := &executionStoreFake{} + service, err := New(store, &activityControllerFake{}, cancelledExecutionPlan{}, Config{}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancelCause(context.Background()) + cancel(errors.New("renew lease for queue job example: lease ownership lost")) + _, err = service.Execute(ctx, requestFixture(domain.ExecutionModeSimulation)) + if !errors.Is(err, context.Canceled) || !strings.Contains(store.failed, "lease ownership lost") { + t.Fatalf("lease cancellation hidden from run: err=%v failure=%q", err, store.failed) + } +} diff --git a/internal/controlplane/execution/supervisor_duplicate_test.go b/internal/controlplane/execution/supervisor_duplicate_test.go new file mode 100644 index 00000000..55d2a2ca --- /dev/null +++ b/internal/controlplane/execution/supervisor_duplicate_test.go @@ -0,0 +1,45 @@ +package execution + +import ( + "context" + "strings" + "testing" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" +) + +type blockingExecutionPlan struct { + started chan struct{} + release chan struct{} +} + +func (p *blockingExecutionPlan) Execute(_ context.Context, request ports.ExecutionRequest) (domain.ExecutionTrace, error) { + close(p.started) + <-p.release + return domain.ExecutionTrace{RunID: request.Run.ID}, nil +} + +func TestSupervisorRejectsConcurrentCommandWithoutMutatingRun(t *testing.T) { + store := &executionStoreFake{} + plan := &blockingExecutionPlan{started: make(chan struct{}), release: make(chan struct{})} + service, err := New(store, &activityControllerFake{}, plan, Config{}) + if err != nil { + t.Fatal(err) + } + request := requestFixture(domain.ExecutionModeSimulation) + done := make(chan error, 1) + go func() { + _, err := service.Execute(context.Background(), request) + done <- err + }() + <-plan.started + _, err = service.Execute(context.Background(), request) + if err == nil || !strings.Contains(err.Error(), "active supervisor") || len(store.runs) != 1 || store.failed != "" { + t.Fatalf("duplicate command mutated original execution: err=%v runs=%d failed=%q", err, len(store.runs), store.failed) + } + close(plan.release) + if err := <-done; err != nil { + t.Fatal(err) + } +} diff --git a/internal/controlplane/execution/supervisor_independent_starts_test.go b/internal/controlplane/execution/supervisor_independent_starts_test.go new file mode 100644 index 00000000..ff109261 --- /dev/null +++ b/internal/controlplane/execution/supervisor_independent_starts_test.go @@ -0,0 +1,137 @@ +package execution + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +type independentStartController struct { + activityControllerFake + slowEntered chan struct{} + releaseSlow chan struct{} + successor chan struct{} + once sync.Once + lateHandle bool + failFast bool +} + +func (c *independentStartController) Start(ctx context.Context, execution domain.ActivityExecutionContext) (domain.ActivityHandle, error) { + if c.failFast && execution.Activity.ID == "a" { + select { + case <-c.slowEntered: + return domain.ActivityHandle{}, errors.New("start rejected") + case <-ctx.Done(): + return domain.ActivityHandle{}, ctx.Err() + } + } + if execution.Activity.ID == "slow" { + close(c.slowEntered) + select { + case <-c.releaseSlow: + case <-ctx.Done(): + if !c.lateHandle { + return domain.ActivityHandle{}, ctx.Err() + } + } + } + if execution.Activity.ID == "b" { + c.once.Do(func() { close(c.successor) }) + } + return c.activityControllerFake.Start(ctx, execution) +} + +func TestStartFailureStillOwnsAndStopsConcurrentLateHandle(t *testing.T) { + request := requestFixture(domain.ExecutionModeReal) + request.Workflow.Dependencies = nil + request.Workflow.Activities[1].ID = "slow" + request.Plan.Assignments[1].ActivityID = "slow" + c := &independentStartController{slowEntered: make(chan struct{}), releaseSlow: make(chan struct{}), successor: make(chan struct{}), lateHandle: true, failFast: true} + s, _ := New(&executionStoreFake{}, c, &planExecutorFake{}, Config{PollInterval: time.Millisecond}) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if _, err := s.Execute(ctx, request); err == nil || err.Error() != "start rejected" { + t.Fatalf("err=%v", err) + } + c.mu.Lock() + defer c.mu.Unlock() + if len(c.stopped) != 1 || c.stopped[0] != "h-slow" { + t.Fatalf("concurrent handle not stopped: %v", c.stopped) + } +} + +func TestIndependentStartPublishesSuccessorWhilePeerStartIsBlocked(t *testing.T) { + request := requestFixture(domain.ExecutionModeReal) + slow := request.Workflow.Activities[0] + slow.ID = "slow" + request.Workflow.Activities = append(request.Workflow.Activities, slow) + request.Plan.Assignments = append(request.Plan.Assignments, domain.PlanAssignment{ID: "ps", ActivityID: "slow", ResourceID: "r"}) + c := &independentStartController{slowEntered: make(chan struct{}), releaseSlow: make(chan struct{}), successor: make(chan struct{})} + s, _ := New(&executionStoreFake{}, c, &planExecutorFake{}, Config{PollInterval: time.Millisecond}) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { _, err := s.Execute(ctx, request); done <- err }() + select { + case <-c.slowEntered: + case <-ctx.Done(): + t.Fatal("slow start never entered") + } + select { + case <-c.successor: + case <-ctx.Done(): + t.Fatal("independent successor waited for blocked peer") + } + close(c.releaseSlow) + if err := <-done; err != nil { + t.Fatal(err) + } + c.mu.Lock() + defer c.mu.Unlock() + counts := map[string]int{} + for _, id := range c.started { + counts[id]++ + } + for _, id := range []string{"a", "b", "slow"} { + if counts[id] != 1 { + t.Fatalf("starts=%v", counts) + } + } +} + +func TestCancellationJoinsAndStopsLateSuccessfulStart(t *testing.T) { + request := requestFixture(domain.ExecutionModeReal) + request.Workflow.Activities = request.Workflow.Activities[:1] + request.Workflow.Activities[0].ID = "slow" + request.Workflow.Dependencies = nil + request.Plan.Assignments = []domain.PlanAssignment{{ID: "ps", ActivityID: "slow", ResourceID: "r"}} + c := &independentStartController{slowEntered: make(chan struct{}), releaseSlow: make(chan struct{}), successor: make(chan struct{}), lateHandle: true} + s, _ := New(&executionStoreFake{}, c, &planExecutorFake{}, Config{PollInterval: time.Millisecond}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { _, err := s.Execute(ctx, request); done <- err }() + select { + case <-c.slowEntered: + case <-time.After(3 * time.Second): + t.Fatal("slow start never entered") + } + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("err=%v", err) + } + case <-time.After(3 * time.Second): + t.Fatal("cancel did not join worker") + } + c.mu.Lock() + defer c.mu.Unlock() + if len(c.stopped) != 1 || c.stopped[0] != "h-slow" { + t.Fatalf("late handle not stopped: %v", c.stopped) + } +} diff --git a/internal/controlplane/execution/supervisor_test.go b/internal/controlplane/execution/supervisor_test.go index b6b4bdf8..d8308873 100644 --- a/internal/controlplane/execution/supervisor_test.go +++ b/internal/controlplane/execution/supervisor_test.go @@ -417,6 +417,27 @@ func TestTransferObservationsPreserveDirectWorkspaceRoute(t *testing.T) { } } +func TestTransferObservationsUseNetworkBytesForComparableTotals(t *testing.T) { + for _, test := range []struct { + name string + observation domain.DataTransferRun + want int64 + }{ + {"compressed relay two hops", domain.DataTransferRun{TransferredBytes: 1000, LogicalBytes: 1000, NetworkBytes: 600}, 600}, + {"private peer archive", domain.DataTransferRun{TransferredBytes: 1000, LogicalBytes: 1000, NetworkBytes: 300}, 300}, + {"same instance copy", domain.DataTransferRun{TransferredBytes: 1000, LogicalBytes: 1000, Route: domain.TransferRoute{SharedFilesystem: "ext2/ext3"}}, 0}, + {"HPC internal copy", domain.DataTransferRun{TransferredBytes: 1000, LogicalBytes: 1000, Route: domain.TransferRoute{SharedFilesystem: "nfs"}}, 0}, + {"legacy evidence absent", domain.DataTransferRun{TransferredBytes: 1000, LogicalBytes: 1000}, 1000}, + } { + t.Run(test.name, func(t *testing.T) { + transfers := transferObservations("run", "consumer", "target", nil, domain.PreparationRequirement{}, []domain.DataTransferRun{test.observation}, domain.NetworkTopology{}) + if len(transfers) != 1 || transfers[0].Bytes != test.want || transfers[0].LogicalBytes != 1000 { + t.Fatalf("physical total or logical evidence lost: %+v", transfers) + } + }) + } +} + func TestSupervisorExecutesDAGInDependencyOrder(t *testing.T) { store := &executionStoreFake{} activities := &activityControllerFake{} @@ -455,7 +476,7 @@ func TestSupervisorStartsTwelveActivitiesOnPlannedCapacity(t *testing.T) { }) } request.Resources[0].CPUCapacity = 12 - controller := &activityControllerFake{minimumStartsBeforeInspect: 12} + controller := &activityControllerFake{} service, err := New(&executionStoreFake{}, controller, &planExecutorFake{}, Config{ PollInterval: time.Microsecond, }) @@ -488,7 +509,7 @@ func TestSupervisorBoundsStartAndInspectionConcurrency(t *testing.T) { }) } request.Resources[0].CPUCapacity = 20 - controller := &activityControllerFake{minimumStartsBeforeInspect: 20, delay: 5 * time.Millisecond} + controller := &activityControllerFake{delay: 5 * time.Millisecond} service, err := New(&executionStoreFake{}, controller, &planExecutorFake{}, Config{PollInterval: time.Microsecond}) if err != nil { t.Fatal(err) @@ -535,7 +556,7 @@ func TestSupervisorStartsIndependentActivitiesDespitePlanOrder(t *testing.T) { request := requestFixture(domain.ExecutionModeReal) request.Workflow.Dependencies = nil request.Plan.Assignments[1].OrderOnResource = 1 - controller := &activityControllerFake{firstInspectStarts: 2} + controller := &activityControllerFake{} store := &executionStoreFake{} service, err := New(store, controller, &planExecutorFake{}, Config{PollInterval: time.Microsecond}) if err != nil { @@ -574,7 +595,7 @@ func TestSupervisorStartsReadyActivityAheadOfBlockedPlanOrder(t *testing.T) { request.RuntimeBindings = append(request.RuntimeBindings, domain.ResourceRuntimeBinding{ ResourceID: "other", RuntimeID: "local", Enabled: true, }) - controller := &activityControllerFake{firstInspectStarts: 2} + controller := &dependencyCompletionController{} service, err := New(&executionStoreFake{}, controller, &planExecutorFake{}, Config{ PollInterval: time.Microsecond, }) @@ -584,8 +605,12 @@ func TestSupervisorStartsReadyActivityAheadOfBlockedPlanOrder(t *testing.T) { if _, err := service.Execute(context.Background(), request); err != nil { t.Fatal(err) } - if len(controller.started) != 3 || controller.started[0] == "a" || controller.started[1] == "a" || controller.started[2] != "a" { - t.Fatalf("blocked activity ran before its dependency; starts=%v", controller.started) + counts := map[string]int{} + for _, id := range controller.started { + counts[id]++ + } + if len(controller.started) != 3 || counts["a"] != 1 || counts["b"] != 1 || counts["c"] != 1 { + t.Fatalf("each activity must start exactly once; starts=%v", controller.started) } } @@ -597,7 +622,7 @@ func TestSupervisorFollowsParallelPlanLanesDespiteOvercommit(t *testing.T) { request.Resources[0].CPUCapacity = 1 request.Plan.Assignments[0].SlotID = "slot-a" request.Plan.Assignments[1].SlotID = "slot-b" - controller := &activityControllerFake{firstInspectStarts: 2} + controller := &activityControllerFake{} service, err := New(&executionStoreFake{}, controller, &planExecutorFake{}, Config{PollInterval: time.Microsecond}) if err != nil { t.Fatal(err) @@ -605,6 +630,9 @@ func TestSupervisorFollowsParallelPlanLanesDespiteOvercommit(t *testing.T) { if _, err := service.Execute(context.Background(), request); err != nil { t.Fatal(err) } + if len(controller.started) != 2 { + t.Fatalf("started=%v, want both planned lanes", controller.started) + } } func TestSupervisorBindsCloudInstanceBeforeActivityStart(t *testing.T) { @@ -668,7 +696,7 @@ func TestSupervisorQueuesAllReadyCloudTargetsBeforeWaitingForFirst(t *testing.T) targets[targetID] = domain.CloudCapacityTarget{ID: targetID, EnvironmentID: "environment"} } allocator := ¶llelPrewarmFake{want: 4} - activities := &activityControllerFake{} + activities := &activityControllerFake{delay: 20 * time.Millisecond} service, err := New(&executionStoreFake{}, activities, &planExecutorFake{}, Config{ PollInterval: time.Microsecond, Cloud: &cloudProvisionerFake{}, CloudStore: cloudAllocationStoreFake{targets: targets}, CloudAllocator: allocator, @@ -682,6 +710,9 @@ func TestSupervisorQueuesAllReadyCloudTargetsBeforeWaitingForFirst(t *testing.T) if len(allocator.prewarmed) != 4 || len(activities.started) != 4 { t.Fatalf("prewarmed=%v started=%v", allocator.prewarmed, activities.started) } + if activities.maxStarts < 2 { + t.Fatalf("already-ready cloud allocations launched serially: max concurrent starts=%d", activities.maxStarts) + } } func TestSupervisorStartsReadyMachineWhileAnotherIsConfiguring(t *testing.T) { diff --git a/internal/controlplane/execution/transfer_metrics.go b/internal/controlplane/execution/transfer_metrics.go new file mode 100644 index 00000000..ccb02a80 --- /dev/null +++ b/internal/controlplane/execution/transfer_metrics.go @@ -0,0 +1,15 @@ +package execution + +import "github.com/UFFeScience/akoflow/internal/domain" + +// A verified internal filesystem copy has explicitly zero network traffic. +// Legacy observations lacking route evidence retain their previous byte count. +func observedNetworkBytes(transfer domain.DataTransferRun) int64 { + if transfer.NetworkBytes > 0 { + return transfer.NetworkBytes + } + if transfer.Route.SharedFilesystem != "" { + return 0 + } + return transfer.TransferredBytes +} diff --git a/internal/controlplane/execution/transfer_metrics_test.go b/internal/controlplane/execution/transfer_metrics_test.go new file mode 100644 index 00000000..edba4ec2 --- /dev/null +++ b/internal/controlplane/execution/transfer_metrics_test.go @@ -0,0 +1,25 @@ +package execution + +import ( + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestObservedNetworkBytesSeparatesSharedCopyFromTraffic(t *testing.T) { + for _, test := range []struct { + name string + run domain.DataTransferRun + want int64 + }{ + {"shared Lustre", domain.DataTransferRun{TransferredBytes: 4096, Route: domain.TransferRoute{SharedFilesystem: "lustre"}}, 0}, + {"network rsync", domain.DataTransferRun{TransferredBytes: 4096, NetworkBytes: 4200}, 4200}, + {"legacy observation", domain.DataTransferRun{TransferredBytes: 4096}, 4096}, + } { + t.Run(test.name, func(t *testing.T) { + if got := observedNetworkBytes(test.run); got != test.want { + t.Fatalf("network bytes=%d want=%d", got, test.want) + } + }) + } +} diff --git a/internal/controlplane/execution/uncertain_start_test.go b/internal/controlplane/execution/uncertain_start_test.go new file mode 100644 index 00000000..facad1ae --- /dev/null +++ b/internal/controlplane/execution/uncertain_start_test.go @@ -0,0 +1,33 @@ +package execution + +import ( + "context" + "fmt" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +type uncertainStartFailure struct{} + +func (uncertainStartFailure) Error() string { return "submission outcome uncertain" } +func (uncertainStartFailure) RuntimeReconciliationMetadata() map[string]any { + return map[string]any{"slurmSubmissionName": "attempt-bound-name"} +} + +func TestUncertainStartFailureKeepsHandleActiveForRecoveryGate(t *testing.T) { + store := &executionStoreFake{} + s := &Supervisor{executions: store} + err := s.recordStartFailure(context.Background(), "run", "activity", domain.PlanAssignment{}, + domain.Resource{ID: "hpc"}, "slurm", 2, fmt.Errorf("wrapped: %w", uncertainStartFailure{})) + if err != nil { + t.Fatal(err) + } + handle := store.handles["run:activity:attempt-2"] + if handle.Status != domain.HandleStarting || handle.FinishedAt != 0 { + t.Fatalf("unconfirmed runtime must remain active, not recoverable: %+v", handle) + } + if len(store.tasks) != 1 || store.tasks[0].Status != domain.TaskFailed { + t.Fatalf("failed attempt must still be recorded: %+v", store.tasks) + } +} diff --git a/internal/controlplane/execution/workspace_lifecycle.go b/internal/controlplane/execution/workspace_lifecycle.go index 70f5780c..df3d84f2 100644 --- a/internal/controlplane/execution/workspace_lifecycle.go +++ b/internal/controlplane/execution/workspace_lifecycle.go @@ -152,7 +152,7 @@ func workspacePath(rawURI string) string { return strings.TrimSpace(rawURI) } -func (s *Supervisor) activateWorkspace(ctx context.Context, runID, activityID string) error { +func (s *Supervisor) activateWorkspace(ctx context.Context, runID, activityID string, allocation domain.RuntimeAllocation) error { store := s.workspaceStore() if store == nil { return nil @@ -161,6 +161,16 @@ func (s *Supervisor) activateWorkspace(ctx context.Context, runID, activityID st if err != nil || workspace == nil { return err } + if err := bindWorkspaceAllocation(workspace, allocation); err != nil { + return err + } + // Persist the concrete cloud allocation before touching the remote + // workspace. Inspect, seal, release and recovery load this same record after + // the activity start call has returned, so the binding cannot live only in + // the in-memory execution request. + if err := store.SaveWorkspace(ctx, *workspace); err != nil { + return err + } if s.config.Workspaces != nil { if err := s.config.Workspaces.Ensure(ctx, *workspace); err != nil { workspace.State, workspace.LastError = domain.WorkspaceFailed, err.Error() @@ -172,6 +182,30 @@ func (s *Supervisor) activateWorkspace(ctx context.Context, runID, activityID st return store.SaveWorkspace(ctx, *workspace) } +func bindWorkspaceAllocation(workspace *domain.ActivityWorkspace, allocation domain.RuntimeAllocation) error { + if workspace == nil { + return nil + } + if allocation.RuntimeID != "" { + workspace.RuntimeID = allocation.RuntimeID + } + if allocation.ConnectionID != "" { + workspace.ConnectionID = allocation.ConnectionID + } + if allocation.CloudInstanceID == "" { + return nil + } + parsed, err := url.Parse(workspace.URI) + if err != nil { + return fmt.Errorf("parse workspace URI %q: %w", workspace.URI, err) + } + query := parsed.Query() + query.Set("cloudInstanceId", allocation.CloudInstanceID) + parsed.RawQuery = query.Encode() + workspace.URI = parsed.String() + return nil +} + func (s *Supervisor) releaseProducerWorkspaceLeases(ctx context.Context, runID, consumerID string) error { store := s.workspaceStore() if store == nil { diff --git a/internal/controlplane/execution/workspace_lifecycle_test.go b/internal/controlplane/execution/workspace_lifecycle_test.go index 2c3a2f46..719c45e8 100644 --- a/internal/controlplane/execution/workspace_lifecycle_test.go +++ b/internal/controlplane/execution/workspace_lifecycle_test.go @@ -1,11 +1,32 @@ package execution import ( + "net/url" "testing" "github.com/UFFeScience/akoflow/internal/domain" ) +func TestBindWorkspaceAllocationPersistsConcreteCloudInstance(t *testing.T) { + workspace := &domain.ActivityWorkspace{ + URI: "ssh://cloud-connection/akoflow/workspace/runs/run-1/activity-1", + RuntimeID: "planned-runtime", ConnectionID: "planned-connection", + } + allocation := domain.RuntimeAllocation{ + RuntimeID: "cloud-runtime", ConnectionID: "cloud-connection", CloudInstanceID: "instance-1", + } + if err := bindWorkspaceAllocation(workspace, allocation); err != nil { + t.Fatal(err) + } + parsed, err := url.Parse(workspace.URI) + if err != nil { + t.Fatal(err) + } + if workspace.RuntimeID != "cloud-runtime" || workspace.ConnectionID != "cloud-connection" || parsed.Query().Get("cloudInstanceId") != "instance-1" { + t.Fatalf("workspace was not bound to allocation: %+v", workspace) + } +} + func TestApplyArtifactManifestClassifiesWorkspaceSnapshots(t *testing.T) { workspace := &domain.ActivityWorkspace{} manifest := &domain.ArtifactManifest{ diff --git a/internal/controlplane/execution/workspace_snapshot_identity.go b/internal/controlplane/execution/workspace_snapshot_identity.go new file mode 100644 index 00000000..56e7665c --- /dev/null +++ b/internal/controlplane/execution/workspace_snapshot_identity.go @@ -0,0 +1,49 @@ +package execution + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/url" + "path" + "sort" + "strings" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +// A seal without a complete content manifest is not an immutable snapshot +// identity. Such legacy workspaces continue to transfer without cache reuse. +func sealedWorkspaceSnapshot(workspace *domain.ActivityWorkspace, instanceID string) (string, []domain.WorkspaceEntry) { + if workspace == nil || instanceID == "" || workspace.SealedAt == nil || + (workspace.State != domain.WorkspaceSealed && workspace.State != domain.WorkspaceReleasable) || + len(workspace.Manifest.Final) == 0 || int64(len(workspace.Manifest.Final)) != workspace.FileCount { + return "", nil + } + u, err := url.Parse(workspace.URI) + if err != nil || u.Query().Get("cloudInstanceId") != instanceID { + return "", nil + } + entries := append([]domain.WorkspaceEntry(nil), workspace.Manifest.Final...) + sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path }) + for i, entry := range entries { + if entry.Path == "" || entry.Path == "." || path.IsAbs(entry.Path) || path.Clean(entry.Path) != entry.Path || + strings.HasPrefix(entry.Path, "../") || strings.ContainsRune(entry.Path, 0) || entry.SizeBytes < 0 || + (i > 0 && entries[i-1].Path == entry.Path) || !strings.HasPrefix(entry.Digest, "sha256:") { + return "", nil + } + digest, err := hex.DecodeString(strings.TrimPrefix(entry.Digest, "sha256:")) + if err != nil || len(digest) != sha256.Size { + return "", nil + } + } + encoded, err := json.Marshal(struct { + WorkspaceID, URI, InstanceID, Seal string + Entries []domain.WorkspaceEntry + }{workspace.ID, workspace.URI, instanceID, workspace.SealedAt.UTC().Format("2006-01-02T15:04:05.999999999Z"), entries}) + if err != nil { + return "", nil + } + digest := sha256.Sum256(encoded) + return hex.EncodeToString(digest[:]), entries +} diff --git a/internal/controlplane/execution/workspace_snapshot_identity_test.go b/internal/controlplane/execution/workspace_snapshot_identity_test.go new file mode 100644 index 00000000..3d22a16f --- /dev/null +++ b/internal/controlplane/execution/workspace_snapshot_identity_test.go @@ -0,0 +1,47 @@ +package execution + +import ( + "strings" + "testing" + "time" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestSealedWorkspaceSnapshotIdentityRequiresCompleteExactGeneration(t *testing.T) { + now := time.Now().UTC() + w := domain.ActivityWorkspace{ID: "workspace", URI: "file:///workspace?cloudInstanceId=vm", State: domain.WorkspaceSealed, SealedAt: &now, FileCount: 2, + Manifest: domain.WorkspaceManifest{Final: []domain.WorkspaceEntry{ + {Path: "b", SizeBytes: 2, Digest: "sha256:" + strings.Repeat("b", 64)}, + {Path: "a", SizeBytes: 1, Digest: "sha256:" + strings.Repeat("a", 64)}, + }}} + key, entries := sealedWorkspaceSnapshot(&w, "vm") + if len(key) != 64 || len(entries) != 2 || entries[0].Path != "a" { + t.Fatal("missing canonical seal identity") + } + w.Manifest.Final[0], w.Manifest.Final[1] = w.Manifest.Final[1], w.Manifest.Final[0] + if reordered, _ := sealedWorkspaceSnapshot(&w, "vm"); reordered != key { + t.Fatal("manifest ordering changed identity") + } + w.Manifest.Final[0].Digest = "sha256:" + strings.Repeat("c", 64) + if changed, _ := sealedWorkspaceSnapshot(&w, "vm"); changed == key || changed == "" { + t.Fatal("changed content failed to invalidate identity") + } + if wrong, _ := sealedWorkspaceSnapshot(&w, "other-generation"); wrong != "" { + t.Fatal("different concrete instance accepted") + } + w.State = domain.WorkspaceActive + if active, _ := sealedWorkspaceSnapshot(&w, "vm"); active != "" { + t.Fatal("mutable workspace accepted") + } + w.State = domain.WorkspaceSealed + w.FileCount++ + if incomplete, _ := sealedWorkspaceSnapshot(&w, "vm"); incomplete != "" { + t.Fatal("incomplete manifest accepted") + } + w.FileCount-- + w.Manifest.Final[0].Path = "../escape" + if unsafe, _ := sealedWorkspaceSnapshot(&w, "vm"); unsafe != "" { + t.Fatal("unsafe manifest path accepted") + } +} diff --git a/internal/domain/cloud/cloud.go b/internal/domain/cloud/cloud.go index 0798a985..6367f5e9 100644 --- a/internal/domain/cloud/cloud.go +++ b/internal/domain/cloud/cloud.go @@ -1,9 +1,20 @@ package cloud import ( + "fmt" "time" ) +func (instance ProvisionedInstance) VerifiedSSHTrust() (string, string, error) { + alias, _ := instance.TerraformOutput["verifiedHostKeyAlias"].(string) + path, _ := instance.TerraformOutput["verifiedKnownHostsFile"].(string) + generation, _ := instance.TerraformOutput["verifiedHostGeneration"].(string) + if alias == "" || path == "" || generation == "" || generation != instance.ProviderID { + return "", "", fmt.Errorf("cloud instance has no verified SSH trust for its provider generation") + } + return alias, path, nil +} + const DefaultConfigurationID = "akoflow-scientific-worker" const DefaultConfigurationVersionID = "akoflow-scientific-worker-v7" @@ -164,6 +175,13 @@ type ProvisionedInstance struct { // InstanceBilling is a rate snapshot and an accumulated compute clock. Disk // continues to accrue while the VM is stopped; compute does not. type InstanceBilling struct { + PricingStatus string `json:"pricingStatus,omitempty"` + PriceCurrency string `json:"priceCurrency,omitempty"` + PriceSource string `json:"priceSource,omitempty"` + PriceObservedAt string `json:"priceObservedAt,omitempty"` + PriceRegion string `json:"priceRegion,omitempty"` + PriceMachineType string `json:"priceMachineType,omitempty"` + MissingPrices []string `json:"missingPrices,omitempty"` StartedAt time.Time `json:"startedAt"` FinishedAt *time.Time `json:"finishedAt,omitempty"` ComputeStartedAt *time.Time `json:"computeStartedAt,omitempty"` diff --git a/internal/domain/domain.go b/internal/domain/domain.go index e5e7050c..5fc75c8d 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -256,6 +256,7 @@ type TransferLocation = execution.TransferLocation type BlobDescriptor = execution.BlobDescriptor type TransferEndpoint = execution.TransferEndpoint type TransferRoute = execution.TransferRoute +type TransferPhase = execution.TransferPhase type TransferConnectorBinding = execution.ConnectorBinding type ArtifactVersion = execution.ArtifactVersion type ArtifactLocation = execution.ArtifactLocation diff --git a/internal/domain/execution/execution.go b/internal/domain/execution/execution.go index 429724ab..a50a3cf3 100644 --- a/internal/domain/execution/execution.go +++ b/internal/domain/execution/execution.go @@ -1,6 +1,7 @@ package execution import ( + "fmt" "time" "github.com/UFFeScience/akoflow/internal/domain/planning" @@ -209,6 +210,7 @@ type DataTransfer struct { } type ActivityExecutionContext struct { + Attempt int `json:"attempt,omitempty"` Run ExecutionRun `json:"run"` Workflow workflow.WorkflowVersion `json:"workflow"` Activity workflow.Activity `json:"activity"` @@ -222,6 +224,16 @@ type ActivityExecutionContext struct { Preparation *PreparationGate `json:"preparation,omitempty"` } +// HandleID is stable within an attempt, but never reuses a prior attempt's +// failed handle when the workflow is explicitly recovered. +func (e ActivityExecutionContext) HandleID() string { + id := e.Run.ID + ":" + e.Activity.ID + if e.Attempt > 1 { + id += fmt.Sprintf(":attempt-%d", e.Attempt) + } + return id +} + // ActivityHandle is the runtime-independent identity of a started activity. // ExternalID belongs to the adapter (PID, Kubernetes Job, Slurm Job or // simulation event); callers never need to understand its format. diff --git a/internal/domain/execution/materialization.go b/internal/domain/execution/materialization.go index 3d450cad..2b174da4 100644 --- a/internal/domain/execution/materialization.go +++ b/internal/domain/execution/materialization.go @@ -69,14 +69,18 @@ func (m ArtifactMaterialization) Committed() bool { } type TransferLocation struct { - ResourceID string `json:"resourceId,omitempty"` - EnvironmentID string `json:"environmentId,omitempty"` - RuntimeID string `json:"runtimeId,omitempty"` - ConnectionID string `json:"connectionId,omitempty"` - CloudInstanceID string `json:"cloudInstanceId,omitempty"` - NetworkDomain string `json:"networkDomain,omitempty"` - URI string `json:"uri"` - Path string `json:"path,omitempty"` + // Snapshot identity is issued only for a fully checksummed, sealed producer + // workspace. Missing identity disables archive reuse, not synchronization. + WorkspaceSnapshotID string `json:"workspaceSnapshotId,omitempty"` + WorkspaceSnapshotEntries []WorkspaceEntry `json:"workspaceSnapshotEntries,omitempty"` + ResourceID string `json:"resourceId,omitempty"` + EnvironmentID string `json:"environmentId,omitempty"` + RuntimeID string `json:"runtimeId,omitempty"` + ConnectionID string `json:"connectionId,omitempty"` + CloudInstanceID string `json:"cloudInstanceId,omitempty"` + NetworkDomain string `json:"networkDomain,omitempty"` + URI string `json:"uri"` + Path string `json:"path,omitempty"` } type ArtifactVersion struct { ID string `json:"id"` @@ -96,34 +100,86 @@ type ArtifactLocation struct { Available bool `json:"available"` } type TransferEndpoint struct { - ID string `json:"id"` - Kind string `json:"kind"` - URI string `json:"uri"` - ResourceID string `json:"resourceId,omitempty"` - EnvironmentID string `json:"environmentId,omitempty"` - RuntimeID string `json:"runtimeId,omitempty"` - ConnectionID string `json:"connectionId,omitempty"` - CloudInstanceID string `json:"cloudInstanceId,omitempty"` - NetworkDomain string `json:"networkDomain,omitempty"` - Configuration map[string]string `json:"configuration,omitempty"` + WorkspaceSnapshotID string `json:"workspaceSnapshotId,omitempty"` + WorkspaceSnapshotEntries []WorkspaceEntry `json:"workspaceSnapshotEntries,omitempty"` + ID string `json:"id"` + Kind string `json:"kind"` + URI string `json:"uri"` + ResourceID string `json:"resourceId,omitempty"` + EnvironmentID string `json:"environmentId,omitempty"` + RuntimeID string `json:"runtimeId,omitempty"` + ConnectionID string `json:"connectionId,omitempty"` + CloudInstanceID string `json:"cloudInstanceId,omitempty"` + NetworkDomain string `json:"networkDomain,omitempty"` + Configuration map[string]string `json:"configuration,omitempty"` } type TransferRoute struct { - Strategy TransferStrategy `json:"strategy"` - ProducerActivityID string `json:"producerActivityId,omitempty"` - ConsumerActivityID string `json:"consumerActivityId,omitempty"` - SourceResourceID string `json:"sourceResourceId,omitempty"` - TargetResourceID string `json:"targetResourceId,omitempty"` - SourceEnvironmentID string `json:"sourceEnvironmentId,omitempty"` - TargetEnvironmentID string `json:"targetEnvironmentId,omitempty"` - SourceAddress string `json:"sourceAddress,omitempty"` - TargetAddress string `json:"targetAddress,omitempty"` - SourceCloudInstanceID string `json:"sourceCloudInstanceId,omitempty"` - TargetCloudInstanceID string `json:"targetCloudInstanceId,omitempty"` - FilesTransferred int64 `json:"filesTransferred,omitempty"` - NetworkDomain string `json:"networkDomain,omitempty"` - Fallback TransferStrategy `json:"fallback,omitempty"` - Reason string `json:"reason"` + Phases []TransferPhase `json:"phases,omitempty"` + ArchiveBytes int64 `json:"archiveBytes,omitempty"` + Packaging string `json:"packaging,omitempty"` + PreparationStartedAt float64 `json:"preparationStartedAt,omitempty"` + PreparationFinishedAt float64 `json:"preparationFinishedAt,omitempty"` + PreparationSeconds float64 `json:"preparationSeconds,omitempty"` + Strategy TransferStrategy `json:"strategy"` + ProducerActivityID string `json:"producerActivityId,omitempty"` + ConsumerActivityID string `json:"consumerActivityId,omitempty"` + SourceResourceID string `json:"sourceResourceId,omitempty"` + TargetResourceID string `json:"targetResourceId,omitempty"` + SourceEnvironmentID string `json:"sourceEnvironmentId,omitempty"` + TargetEnvironmentID string `json:"targetEnvironmentId,omitempty"` + SourceAddress string `json:"sourceAddress,omitempty"` + TargetAddress string `json:"targetAddress,omitempty"` + SourceCloudInstanceID string `json:"sourceCloudInstanceId,omitempty"` + TargetCloudInstanceID string `json:"targetCloudInstanceId,omitempty"` + FilesTransferred int64 `json:"filesTransferred,omitempty"` + NetworkDomain string `json:"networkDomain,omitempty"` + SharedFilesystem string `json:"sharedFilesystem,omitempty"` + SharedFilesystemID string `json:"sharedFilesystemId,omitempty"` + SharedStorageProofFailure string `json:"sharedStorageProofFailure,omitempty"` + RetryCount int `json:"retryCount,omitempty"` + SSHChannelWaitSeconds float64 `json:"sshChannelWaitSeconds,omitempty"` + SourcePackPreflightSeconds float64 `json:"sourcePackPreflightSeconds,omitempty"` + SourcePackPreflightSSHChannelWaitSeconds float64 `json:"sourcePackPreflightSshChannelWaitSeconds,omitempty"` + SourcePackPreflightServiceSeconds float64 `json:"sourcePackPreflightServiceSeconds,omitempty"` + SourcePackSSHChannelWaitSeconds float64 `json:"sourcePackSshChannelWaitSeconds,omitempty"` + SourcePackExecutionSeconds float64 `json:"sourcePackExecutionSeconds,omitempty"` + RsyncExecutionSeconds float64 `json:"rsyncExecutionSeconds,omitempty"` + MergeCheckSeconds float64 `json:"mergeCheckSeconds,omitempty"` + MergeCopySeconds float64 `json:"mergeCopySeconds,omitempty"` + MergeVerifySeconds float64 `json:"mergeVerifySeconds,omitempty"` + MergeSSHChannelWaitSeconds float64 `json:"mergeSshChannelWaitSeconds,omitempty"` + MergeExecutionSeconds float64 `json:"mergeExecutionSeconds,omitempty"` + MergeCommandOverheadSeconds float64 `json:"mergeCommandOverheadSeconds,omitempty"` + LastRetryError string `json:"lastRetryError,omitempty"` + NetworkBytesIncomplete bool `json:"networkBytesIncomplete,omitempty"` + ServerNetworkBytes *int64 `json:"serverNetworkBytes,omitempty"` + Fallback TransferStrategy `json:"fallback,omitempty"` + Reason string `json:"reason"` + PeerProbeFailure string `json:"peerProbeFailure,omitempty"` + SnapshotReused bool `json:"snapshotReused,omitempty"` + // Diagnostics, not additive phases. Build wall time includes SSH admission; + // a reused caller's acquisition wait may overlap that same shared build. + SnapshotBuildWallSeconds float64 `json:"snapshotBuildWallSeconds,omitempty"` + SnapshotBuildSSHWaitSeconds *float64 `json:"snapshotBuildSshWaitSeconds,omitempty"` + SnapshotBuildCommandSeconds *float64 `json:"snapshotBuildCommandSeconds,omitempty"` + SnapshotBuildServiceSeconds *float64 `json:"snapshotBuildServiceSeconds,omitempty"` + SnapshotBuildCPUSeconds *float64 `json:"snapshotBuildCpuSeconds,omitempty"` + SnapshotAcquireWaitSeconds float64 `json:"snapshotAcquireWaitSeconds,omitempty"` + PeerReceiverServiceSeconds *float64 `json:"peerReceiverServiceSeconds,omitempty"` + PeerReceiverCPUSeconds *float64 `json:"peerReceiverCpuSeconds,omitempty"` +} + +// TransferPhase is elapsed wall time, not exclusive work: concurrent origins +// and the shared publication barrier must not be summed as consumer latency. +type TransferPhase struct { + Name string `json:"name"` + StartedAt float64 `json:"startedAt"` + FinishedAt float64 `json:"finishedAt"` + DurationSeconds float64 `json:"durationSeconds"` + SSHChannelWaitSeconds *float64 `json:"sshChannelWaitSeconds,omitempty"` + ExecutionSeconds *float64 `json:"executionSeconds,omitempty"` + Failed bool `json:"failed,omitempty"` } type ConnectorBinding struct { ID string `json:"id"` diff --git a/internal/domain/materializationmodel/model.go b/internal/domain/materializationmodel/model.go new file mode 100644 index 00000000..75e73059 --- /dev/null +++ b/internal/domain/materializationmodel/model.go @@ -0,0 +1,457 @@ +// Package materializationmodel describes a frozen causal preparation estimate. +// It does not select live SSH routes or grant shared-storage trust. Those remain +// executor decisions; the model must carry evidence for the matching strategy. +package materializationmodel + +import ( + "encoding/json" + "fmt" + "math" +) + +type Model struct { + SchemaVersion int `json:"schemaVersion"` + Source string `json:"source"` + RelayResourceID string `json:"relayResourceId"` + Resources map[string]Resource `json:"resources"` + Profiles []Profile `json:"profiles"` + MergeProfiles []MergeProfile `json:"mergeProfiles"` + InputObservationProfiles []InputObservationProfile `json:"inputObservationProfiles,omitempty"` + PublicationProfiles []PublicationProfile `json:"publicationProfiles,omitempty"` + AdmissionProfiles []AdmissionProfile `json:"admissionProfiles,omitempty"` +} + +// Admission is external submit-to-job-start wall, not CPU service or merge. +type AdmissionProfile struct { + TargetClass string `json:"targetClass"` + RuntimeID string `json:"runtimeId"` + Source string `json:"source"` + Samples int `json:"samples"` + FixedSeconds float64 `json:"fixedSeconds"` +} + +// Publication follows scientific compute and gates both dependencies and lane +// release. Scan and inherited-input cleanup are service, not compute or network. +type PublicationProfile struct { + FixedSeconds float64 `json:"fixedSeconds,omitempty"` + TargetClass string `json:"targetClass"` + Source string `json:"source"` + Samples int `json:"samples"` + ScanSecondsPerWorkspaceByte float64 `json:"scanSecondsPerWorkspaceByte"` + CleanupSecondsPerInputByte float64 `json:"cleanupSecondsPerInputByte"` +} + +// Input observation occurs after merge and before the command. Its measured +// service must exclude admission wait and must not already be in merge/compute. +type InputObservationProfile struct { + FixedSeconds float64 `json:"fixedSeconds,omitempty"` + TargetClass string `json:"targetClass"` + Source string `json:"source"` + Samples int `json:"samples"` + SecondsPerUniqueByte float64 `json:"secondsPerUniqueByte"` +} + +type Resource struct { + ExecutionRuntimeID string `json:"executionRuntimeId,omitempty"` + Class string `json:"class"` + Domain string `json:"domain"` // cloud, hpc or relay + // An equal nonempty identity is usable only with explicit proof on both. + SharedStorageID string `json:"sharedStorageId,omitempty"` + SharedStorageProven bool `json:"sharedStorageProven,omitempty"` + BulkServiceGroupID string `json:"bulkServiceGroupId,omitempty"` + BulkServiceSlots int `json:"bulkServiceSlots,omitempty"` + BulkServiceSource string `json:"bulkServiceSource,omitempty"` +} + +type Profile struct { + Strategy string `json:"strategy"` + SourceClass string `json:"sourceClass"` + TargetClass string `json:"targetClass"` + Source string `json:"source"` + Samples int `json:"samples"` + ArchiveToWorkspaceRatio float64 `json:"archiveToWorkspaceRatio"` + ProbeSeconds float64 `json:"probeSeconds"` + SnapshotSecondsPerByte float64 `json:"snapshotSecondsPerByte"` + IngressArchiveBytesPerSecond float64 `json:"ingressArchiveBytesPerSecond"` + ValidationSecondsPerByte float64 `json:"validationSecondsPerByte"` + UploadArchiveBytesPerSecond float64 `json:"uploadArchiveBytesPerSecond"` + ReceiverSecondsPerByte float64 `json:"receiverSecondsPerByte"` + // Pack-and-receive is measured as one combined phase in real telemetry. + // Do not add source CPU preparation a second time to this effective rate. + IngressIncludesSourcePreparation bool `json:"ingressIncludesSourcePreparation,omitempty"` + // Independent service measurements must exclude admission queue. Without + // this evidence, ingress remains a combined pack-and-receive measurement. + SeparatedIngressServiceSource string `json:"separatedIngressServiceSource,omitempty"` +} + +type MergeProfile struct { + TargetClass string `json:"targetClass"` + Source string `json:"source"` + Samples int `json:"samples"` + CommandOverheadSeconds float64 `json:"commandOverheadSeconds,omitempty"` + CheckSecondsPerInputByte float64 `json:"checkSecondsPerInputByte"` + CopySecondsPerUniqueByte float64 `json:"copySecondsPerUniqueByte"` + VerifySecondsPerUniqueByte float64 `json:"verifySecondsPerUniqueByte"` +} + +type Stage struct { + Name string `json:"name"` + SourceResourceID string `json:"sourceResourceId"` + TargetResourceID string `json:"targetResourceId"` + Bytes int64 `json:"bytes"` + EffectiveBytesPerSecond float64 `json:"effectiveBytesPerSecond,omitempty"` + BeforeSeconds float64 `json:"beforeSeconds,omitempty"` + AfterSeconds float64 `json:"afterSeconds,omitempty"` + ServiceGroupID string `json:"serviceGroupId,omitempty"` + ServiceSlots int `json:"serviceSlots,omitempty"` +} + +type Route struct { + Strategy string `json:"strategy"` + LogicalBytes int64 `json:"logicalBytes"` + NetworkBytes int64 `json:"networkBytes"` + RelayNetworkBytes int64 `json:"relayNetworkBytes"` + Stages []Stage `json:"stages"` +} + +func Decode(metadata map[string]any) (*Model, error) { + raw := metadata["workspaceMaterializationModel"] + if raw == nil { + return nil, nil + } + payload, err := json.Marshal(raw) + if err != nil { + return nil, err + } + var model Model + if err = json.Unmarshal(payload, &model); err != nil { + return nil, fmt.Errorf("decode workspace materialization model: %w", err) + } + if err = model.Validate(); err != nil { + return nil, err + } + return &model, nil +} + +func validNonnegative(values ...float64) bool { + for _, value := range values { + if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 { + return false + } + } + return true +} + +func (m Model) Validate() error { + if (m.SchemaVersion != 1 && m.SchemaVersion != 2) || m.Source == "" { + return fmt.Errorf("materialization model requires supported version and provenance") + } + if m.SchemaVersion == 1 { + if len(m.AdmissionProfiles) > 0 { + return fmt.Errorf("runtime admission requires materialization schema version 2") + } + for _, p := range m.InputObservationProfiles { + if p.FixedSeconds != 0 { + return fmt.Errorf("fixed stage service requires materialization schema version 2") + } + } + for _, p := range m.PublicationProfiles { + if p.FixedSeconds != 0 { + return fmt.Errorf("fixed stage service requires materialization schema version 2") + } + } + } + if relay, exists := m.Resources[m.RelayResourceID]; !exists || relay.Domain != "relay" { + return fmt.Errorf("materialization model requires an explicit relay resource") + } + serviceLimits := map[string]int{} + for id, resource := range m.Resources { + if id == "" || resource.Class == "" || (resource.Domain != "cloud" && resource.Domain != "hpc" && resource.Domain != "relay") { + return fmt.Errorf("invalid materialization resource %q", id) + } + if resource.SharedStorageProven && resource.SharedStorageID == "" { + return fmt.Errorf("shared storage proof requires identity for %q", id) + } + if resource.BulkServiceGroupID != "" { + if resource.Domain == "relay" || resource.BulkServiceSlots < 1 || resource.BulkServiceSource == "" { + return fmt.Errorf("bulk service %q requires positive slots and evidence", id) + } + if n, ok := serviceLimits[resource.BulkServiceGroupID]; ok && n != resource.BulkServiceSlots { + return fmt.Errorf("inconsistent bulk service slots") + } + serviceLimits[resource.BulkServiceGroupID] = resource.BulkServiceSlots + } else if resource.BulkServiceSlots != 0 || resource.BulkServiceSource != "" { + return fmt.Errorf("bulk service resource %q lacks group identity", id) + } + } + if err := m.validateTransferProfiles(); err != nil { + return err + } + seen := map[string]bool{} + for _, profile := range m.MergeProfiles { + if seen[profile.TargetClass] || profile.TargetClass == "" || profile.Source == "" || profile.Samples < 1 || !validNonnegative( + profile.CommandOverheadSeconds, + profile.CheckSecondsPerInputByte, + profile.CopySecondsPerUniqueByte, + profile.VerifySecondsPerUniqueByte, + ) { + return fmt.Errorf("invalid or duplicate merge profile %q", profile.TargetClass) + } + seen[profile.TargetClass] = true + } + seen = map[string]bool{} + for _, p := range m.InputObservationProfiles { + if p.TargetClass == "" || p.Source == "" || p.Samples < 1 || seen[p.TargetClass] || !validNonnegative(p.FixedSeconds, p.SecondsPerUniqueByte) { + return fmt.Errorf("invalid input observation profile %q", p.TargetClass) + } + seen[p.TargetClass] = true + } + seen = map[string]bool{} + for _, p := range m.PublicationProfiles { + if p.TargetClass == "" || p.Source == "" || p.Samples < 1 || seen[p.TargetClass] || !validNonnegative(p.FixedSeconds, p.ScanSecondsPerWorkspaceByte, p.CleanupSecondsPerInputByte) { + return fmt.Errorf("invalid publication profile %q", p.TargetClass) + } + seen[p.TargetClass] = true + } + return m.validateAdmissionProfiles() +} + +func (m Model) validateTransferProfiles() error { + seen := map[string]bool{} + for _, profile := range m.Profiles { + key := profile.Strategy + "\x00" + profile.SourceClass + "\x00" + profile.TargetClass + if seen[key] { + return fmt.Errorf("duplicate materialization profile %q", key) + } + seen[key] = true + if profile.Source == "" || profile.Samples < 1 || profile.SourceClass == "" || profile.TargetClass == "" || !validNonnegative( + profile.ArchiveToWorkspaceRatio, + profile.ProbeSeconds, + profile.SnapshotSecondsPerByte, + profile.IngressArchiveBytesPerSecond, + profile.ValidationSecondsPerByte, + profile.UploadArchiveBytesPerSecond, + profile.ReceiverSecondsPerByte, + ) { + return fmt.Errorf("invalid materialization profile %q", key) + } + if profile.Strategy != "gateway" && profile.Strategy != "direct-runtime" && profile.Strategy != "shared-storage" { + return fmt.Errorf("unknown materialization strategy %q", profile.Strategy) + } + if profile.Strategy != "shared-storage" && (profile.ArchiveToWorkspaceRatio <= 0 || profile.UploadArchiveBytesPerSecond <= 0) { + return fmt.Errorf("materialization profile lacks archive ratio or upload rate") + } + if profile.Strategy == "gateway" { + if profile.IngressArchiveBytesPerSecond <= 0 { + return fmt.Errorf("gateway profile requires measured ingress rate") + } + if profile.IngressIncludesSourcePreparation { + if profile.SnapshotSecondsPerByte != 0 || profile.SeparatedIngressServiceSource != "" { + return fmt.Errorf("combined ingress cannot also charge separate source preparation") + } + } else if profile.SeparatedIngressServiceSource == "" { + return fmt.Errorf("gateway separated ingress requires queue-free service provenance") + } + } + } + return nil +} + +func (m Model) validateAdmissionProfiles() error { + seen := map[string]bool{} + for _, p := range m.AdmissionProfiles { + key := p.TargetClass + "\x00" + p.RuntimeID + if p.Source == "" || p.Samples < 1 || p.TargetClass == "" || p.RuntimeID == "" || seen[key] || !validNonnegative(p.FixedSeconds) { + return fmt.Errorf("invalid runtime admission profile") + } + seen[key] = true + } + return nil +} + +func (m Model) AdmissionSeconds(targetID string) (float64, error) { + if err := m.validateAdmissionProfiles(); err != nil { + return 0, err + } + r, ok := m.Resources[targetID] + if !ok { + return 0, fmt.Errorf("admission resource missing") + } + for _, p := range m.AdmissionProfiles { + if p.TargetClass == r.Class && p.RuntimeID == r.ExecutionRuntimeID { + return p.FixedSeconds, nil + } + } + return 0, nil +} + +func (m Model) PublicationSeconds(targetID string, workspaceBytes, inputBytes int64) (float64, error) { + if workspaceBytes < 0 || inputBytes < 0 { + return 0, fmt.Errorf("negative publication volume") + } + for _, p := range m.PublicationProfiles { + if p.TargetClass == m.Resources[targetID].Class { + seconds := p.FixedSeconds + float64(workspaceBytes)*p.ScanSecondsPerWorkspaceByte + float64(inputBytes)*p.CleanupSecondsPerInputByte + if !validNonnegative(seconds) { + return 0, fmt.Errorf("publication estimate overflow") + } + return seconds, nil + } + } + return 0, nil +} + +// PreparationSeconds charges the consumer's unique input scan once, never +// once per predecessor. Absence of an evidenced profile preserves legacy data. +func (m Model) PreparationSeconds(targetID string, totalBytes, uniqueBytes int64) (float64, error) { + merge, err := m.MergeSeconds(targetID, totalBytes, uniqueBytes) + if err != nil { + return 0, err + } + for _, p := range m.InputObservationProfiles { + if p.TargetClass == m.Resources[targetID].Class { + seconds := merge + p.FixedSeconds + float64(uniqueBytes)*p.SecondsPerUniqueByte + if !validNonnegative(seconds) { + return 0, fmt.Errorf("input observation estimate overflow") + } + return seconds, nil + } + } + return merge, nil +} + +func (m Model) Route(sourceID, targetID string, workspaceBytes int64) (Route, error) { + if err := m.Validate(); err != nil { + return Route{}, err + } + if workspaceBytes < 0 { + return Route{}, fmt.Errorf("negative workspace bytes") + } + source, sourceOK := m.Resources[sourceID] + target, targetOK := m.Resources[targetID] + if !sourceOK || !targetOK || source.Domain == "relay" || target.Domain == "relay" { + return Route{}, fmt.Errorf("materialization route resource missing or invalid") + } + result := Route{LogicalBytes: workspaceBytes} + if sourceID == targetID { + result.Strategy = "runtime-local" + return result, nil + } + strategy := materializationRouteStrategy(source, target) + p, err := m.findRouteProfile(strategy, source.Class, target.Class) + if err != nil { + return result, err + } + result.Strategy = strategy + bytes := float64(workspaceBytes) + if strategy == "shared-storage" { + result.Stages = []Stage{{Name: "shared-storage-copy-and-verify", SourceResourceID: sourceID, TargetResourceID: targetID, BeforeSeconds: p.ProbeSeconds + bytes*p.SnapshotSecondsPerByte, AfterSeconds: bytes * p.ReceiverSecondsPerByte}} + return m.withBulkServices(result), nil + } + compressed := math.Ceil(bytes * p.ArchiveToWorkspaceRatio) + // Reserve room for both relay hops without overflowing integer counters. + if compressed >= float64(math.MaxInt64/2) { + return result, fmt.Errorf("archive byte estimate overflow") + } + archiveBytes := int64(compressed) + if strategy == "direct-runtime" { + result.NetworkBytes = archiveBytes + result.Stages = []Stage{ + { + Name: "peer-snapshot-upload-and-extract", + SourceResourceID: sourceID, + TargetResourceID: targetID, + Bytes: archiveBytes, + EffectiveBytesPerSecond: p.UploadArchiveBytesPerSecond, + BeforeSeconds: p.ProbeSeconds + bytes*p.SnapshotSecondsPerByte, + AfterSeconds: bytes * p.ReceiverSecondsPerByte, + }, + } + if source.BulkServiceGroupID != "" || target.BulkServiceGroupID != "" { + // The source SSH command uploads through the private peer route; + // receiver verification is a separate command on the target session. + result.Stages[0].Name = "peer-snapshot-and-upload" + result.Stages[0].AfterSeconds = 0 + result.Stages = append(result.Stages, Stage{Name: "peer-receiver-verify-and-extract", SourceResourceID: targetID, TargetResourceID: targetID, BeforeSeconds: bytes * p.ReceiverSecondsPerByte}) + } + } else { + result.NetworkBytes = 2 * archiveBytes + result.RelayNetworkBytes = result.NetworkBytes + result.Stages = []Stage{ + { + Name: "relay-pack-and-receive", + SourceResourceID: sourceID, + TargetResourceID: m.RelayResourceID, + Bytes: archiveBytes, + EffectiveBytesPerSecond: p.IngressArchiveBytesPerSecond, + BeforeSeconds: p.ProbeSeconds + bytes*p.SnapshotSecondsPerByte, + AfterSeconds: bytes * p.ValidationSecondsPerByte, + }, + {Name: "relay-upload-and-extract", SourceResourceID: m.RelayResourceID, TargetResourceID: targetID, Bytes: archiveBytes, EffectiveBytesPerSecond: p.UploadArchiveBytesPerSecond, AfterSeconds: bytes * p.ReceiverSecondsPerByte}, + } + if !p.IngressIncludesSourcePreparation { + result.Stages[0].Name = "relay-source-preparation-and-ingress" + } + } + return m.withBulkServices(result), nil +} + +func materializationRouteStrategy(source, target Resource) string { + if source.Domain == "cloud" && target.Domain == "cloud" { + return "direct-runtime" + } + if source.Domain == "hpc" && target.Domain == "hpc" && source.SharedStorageProven && target.SharedStorageProven && source.SharedStorageID == target.SharedStorageID { + return "shared-storage" + } + return "gateway" +} + +func (m Model) findRouteProfile(strategy, sourceClass, targetClass string) (Profile, error) { + for _, profile := range m.Profiles { + if profile.Strategy == strategy && profile.SourceClass == sourceClass && profile.TargetClass == targetClass { + return profile, nil + } + } + return Profile{}, fmt.Errorf("no evidenced %s profile for %s -> %s", strategy, sourceClass, targetClass) +} + +func (m Model) withBulkServices(route Route) Route { + for i := range route.Stages { + stage := &route.Stages[i] + owner := stage.SourceResourceID + if route.Strategy == "gateway" && i == 1 { + owner = stage.TargetResourceID + } + r := m.Resources[owner] + stage.ServiceGroupID, stage.ServiceSlots = r.BulkServiceGroupID, r.BulkServiceSlots + } + return route +} + +// MergeSeconds is charged once per consumer after every dependency completes. +// Checking scans all parents; copy and final verification use the unique union. +func (m Model) MergeSeconds(targetID string, inputTotalBytes, inputUniqueBytes int64) (float64, error) { + if err := m.Validate(); err != nil { + return 0, err + } + if inputTotalBytes < 0 || inputUniqueBytes < 0 || inputUniqueBytes > inputTotalBytes { + return 0, fmt.Errorf("invalid merge volume") + } + resource, exists := m.Resources[targetID] + if !exists { + return 0, fmt.Errorf("merge resource missing") + } + if inputTotalBytes == 0 { + return 0, nil + } + for _, profile := range m.MergeProfiles { + if profile.TargetClass == resource.Class { + seconds := profile.CommandOverheadSeconds + float64(inputTotalBytes)*profile.CheckSecondsPerInputByte + float64(inputUniqueBytes)*(profile.CopySecondsPerUniqueByte+profile.VerifySecondsPerUniqueByte) + if !validNonnegative(seconds) { + return 0, fmt.Errorf("merge estimate overflow") + } + return seconds, nil + } + } + return 0, fmt.Errorf("no evidenced merge profile for %s", resource.Class) +} diff --git a/internal/domain/materializationmodel/model_test.go b/internal/domain/materializationmodel/model_test.go new file mode 100644 index 00000000..05e57122 --- /dev/null +++ b/internal/domain/materializationmodel/model_test.go @@ -0,0 +1,299 @@ +package materializationmodel + +import ( + "encoding/json" + "github.com/stretchr/testify/require" + "math" + "os" + "testing" +) + +func TestPublicationServiceUsesWorkspaceAndInheritedInputs(t *testing.T) { + m := testModel() + m.PublicationProfiles = []PublicationProfile{{TargetClass: "micro", Source: "measured-fixture", Samples: 1, ScanSecondsPerWorkspaceByte: .02, CleanupSecondsPerInputByte: .01}} + seconds, err := m.PublicationSeconds("cloud-a", 100, 50) + require.NoError(t, err) + require.InDelta(t, 2.5, seconds, 1e-9) + _, err = m.PublicationSeconds("cloud", -1, 0) + require.Error(t, err) + m.PublicationProfiles[0].Source = "" + require.Error(t, m.Validate()) +} + +func TestAdmissionIsRuntimeScopedAndNotPreparationOrCompute(t *testing.T) { + m := testModel() + r := m.Resources["cloud-a"] + r.ExecutionRuntimeID = "cloud-runtime" + m.Resources["cloud-a"] = r + m.AdmissionProfiles = []AdmissionProfile{{TargetClass: "micro", RuntimeID: "cloud-runtime", Source: "native-submission", Samples: 3, FixedSeconds: 7}} + m.SchemaVersion = 2 + a, err := m.AdmissionSeconds("cloud-a") + require.NoError(t, err) + require.Equal(t, 7.0, a) + a, err = m.AdmissionSeconds("cloud-b") + require.NoError(t, err) + require.Zero(t, a) + p, err := m.PreparationSeconds("cloud-a", 0, 0) + require.NoError(t, err) + require.Zero(t, p) + m.AdmissionProfiles[0].FixedSeconds = -1 + require.Error(t, m.Validate()) +} + +func TestFixedStageServiceIsCommonAndClassScoped(t *testing.T) { + m := testModel() + m.InputObservationProfiles = []InputObservationProfile{{TargetClass: "micro", Source: "fixture", Samples: 1, FixedSeconds: 3, SecondsPerUniqueByte: .01}} + m.SchemaVersion = 2 + m.PublicationProfiles = []PublicationProfile{{TargetClass: "micro", Source: "fixture", Samples: 1, FixedSeconds: 5, ScanSecondsPerWorkspaceByte: .02}} + p, err := m.PreparationSeconds("cloud-a", 0, 0) + require.NoError(t, err) + require.Equal(t, 3.0, p) + p, err = m.PublicationSeconds("cloud-a", 100, 0) + require.NoError(t, err) + require.Equal(t, 7.0, p) + m.InputObservationProfiles[0].FixedSeconds = -1 + require.Error(t, m.Validate()) +} + +func TestFrozenMontageMeasuredModelDecodesAndRejectsUnobservedPair(t *testing.T) { + payload, err := os.ReadFile("../../../experiments/montage-micro-hpc-e2/materialization-model-v2.json") + require.NoError(t, err) + var model Model + require.NoError(t, json.Unmarshal(payload, &model)) + require.NoError(t, model.Validate()) + route, err := model.Route("plafrim-execution-ssh-node-diablo03", "montage-gcp-fog-e2-micro-01", 200_000_000) + require.NoError(t, err) + require.Len(t, route.Stages, 2) + require.Equal(t, "montage-relay-controller", route.Stages[0].TargetResourceID) + require.Equal(t, route.NetworkBytes, route.RelayNetworkBytes) + _, err = model.Route("plafrim-execution-ssh-node-diablo03", "montage-gcp-e2-medium-1", 200_000_000) + require.ErrorContains(t, err, "no evidenced") +} + +func testModel() Model { + return Model{SchemaVersion: 1, Source: "completed-baseline", RelayResourceID: "relay", Resources: map[string]Resource{ + "cloud-a": {Class: "micro", Domain: "cloud"}, "cloud-b": {Class: "micro", Domain: "cloud"}, + "hpc-a": {Class: "hpc", Domain: "hpc"}, "hpc-b": {Class: "hpc", Domain: "hpc"}, "relay": {Class: "relay", Domain: "relay"}, + }, Profiles: []Profile{ + { + Strategy: "direct-runtime", + SourceClass: "micro", + TargetClass: "micro", + Source: "baseline", + Samples: 3, + ArchiveToWorkspaceRatio: .5, + ProbeSeconds: 1, + SnapshotSecondsPerByte: .01, + UploadArchiveBytesPerSecond: 25, + ReceiverSecondsPerByte: .02, + }, + { + Strategy: "gateway", + SourceClass: "micro", + TargetClass: "hpc", + Source: "baseline", + Samples: 3, + ArchiveToWorkspaceRatio: .5, + ProbeSeconds: 1, + IngressArchiveBytesPerSecond: 50, + ValidationSecondsPerByte: .01, + UploadArchiveBytesPerSecond: 25, + ReceiverSecondsPerByte: .02, + IngressIncludesSourcePreparation: true, + }, + { + Strategy: "gateway", + SourceClass: "hpc", + TargetClass: "micro", + Source: "baseline", + Samples: 3, + ArchiveToWorkspaceRatio: .5, + ProbeSeconds: 1, + IngressArchiveBytesPerSecond: 50, + ValidationSecondsPerByte: .01, + UploadArchiveBytesPerSecond: 25, + ReceiverSecondsPerByte: .02, + IngressIncludesSourcePreparation: true, + }, + {Strategy: "shared-storage", SourceClass: "hpc", TargetClass: "hpc", Source: "storage-probe", Samples: 3, SnapshotSecondsPerByte: .01, ReceiverSecondsPerByte: .02}, + }, MergeProfiles: []MergeProfile{{TargetClass: "micro", Source: "baseline", Samples: 3, CheckSecondsPerInputByte: .01, CopySecondsPerUniqueByte: .02, VerifySecondsPerUniqueByte: .03}}} +} + +func TestInputObservationIsChargedOnceOnUniqueInputs(t *testing.T) { + m := testModel() + m.InputObservationProfiles = []InputObservationProfile{{TargetClass: "micro", Source: "measured-snapshot-service", Samples: 3, SecondsPerUniqueByte: .5}} + merge, err := m.MergeSeconds("cloud-a", 200, 100) + require.NoError(t, err) + preparation, err := m.PreparationSeconds("cloud-a", 200, 100) + require.NoError(t, err) + require.Equal(t, merge+50, preparation) + root, err := m.PreparationSeconds("cloud-a", 0, 0) + require.NoError(t, err) + require.Zero(t, root) + m.InputObservationProfiles[0].Source = "" + require.Error(t, m.Validate()) +} + +func TestSeparatedIngressUsesIndependentPreparationWithoutExtraNetworkBytes(t *testing.T) { + m := testModel() + p := &m.Profiles[2] + p.IngressIncludesSourcePreparation = false + p.SeparatedIngressServiceSource = "queue-free-independent-snapshot-and-ingress" + p.SnapshotSecondsPerByte = .03 + route, err := m.Route("hpc-a", "cloud-a", 100) + require.NoError(t, err) + require.Len(t, route.Stages, 2) + require.Equal(t, "relay-source-preparation-and-ingress", route.Stages[0].Name) + require.Equal(t, 4.0, route.Stages[0].BeforeSeconds) + require.Equal(t, 50.0, route.Stages[0].EffectiveBytesPerSecond) + require.EqualValues(t, 100, route.NetworkBytes) + p.IngressIncludesSourcePreparation = true + require.ErrorContains(t, m.Validate(), "cannot also charge") +} + +func TestBulkServiceGroupsFollowControlTransportNotRelayHost(t *testing.T) { + m := testModel() + hpc := m.Resources["hpc-a"] + hpc.BulkServiceGroupID, hpc.BulkServiceSlots, hpc.BulkServiceSource = "hpc-connection", 3, "provider-config" + m.Resources["hpc-a"] = hpc + cloud := m.Resources["cloud-a"] + cloud.BulkServiceGroupID, cloud.BulkServiceSlots, cloud.BulkServiceSource = "cloud-instance", 3, "provider-config" + m.Resources["cloud-a"] = cloud + require.NoError(t, m.Validate()) + route, err := m.Route("hpc-a", "cloud-a", 100) + require.NoError(t, err) + require.Equal(t, "hpc-connection", route.Stages[0].ServiceGroupID) + require.Equal(t, "cloud-instance", route.Stages[1].ServiceGroupID) + require.Equal(t, 3, route.Stages[0].ServiceSlots) + require.Equal(t, 3, route.Stages[1].ServiceSlots) + other := m.Resources["hpc-b"] + other.BulkServiceGroupID, other.BulkServiceSlots, other.BulkServiceSource = "hpc-connection", 2, "provider-config" + m.Resources["hpc-b"] = other + require.Error(t, m.Validate(), "one transport cannot have conflicting slot capacities") +} + +func TestPeerReceiverUsesDestinationServicePoolWithoutExtraNetworkCopy(t *testing.T) { + m := testModel() + for _, id := range []string{"cloud-a", "cloud-b"} { + r := m.Resources[id] + r.BulkServiceGroupID, r.BulkServiceSlots, r.BulkServiceSource = id+"-session", 3, "provider-config" + m.Resources[id] = r + } + route, err := m.Route("cloud-a", "cloud-b", 100) + require.NoError(t, err) + require.Len(t, route.Stages, 2) + require.Equal(t, "cloud-a-session", route.Stages[0].ServiceGroupID) + require.Equal(t, "cloud-b-session", route.Stages[1].ServiceGroupID) + require.EqualValues(t, 50, route.Stages[0].Bytes) + require.Zero(t, route.Stages[1].Bytes) + require.Zero(t, route.Stages[0].AfterSeconds) + require.Equal(t, 2.0, route.Stages[1].BeforeSeconds) + require.EqualValues(t, 50, route.NetworkBytes) + require.Zero(t, route.RelayNetworkBytes) +} + +func TestRelayIsMandatoryInBothDirectionsAndHasTwoSerialStages(t *testing.T) { + m := testModel() + for _, pair := range [][2]string{{"cloud-a", "hpc-a"}, {"hpc-a", "cloud-a"}} { + route, err := m.Route(pair[0], pair[1], 100) + require.NoError(t, err) + require.Equal(t, "gateway", route.Strategy) + require.Len(t, route.Stages, 2) + require.Equal(t, pair[0], route.Stages[0].SourceResourceID) + require.Equal(t, "relay", route.Stages[0].TargetResourceID) + require.Equal(t, "relay", route.Stages[1].SourceResourceID) + require.Equal(t, pair[1], route.Stages[1].TargetResourceID) + require.EqualValues(t, 100, route.NetworkBytes) + require.Equal(t, route.NetworkBytes, route.RelayNetworkBytes) + // ingress 1s + validation 1s precedes egress 2s + extraction 2s. + require.InDelta(t, 1, route.Stages[0].AfterSeconds, 1e-9) + require.InDelta(t, 2, route.Stages[1].AfterSeconds, 1e-9) + } +} + +func TestCloudPeerAndLocalAvoidRelay(t *testing.T) { + m := testModel() + peer, err := m.Route("cloud-a", "cloud-b", 100) + require.NoError(t, err) + require.Equal(t, "direct-runtime", peer.Strategy) + require.Len(t, peer.Stages, 1) + require.EqualValues(t, 50, peer.NetworkBytes) + require.Zero(t, peer.RelayNetworkBytes) + require.InDelta(t, 2, peer.Stages[0].BeforeSeconds, 1e-9) + local, err := m.Route("cloud-a", "cloud-a", 100) + require.NoError(t, err) + require.Equal(t, "runtime-local", local.Strategy) + require.Empty(t, local.Stages) + require.Zero(t, local.NetworkBytes) +} + +func TestSharedStorageNeedsProofOnBothNodes(t *testing.T) { + m := testModel() + a := m.Resources["hpc-a"] + a.SharedStorageID = "nfs-evidence" + a.SharedStorageProven = true + m.Resources["hpc-a"] = a + b := m.Resources["hpc-b"] + b.SharedStorageID = "nfs-evidence" + m.Resources["hpc-b"] = b + _, err := m.Route("hpc-a", "hpc-b", 100) + require.ErrorContains(t, err, "no evidenced gateway profile") + b.SharedStorageProven = true + m.Resources["hpc-b"] = b + shared, err := m.Route("hpc-a", "hpc-b", 100) + require.NoError(t, err) + require.Equal(t, "shared-storage", shared.Strategy) + require.Zero(t, shared.NetworkBytes) + require.InDelta(t, 3, shared.Stages[0].BeforeSeconds+shared.Stages[0].AfterSeconds, 1e-9) +} + +func TestMergeChecksAllInputButCopiesUnionOnce(t *testing.T) { + m := testModel() + seconds, err := m.MergeSeconds("cloud-b", 250, 150) + require.NoError(t, err) + require.InDelta(t, 10, seconds, 1e-9) // check 2.5, copy 3, verify 4.5. + zero, err := m.MergeSeconds("hpc-a", 0, 0) + require.NoError(t, err) + require.Zero(t, zero) + _, err = m.MergeSeconds("cloud-a", 100, 200) + require.Error(t, err) +} + +func TestMergeCommandOverheadChargedOnceAndNotForRoots(t *testing.T) { + m := testModel() + m.MergeProfiles[0].CommandOverheadSeconds = 7 + seconds, err := m.MergeSeconds("cloud-b", 250, 150) + require.NoError(t, err) + require.InDelta(t, 17, seconds, 1e-9) + zero, err := m.MergeSeconds("cloud-b", 0, 0) + require.NoError(t, err) + require.Zero(t, zero) + m.MergeProfiles[0].CommandOverheadSeconds = -1 + require.Error(t, m.Validate()) +} + +func TestModelRejectsUnknownRatesAndNonfiniteInputs(t *testing.T) { + m := testModel() + _, err := m.Route("unknown", "cloud-a", 100) + require.Error(t, err) + _, err = m.Route("hpc-a", "hpc-b", 100) + require.Error(t, err) + _, err = m.Route("cloud-a", "cloud-b", -1) + require.Error(t, err) + _, err = m.Route("cloud-a", "cloud-b", math.MaxInt64) + require.ErrorContains(t, err, "overflow") + m.Profiles[0].UploadArchiveBytesPerSecond = math.Inf(1) + require.Error(t, m.Validate()) + m = testModel() + m.Profiles[1].IngressIncludesSourcePreparation = false + require.Error(t, m.Validate()) + m = testModel() + m.Profiles = append(m.Profiles, m.Profiles[0]) + require.Error(t, m.Validate()) + legacy, err := Decode(nil) + require.NoError(t, err) + require.Nil(t, legacy) + decoded, err := Decode(map[string]any{"workspaceMaterializationModel": testModel()}) + require.NoError(t, err) + require.NotNil(t, decoded) +} diff --git a/internal/domain/materializationmodel/stage_version_test.go b/internal/domain/materializationmodel/stage_version_test.go new file mode 100644 index 00000000..41fdf500 --- /dev/null +++ b/internal/domain/materializationmodel/stage_version_test.go @@ -0,0 +1,16 @@ +package materializationmodel + +import ( + "github.com/stretchr/testify/require" + "testing" +) + +func TestLegacySchemaCannotSilentlyCarryNewStageFields(t *testing.T) { + m := testModel() + m.InputObservationProfiles = []InputObservationProfile{{TargetClass: "micro", Source: "fixture", Samples: 1, FixedSeconds: 3}} + require.ErrorContains(t, m.Validate(), "schema version 2") + m.SchemaVersion = 2 + require.NoError(t, m.Validate()) + m.SchemaVersion = 3 + require.Error(t, m.Validate()) +} diff --git a/internal/domain/resource/cpu_capacity_test.go b/internal/domain/resource/cpu_capacity_test.go new file mode 100644 index 00000000..50901e5d --- /dev/null +++ b/internal/domain/resource/cpu_capacity_test.go @@ -0,0 +1,18 @@ +package resource + +import "testing" + +func TestSchedulingCPUCoresRespectsExplicitCapacity(t *testing.T) { + for _, test := range []struct { + cores int + capacity float64 + want int + }{ + {2, 1, 1}, {8, 3.5, 3}, {2, 4, 2}, {2, 0, 2}, {0, 0, 1}, {2, 0.25, 1}, + } { + r := Resource{CPUCores: test.cores, CPUCapacity: test.capacity} + if got := r.SchedulingCPUCores(); got != test.want { + t.Fatalf("resource=%+v got=%d want=%d", r, got, test.want) + } + } +} diff --git a/internal/domain/resource/resource.go b/internal/domain/resource/resource.go index 804f1559..8ac94e44 100644 --- a/internal/domain/resource/resource.go +++ b/internal/domain/resource/resource.go @@ -1,6 +1,9 @@ package resource -import "time" +import ( + "math" + "time" +) type ResourceType string type ResourceExecutionTarget string @@ -56,6 +59,17 @@ type Resource struct { Metadata map[string]any `json:"metadata,omitempty"` } +// SchedulingCPUCores limits physical lanes to the explicitly available CPU +// capacity. Fractional capacity still has one lane; activity compatibility +// independently rejects requests exceeding that fractional capacity. +func (r Resource) SchedulingCPUCores() int { + count := max(1, r.CPUCores) + if r.CPUCapacity > 0 && r.CPUCapacity < float64(count) { + count = max(1, int(math.Floor(r.CPUCapacity))) + } + return count +} + type ResourceRuntimeBinding struct { ResourceID string `json:"resourceId"` RuntimeID string `json:"runtimeId"` diff --git a/internal/domain/storagebudget/budget.go b/internal/domain/storagebudget/budget.go new file mode 100644 index 00000000..944a6e15 --- /dev/null +++ b/internal/domain/storagebudget/budget.go @@ -0,0 +1,312 @@ +// Package storagebudget checks explicit effective quota snapshots, not nominal +// filesystem capacity. It does not assume cleanup or deduplication across +// separate workspaces. Reservations are conservative bounds, not measured peaks. +package storagebudget + +import ( + "encoding/json" + "fmt" + "math" + "sort" + "time" + + "github.com/UFFeScience/akoflow/internal/domain/resource" + "github.com/UFFeScience/akoflow/internal/domain/workflow" +) + +type Snapshot struct { + StorageID string `json:"storageId"` + AvailableBytes int64 `json:"availableBytes"` + ObservedAt time.Time `json:"observedAt"` + Source string `json:"source"` +} + +type Reservation struct { + StorageID string `json:"storageId"` + AvailableBytes int64 `json:"availableBytes"` + ReservedUpperBoundBytes int64 `json:"reservedUpperBoundBytes"` + Source string `json:"source"` + ObservedAt time.Time `json:"observedAt"` +} + +// InsufficientQuota is candidate infeasibility, not a malformed model. +type InsufficientQuota struct{ Reservation Reservation } + +func (e *InsufficientQuota) Error() string { + r := e.Reservation + return fmt.Sprintf("workspace storage quota %s insufficient: available %d bytes, conservative reservation %d bytes; verified cleanup/export or more quota required", r.StorageID, r.AvailableBytes, r.ReservedUpperBoundBytes) +} + +type Timing struct { + ActivityID string + ResourceID string + Start float64 + Finish float64 +} + +// CheckScheduled credits producer release only after every consumer's command +// barrier. Staging is still reserved for the entire run: no unproved archive +// cleanup savings are claimed. This is a forecast bound, not a live disk lease. +func CheckScheduled(version workflow.WorkflowVersion, resources []resource.Resource, timings []Timing) ([]Reservation, []string, error) { + assignments := map[string]string{} + byActivity := map[string]Timing{} + for _, t := range timings { + if t.ActivityID == "" || math.IsNaN(t.Start) || math.IsNaN(t.Finish) || math.IsInf(t.Start, 0) || math.IsInf(t.Finish, 0) || t.Start < 0 || t.Finish < t.Start { + return nil, nil, fmt.Errorf("invalid storage schedule for %s", t.ActivityID) + } + if _, exists := byActivity[t.ActivityID]; exists { + return nil, nil, fmt.Errorf("duplicate storage schedule activity %s", t.ActivityID) + } + assignments[t.ActivityID], byActivity[t.ActivityID] = t.ResourceID, t + } + for _, a := range version.Activities { + if _, exists := byActivity[a.ID]; !exists { + return nil, nil, fmt.Errorf("missing storage schedule activity %s", a.ID) + } + } + return check(version, resources, assignments, byActivity) +} + +// Check projects runtime-specific manifests and reserves all workspaces plus +// packed/received input staging until run end. This deliberately does not grant +// early-release savings unless the execution contract proves those releases. +// Shared logical nodes count against one identical effective quota snapshot. +// Missing budgets are returned explicitly; legacy workflows remain compatible. +func Check(version workflow.WorkflowVersion, resources []resource.Resource, assignments map[string]string) ([]Reservation, []string, error) { + return check(version, resources, assignments, nil) +} + +func check(version workflow.WorkflowVersion, resources []resource.Resource, assignments map[string]string, timings map[string]Timing) ([]Reservation, []string, error) { + byID, types, budgets, missingIDs, err := storageBudgetSnapshots(resources, assignments) + if err != nil { + return nil, nil, err + } + if len(budgets) == 0 { + return nil, missingIDs, nil + } + // The execution synchronizes both control and explicit data predecessors. + // Include data-only edges in the projection as well as in lease timing. + projectionVersion := version + projectionVersion.Dependencies = append([]workflow.ActivityDependency(nil), version.Dependencies...) + seenEdges := map[[2]string]bool{} + for _, d := range projectionVersion.Dependencies { + seenEdges[[2]string{d.ActivityID, d.DependsOnActivityID}] = true + } + for _, d := range version.DataDependencies { + key := [2]string{d.ConsumerActivityID, d.ProducerActivityID} + if !seenEdges[key] { + projectionVersion.Dependencies = append(projectionVersion.Dependencies, workflow.ActivityDependency{ActivityID: key[0], DependsOnActivityID: key[1]}) + seenEdges[key] = true + } + } + projected, err := workflow.ProjectWorkspaceDependenciesForResourceTypes(projectionVersion, types) + if err != nil { + return nil, missingIDs, err + } + edges := append([]workflow.ActivityDependency(nil), version.Dependencies...) + for _, d := range version.DataDependencies { + edges = append(edges, workflow.ActivityDependency{ActivityID: d.ConsumerActivityID, DependsOnActivityID: d.ProducerActivityID}) + } + reserved, events, err := reserveProjectedStorage(projected, byID, assignments, timings, edges) + if err != nil { + return nil, missingIDs, err + } + if err := accumulateStoragePeaks(reserved, events); err != nil { + return nil, missingIDs, err + } + ids := []string{} + for id := range budgets { + ids = append(ids, id) + } + sort.Strings(ids) + result := []Reservation{} + for _, id := range ids { + b := budgets[id] + result = append(result, Reservation{id, b.AvailableBytes, reserved[id], b.Source, b.ObservedAt}) + } + for _, r := range result { + if r.ReservedUpperBoundBytes > r.AvailableBytes { + return result, missingIDs, &InsufficientQuota{Reservation: r} + } + } + return result, missingIDs, nil +} + +func storageBudgetSnapshots(resources []resource.Resource, assignments map[string]string) (map[string]resource.Resource, map[string]string, map[string]Snapshot, []string, error) { + byID := map[string]resource.Resource{} + types := map[string]string{} + budgets := map[string]Snapshot{} + for _, r := range resources { + byID[r.ID] = r + } + missing := map[string]bool{} + for activity, id := range assignments { + r, ok := byID[id] + if !ok { + return nil, nil, nil, nil, fmt.Errorf("storage budget: unknown resource %s", id) + } + types[activity] = string(r.Type) + raw := r.Metadata["workspaceStorageBudget"] + if raw == nil { + missing[id] = true + continue + } + payload, err := json.Marshal(raw) + if err != nil { + return nil, nil, nil, nil, err + } + var b Snapshot + if err := json.Unmarshal(payload, &b); err != nil { + return nil, nil, nil, nil, fmt.Errorf("storage budget %s: %w", id, err) + } + if b.StorageID == "" || b.Source == "" || b.ObservedAt.IsZero() || b.AvailableBytes < 0 { + return nil, nil, nil, nil, fmt.Errorf("storage budget %s requires storage identity, source, observation and nonnegative available bytes", id) + } + if previous, ok := budgets[b.StorageID]; ok && previous != b { + return nil, nil, nil, nil, fmt.Errorf("inconsistent quota snapshots for shared storage %s", b.StorageID) + } + budgets[b.StorageID] = b + } + missingIDs := []string{} + for id := range missing { + missingIDs = append(missingIDs, id) + } + sort.Strings(missingIDs) + return byID, types, budgets, missingIDs, nil +} + +type storageReservationEvent struct { + at float64 + bytes int64 +} + +func addTimedStorageReservation( + activityID string, r resource.Resource, b Snapshot, + workspaceBytes int64, bytesToReserve []int64, timings map[string]Timing, + edges []workflow.ActivityDependency, storageReservationEvents map[string][]storageReservationEvent, +) ([]int64, error) { + // Start preparation at the earliest predecessor completion (or zero + // for roots), conservatively before the consumer command starts. + begin, end, hasParent, hasConsumer := float64(0), timings[activityID].Finish, false, false + for _, d := range edges { + if d.ActivityID == activityID { + parent, ok := timings[d.DependsOnActivityID] + if !ok || parent.Finish > timings[activityID].Start { + return nil, fmt.Errorf("noncausal storage schedule for %s", activityID) + } + if !hasParent || parent.Finish < begin { + begin = parent.Finish + } + hasParent = true + } + if d.DependsOnActivityID == activityID { + consumer, ok := timings[d.ActivityID] + if !ok { + return nil, fmt.Errorf("missing storage consumer %s", d.ActivityID) + } + end = math.Max(end, consumer.Start) + hasConsumer = true + } + } + rolling, _ := r.Metadata["workspaceStorageLifecycle"].(string) + if rolling == "consumer-barrier-v1" { + // Runtime removes preparation scratch before successful preparation + // returns, then releases producer leases before starting commands. + // Keep scratch until the command barrier conservatively; leaves are + // retained from creation, not incorrectly allocated at run time zero. + storageReservationEvents[b.StorageID] = append(storageReservationEvents[b.StorageID], storageReservationEvent{begin, workspaceBytes}) + if hasConsumer { + storageReservationEvents[b.StorageID] = append(storageReservationEvents[b.StorageID], storageReservationEvent{end, -workspaceBytes}) + } + for _, staging := range bytesToReserve[1:] { + if staging < 0 { + return nil, fmt.Errorf("negative staging reservation") + } + storageReservationEvents[b.StorageID] = append(storageReservationEvents[b.StorageID], storageReservationEvent{begin, staging}, storageReservationEvent{timings[activityID].Start, -staging}) + } + bytesToReserve = nil + } else if hasConsumer { + storageReservationEvents[b.StorageID] = append(storageReservationEvents[b.StorageID], storageReservationEvent{begin, workspaceBytes}, storageReservationEvent{end, -workspaceBytes}) + bytesToReserve = bytesToReserve[1:] + } + // Leaves remain retained; no final-product pruning/export assumed. + return bytesToReserve, nil +} + +func reserveProjectedStorage( + projected workflow.WorkflowVersion, byID map[string]resource.Resource, + assignments map[string]string, timings map[string]Timing, edges []workflow.ActivityDependency, +) (map[string]int64, map[string][]storageReservationEvent, error) { + reserved := map[string]int64{} + storageReservationEvents := map[string][]storageReservationEvent{} + for _, activity := range projected.Activities { + r := byID[assignments[activity.ID]] + if r.Metadata["workspaceStorageBudget"] == nil { + continue + } + payload, _ := json.Marshal(r.Metadata["workspaceStorageBudget"]) + var b Snapshot + _ = json.Unmarshal(payload, &b) + var volume workflow.WorkspaceProjectionVolumes + if activity.Metadata["workspaceProjectionVolumes"] == nil { + return nil, nil, fmt.Errorf("storage budget needs whole-workspace projection for %s", activity.ID) + } + payload, err := json.Marshal(activity.Metadata["workspaceProjectionVolumes"]) + if err != nil { + return nil, nil, err + } + if err = json.Unmarshal(payload, &volume); err != nil { + return nil, nil, err + } + // Both archive and received/extracted inputs may coexist with the merged + // consumer and producer workspaces. Never account the same quota per node. + workspaceBytes := int64(0) + for _, bytes := range []int64{volume.PreExecutionBytes, volume.CreatedOutputBytes} { + if bytes < 0 || bytes > math.MaxInt64-workspaceBytes { + return nil, nil, fmt.Errorf("storage reservation overflow") + } + workspaceBytes += bytes + } + bytesToReserve := []int64{workspaceBytes, volume.InputTotalBytes, volume.InputTotalBytes} + if timings != nil { + var timingErr error + bytesToReserve, timingErr = addTimedStorageReservation(activity.ID, r, b, workspaceBytes, bytesToReserve, timings, edges, storageReservationEvents) + if timingErr != nil { + return nil, nil, timingErr + } + } + for _, bytes := range bytesToReserve { + if bytes < 0 || bytes > math.MaxInt64-reserved[b.StorageID] { + return nil, nil, fmt.Errorf("storage reservation overflow") + } + reserved[b.StorageID] += bytes + } + } + return reserved, storageReservationEvents, nil +} + +func accumulateStoragePeaks(reserved map[string]int64, events map[string][]storageReservationEvent) error { + for id, list := range events { + sort.Slice(list, func(i, j int) bool { + if list[i].at == list[j].at { + return list[i].bytes > list[j].bytes + } + return list[i].at < list[j].at + }) + current, peak := reserved[id], reserved[id] + for _, e := range list { + if e.bytes > 0 && e.bytes > math.MaxInt64-current { + return fmt.Errorf("storage reservation overflow") + } + current += e.bytes + if current < 0 { + return fmt.Errorf("negative storage reservation") + } + if current > peak { + peak = current + } + } + reserved[id] = peak + } + return nil +} diff --git a/internal/domain/storagebudget/budget_test.go b/internal/domain/storagebudget/budget_test.go new file mode 100644 index 00000000..07450e04 --- /dev/null +++ b/internal/domain/storagebudget/budget_test.go @@ -0,0 +1,112 @@ +package storagebudget + +import ( + "testing" + "time" + + "github.com/UFFeScience/akoflow/internal/domain/resource" + "github.com/UFFeScience/akoflow/internal/domain/workflow" + "github.com/stretchr/testify/require" +) + +func fixture(available int64) (workflow.WorkflowVersion, []resource.Resource, map[string]string) { + file := workflow.ProjectedFile{Path: "data", SizeBytes: 10, ContentID: "source"} + activity := func(id string, inputs []workflow.ProjectedFile) workflow.Activity { + return workflow.Activity{ID: id, Metadata: map[string]any{"workspaceProjection": workflow.WorkspaceProjection{Contract: "whole-workspace-v1", Source: "measured", Inputs: inputs}}} + } + v := workflow.WorkflowVersion{Activities: []workflow.Activity{activity("a", []workflow.ProjectedFile{file}), activity("b", nil)}, Dependencies: []workflow.ActivityDependency{{ActivityID: "b", DependsOnActivityID: "a"}}} + b := Snapshot{StorageID: "shared-home-user-quota", AvailableBytes: available, Source: "quota-api", ObservedAt: time.Date(2026, 10, 11, 0, 0, 0, 0, time.UTC)} + r := []resource.Resource{{ID: "node1", Metadata: map[string]any{"workspaceStorageBudget": b}}, {ID: "node2", Metadata: map[string]any{"workspaceStorageBudget": b}}} + return v, r, map[string]string{"a": "node1", "b": "node2"} +} + +func TestSharedQuotaCountsWorkspacesAndStagingNotNominalCapacity(t *testing.T) { + v, r, a := fixture(40) + r[0].StorageBytes = 1 << 40 + rows, missing, err := Check(v, r, a) + require.NoError(t, err) + require.Empty(t, missing) + require.Len(t, rows, 1) + require.Equal(t, int64(40), rows[0].ReservedUpperBoundBytes) + r[0].Metadata["workspaceStorageBudget"] = r[1].Metadata["workspaceStorageBudget"] + v, r, a = fixture(0) + _, _, err = Check(v, r, a) + require.ErrorContains(t, err, "quota") +} + +func TestInconsistentAndMissingQuotaSnapshots(t *testing.T) { + v, r, a := fixture(40) + r[1].Metadata["workspaceStorageBudget"] = Snapshot{StorageID: "shared-home-user-quota", AvailableBytes: 80, Source: "quota-api", ObservedAt: time.Now()} + _, _, err := Check(v, r, a) + require.ErrorContains(t, err, "inconsistent") + r[0].Metadata = nil + r[1].Metadata = nil + rows, missing, err := Check(v, r, a) + require.NoError(t, err) + require.Empty(t, rows) + require.Equal(t, []string{"node1", "node2"}, missing) +} + +func TestRootInputRemovedStillReservesPreparationBytes(t *testing.T) { + v, r, a := fixture(40) + model := v.Activities[0].Metadata["workspaceProjection"].(workflow.WorkspaceProjection) + model.RemovedPaths = []string{"data"} + v.Activities[0].Metadata["workspaceProjection"] = model + rows, _, err := Check(v, r, a) + require.NoError(t, err) + require.Equal(t, int64(10), rows[0].ReservedUpperBoundBytes) +} + +func TestScheduledBoundCreditsOnlyReleasedProducersAndRetainsStaging(t *testing.T) { + v, r, _ := fixture(90) + model := v.Activities[1].Metadata["workspaceProjection"].(workflow.WorkspaceProjection) + v.Activities = append(v.Activities, workflow.Activity{ID: "c", Metadata: map[string]any{"workspaceProjection": model}}, workflow.Activity{ID: "d", Metadata: map[string]any{"workspaceProjection": model}}) + v.Dependencies = append(v.Dependencies, workflow.ActivityDependency{ActivityID: "c", DependsOnActivityID: "b"}, workflow.ActivityDependency{ActivityID: "d", DependsOnActivityID: "c"}) + timings := []Timing{{"a", "node1", 0, 1}, {"b", "node2", 2, 3}, {"c", "node1", 4, 5}, {"d", "node2", 6, 7}} + rows, _, err := CheckScheduled(v, r, timings) + require.NoError(t, err) + // 60 staging + retained final 10 + at most two intermediate 10-byte + // workspaces at equal-time allocation/release barriers = 90. + require.Equal(t, int64(90), rows[0].ReservedUpperBoundBytes) + _, _, err = Check(v, r, map[string]string{"a": "node1", "b": "node2", "c": "node1", "d": "node2"}) + require.Error(t, err) +} + +func TestScheduledFanoutWaitsForLastConsumerAndRejectsInvalidTiming(t *testing.T) { + v, r, _ := fixture(1000) + v.Activities = append(v.Activities, workflow.Activity{ID: "c", Metadata: v.Activities[1].Metadata}) + v.DataDependencies = []workflow.ActivityDataDependency{{ProducerActivityID: "a", ConsumerActivityID: "c"}} + timings := []Timing{{"a", "node1", 0, 1}, {"b", "node2", 2, 3}, {"c", "node1", 8, 9}} + rows, _, err := CheckScheduled(v, r, timings) + require.NoError(t, err) + require.Equal(t, int64(70), rows[0].ReservedUpperBoundBytes) + timings[2].Start = .5 + _, _, err = CheckScheduled(v, r, timings) + require.ErrorContains(t, err, "noncausal") + _, _, err = CheckScheduled(v, r, timings[:2]) + require.ErrorContains(t, err, "missing storage schedule") +} + +func TestRollingLifecycleBoundsScratchAndRetainsLeaf(t *testing.T) { + v, r, _ := fixture(40) + model := v.Activities[1].Metadata["workspaceProjection"].(workflow.WorkspaceProjection) + v.Activities = append(v.Activities, workflow.Activity{ID: "c", Metadata: map[string]any{"workspaceProjection": model}}, workflow.Activity{ID: "d", Metadata: map[string]any{"workspaceProjection": model}}) + v.Dependencies = append(v.Dependencies, workflow.ActivityDependency{ActivityID: "c", DependsOnActivityID: "b"}, workflow.ActivityDependency{ActivityID: "d", DependsOnActivityID: "c"}) + for i := range r { + r[i].Metadata["workspaceStorageLifecycle"] = "consumer-barrier-v1" + } + timings := []Timing{{"a", "node1", 0, 1}, {"b", "node2", 2, 3}, {"c", "node1", 4, 5}, {"d", "node2", 6, 7}} + rows, missing, err := CheckScheduled(v, r, timings) + require.NoError(t, err) + require.Empty(t, missing) + // Producer + consumer + two input scratch copies coexist at the barrier. + // Scratch does not accumulate, but final leaf remains retained. + require.Equal(t, int64(40), rows[0].ReservedUpperBoundBytes) + for i := range r { + b := r[i].Metadata["workspaceStorageBudget"].(Snapshot) + b.AvailableBytes = 39 + r[i].Metadata["workspaceStorageBudget"] = b + } + _, _, err = CheckScheduled(v, r, timings) + require.ErrorContains(t, err, "insufficient") +} diff --git a/internal/domain/workflow/profile_runtime.go b/internal/domain/workflow/profile_runtime.go new file mode 100644 index 00000000..81b802cf --- /dev/null +++ b/internal/domain/workflow/profile_runtime.go @@ -0,0 +1,27 @@ +package workflow + +import "math" + +func (profile ActivityResourceProfile) UsesWorkUnits() bool { + return profile.ModelVersion == "reference-work-v1" || profile.ModelVersion == "compute-work-v1" +} + +// reference-work-v1 uses the existing persisted ModelVersion discriminator: +// RuntimeSeconds is seconds of measured wall time per one reference second. +// It must not be treated as an absolute per-activity replay after API reload. +func (profile ActivityResourceProfile) RuntimeFor(activity Activity) float64 { + if !profile.UsesWorkUnits() { + return profile.RuntimeSeconds + } + work := 0.0 + if profile.ModelVersion == "compute-work-v1" { + work, _ = activity.Metadata["computeWorkUnits"].(float64) + } else if activity.Simulation != nil { + work = activity.Simulation.DurationSeconds + } + seconds := profile.RuntimeSeconds * work + if math.IsNaN(seconds) || math.IsInf(seconds, 0) || seconds <= 0 { + return 0 + } + return seconds +} diff --git a/internal/domain/workflow/profile_runtime_test.go b/internal/domain/workflow/profile_runtime_test.go new file mode 100644 index 00000000..f9b454a5 --- /dev/null +++ b/internal/domain/workflow/profile_runtime_test.go @@ -0,0 +1,35 @@ +package workflow + +import "testing" + +func TestIndependentComputeWorkDoesNotReuseOldPredictedSeconds(t *testing.T) { + profile := ActivityResourceProfile{RuntimeSeconds: 4, ModelVersion: "compute-work-v1"} + activity := Activity{Metadata: map[string]any{"computeWorkUnits": 100.0}, Simulation: &ActivitySimulation{DurationSeconds: 999}} + if got := profile.RuntimeFor(activity); got != 400 { + t.Fatalf("work-based runtime=%g", got) + } + activity.Simulation.DurationSeconds = 10 + if got := profile.RuntimeFor(activity); got != 400 { + t.Fatalf("old machine-specific prediction changed work=%g", got) + } + delete(activity.Metadata, "computeWorkUnits") + if got := profile.RuntimeFor(activity); got != 0 { + t.Fatalf("missing work must fail closed, got %g", got) + } +} + +func TestNormalizedProfileScalesReferenceWorkInsteadOfReplayingActivity(t *testing.T) { + profile := ActivityResourceProfile{RuntimeSeconds: 2, ModelVersion: "reference-work-v1"} + for _, work := range []float64{9, 100} { + if got := profile.RuntimeFor(Activity{Simulation: &ActivitySimulation{DurationSeconds: work}}); got != 2*work { + t.Fatalf("work %g: runtime %g", work, got) + } + } + if got := profile.RuntimeFor(Activity{}); got != 0 { + t.Fatalf("unknown work must not reuse a measured activity duration: %g", got) + } + profile.ModelVersion = "legacy" + if got := profile.RuntimeFor(Activity{}); got != 2 { + t.Fatalf("legacy absolute profile changed: %g", got) + } +} diff --git a/internal/domain/workflow/speedup_profiles.go b/internal/domain/workflow/speedup_profiles.go new file mode 100644 index 00000000..239eff9e --- /dev/null +++ b/internal/domain/workflow/speedup_profiles.go @@ -0,0 +1,78 @@ +package workflow + +import ( + "fmt" + "math" + "sort" + + "github.com/UFFeScience/akoflow/internal/domain/resource" +) + +// CompleteSpeedupProfiles preserves measured pairs and fills only missing pairs. +// Estimates retain the donor's work-unit model, never claim measured samples, +// and do not remove resource/runtime or materialization compatibility checks. +func CompleteSpeedupProfiles(activities []Activity, resources []resource.Resource, profiles []ActivityResourceProfile) ([]ActivityResourceProfile, error) { + valid := func(v float64) bool { return v > 0 && !math.IsInf(v, 0) && !math.IsNaN(v) } + byID := map[string]resource.Resource{} + for _, r := range resources { + byID[r.ID] = r + } + pairs := map[string]bool{} + donors := map[string][]ActivityResourceProfile{} + for _, p := range profiles { + pairs[p.ActivityTypeID+"\x00"+p.ResourceID] = true + if p.UsesWorkUnits() && valid(p.RuntimeSeconds) && valid(byID[p.ResourceID].ComputeSpeedup) && p.SampleSize > 0 { + donors[p.ActivityTypeID] = append(donors[p.ActivityTypeID], p) + } + } + for family := range donors { + sort.Slice(donors[family], func(i, j int) bool { + a, b := donors[family][i], donors[family][j] + if a.SampleSize != b.SampleSize { + return a.SampleSize > b.SampleSize + } + return a.ID < b.ID + }) + } + out := append([]ActivityResourceProfile(nil), profiles...) + for _, a := range activities { + for _, r := range resources { + key := a.ActivityTypeID + "\x00" + r.ID + if !r.Schedulable || pairs[key] { + continue + } + if !valid(r.ComputeSpeedup) || len(donors[a.ActivityTypeID]) == 0 { + return nil, fmt.Errorf("speedup estimate requires measured work-unit donor and positive speedup: %s/%s", a.ActivityTypeID, r.ID) + } + d := donors[a.ActivityTypeID][0] + sourceSpeed := byID[d.ResourceID].ComputeSpeedup + seconds := d.RuntimeSeconds * sourceSpeed / r.ComputeSpeedup + if !valid(seconds) { + return nil, fmt.Errorf("invalid speedup estimate: %s", key) + } + out = append( + out, + ActivityResourceProfile{ + ID: "speedup-estimate-" + a.ActivityTypeID + "-" + r.ID, + ActivityTypeID: a.ActivityTypeID, + ResourceID: r.ID, + RuntimeSeconds: seconds, + ModelVersion: d.ModelVersion, + Source: "speedup-estimate-v1:" + d.ID, + Metadata: map[string]any{ + "estimated": true, + "calibrationStatus": "extrapolated-not-measured", + "donorProfileId": d.ID, + "donorSource": d.Source, + "donorSampleSize": d.SampleSize, + "sourceSpeedup": sourceSpeed, + "targetSpeedup": r.ComputeSpeedup, + "formula": "donorSecondsPerWorkUnit * sourceSpeedup / targetSpeedup", + }, + }, + ) + pairs[key] = true + } + } + return out, nil +} diff --git a/internal/domain/workflow/speedup_profiles_test.go b/internal/domain/workflow/speedup_profiles_test.go new file mode 100644 index 00000000..44fc6d51 --- /dev/null +++ b/internal/domain/workflow/speedup_profiles_test.go @@ -0,0 +1,23 @@ +package workflow + +import ( + "github.com/UFFeScience/akoflow/internal/domain/resource" + "github.com/stretchr/testify/require" + "testing" +) + +func TestSpeedupProfilesPreserveMeasuredAndScaleOnlyMissing(t *testing.T) { + a := []Activity{{ActivityTypeID: "family"}, {ActivityTypeID: "family"}} + r := []resource.Resource{{ID: "micro", ComputeSpeedup: .25, Schedulable: true}, {ID: "hpc", ComputeSpeedup: 1, Schedulable: true}, {ID: "relay", Schedulable: false}} + p := []ActivityResourceProfile{{ID: "measured", ActivityTypeID: "family", ResourceID: "micro", RuntimeSeconds: 8, ModelVersion: "compute-work-v1", SampleSize: 3}} + out, err := CompleteSpeedupProfiles(a, r, p) + require.NoError(t, err) + require.Len(t, out, 2) + require.Equal(t, p[0], out[0]) + require.Equal(t, 2.0, out[1].RuntimeSeconds) + require.Zero(t, out[1].SampleSize) + require.Equal(t, true, out[1].Metadata["estimated"]) + require.Equal(t, 6.0, out[1].RuntimeFor(Activity{Metadata: map[string]any{"computeWorkUnits": 3.0}})) + _, err = CompleteSpeedupProfiles(a, r, nil) + require.Error(t, err) +} diff --git a/internal/domain/workflow/workspace_projection.go b/internal/domain/workflow/workspace_projection.go new file mode 100644 index 00000000..ee9e2f5f --- /dev/null +++ b/internal/domain/workflow/workspace_projection.go @@ -0,0 +1,280 @@ +package workflow + +import ( + "encoding/json" + "fmt" + "math" + "path" + "strings" +) + +// WorkspaceProjection describes expected files, not observed future execution. +// Inputs are staged before execution; Outputs replace paths after execution. +// ContentID can be a calibrated symbolic identity when an output hash is unknown. +type WorkspaceProjection struct { + Contract string `json:"contract"` + Source string `json:"source"` + Inputs []ProjectedFile `json:"inputs,omitempty"` + Outputs []ProjectedFile `json:"outputs,omitempty"` + RemovedPaths []string `json:"removedPaths,omitempty"` + // A measured pre-prune snapshot can be larger than the exported manifest. + TerminalObservationBytes *int64 `json:"terminalObservationBytes,omitempty"` + PublicationCleanupBytes *int64 `json:"publicationCleanupBytes,omitempty"` + // Runtime packaging can put additional files into a cloud workspace while + // an HPC image reads those files outside the exported workspace. Inputs is + // the conservative search default; a complete assignment selects a variant. + InputsByResourceType map[string][]ProjectedFile `json:"inputsByResourceType,omitempty"` +} + +type ProjectedFile struct { + Path string `json:"path"` + SizeBytes int64 `json:"sizeBytes"` + ContentID string `json:"contentId"` +} + +type WorkspaceProjectionVolumes struct { + PreExecutionBytes int64 `json:"preExecutionBytes"` + CreatedOutputBytes int64 `json:"createdOutputBytes"` + InputTotalBytes int64 `json:"inputTotalBytes"` + InputUniqueBytes int64 `json:"inputUniqueBytes"` + InputFiles int `json:"inputFiles"` + OutputBytes int64 `json:"outputBytes"` + OutputFiles int `json:"outputFiles"` + TerminalObservationBytes int64 `json:"terminalObservationBytes"` + PublicationCleanupBytes int64 `json:"publicationCleanupBytes"` +} + +// ProjectWorkspaceDependencies preserves legacy edge semantics unless an +// explicit whole-workspace model is supplied. Partial models fail closed: a +// missing producer manifest must not silently turn into zero transfer bytes. +// This pure transformation is shared by planning and both simulation adapters. +func ProjectWorkspaceDependencies(version WorkflowVersion) (WorkflowVersion, error) { + return ProjectWorkspaceDependenciesForResourceTypes(version, nil) +} + +func ProjectWorkspaceDependenciesForResourceTypes(version WorkflowVersion, resourceTypes map[string]string) (WorkflowVersion, error) { + enabled := false + for _, activity := range version.Activities { + if activity.Metadata["workspaceProjection"] != nil { + enabled = true + } + } + if !enabled { + return version, nil + } + models, err := loadWorkspaceProjectionModels(version, resourceTypes) + if err != nil { + return version, err + } + preds := map[string][]string{} + for _, edge := range version.Dependencies { + if _, ok := models[edge.ActivityID]; !ok { + return version, fmt.Errorf("unknown consumer %q", edge.ActivityID) + } + if _, ok := models[edge.DependsOnActivityID]; !ok { + return version, fmt.Errorf("unknown producer %q", edge.DependsOnActivityID) + } + preds[edge.ActivityID] = append(preds[edge.ActivityID], edge.DependsOnActivityID) + } + manifests := map[string]map[string]ProjectedFile{} + volumes := map[string]WorkspaceProjectionVolumes{} + for len(manifests) < len(models) { + progress := false + for _, activity := range version.Activities { + if _, done := manifests[activity.ID]; done { + continue + } + ready := true + for _, producer := range preds[activity.ID] { + if _, done := manifests[producer]; !done { + ready = false + } + } + if !ready { + continue + } + manifest, volume, err := projectActivityWorkspace(activity.ID, models[activity.ID], preds[activity.ID], manifests) + if err != nil { + return version, err + } + manifests[activity.ID], volumes[activity.ID] = manifest, volume + progress = true + } + if !progress { + return version, fmt.Errorf("workspace projection dependency cycle") + } + } + // Each dependency transfers the producer's complete workspace, even when + // siblings inherit identical paths. Merge deduplication is a separate cost; + // this must not assume a destination cache or skip a route optimistically. + version.DataDependencies = make([]ActivityDataDependency, 0, len(version.Dependencies)) + for _, edge := range version.Dependencies { + var bytes int64 + for _, file := range manifests[edge.DependsOnActivityID] { + if file.SizeBytes > math.MaxInt64-bytes { + return version, fmt.Errorf("workspace size overflow for %q", edge.DependsOnActivityID) + } + bytes += file.SizeBytes + } + version.DataDependencies = append(version.DataDependencies, ActivityDataDependency{ProducerActivityID: edge.DependsOnActivityID, ConsumerActivityID: edge.ActivityID, LogicalName: "whole-workspace-v1", SizeBytes: bytes}) + } + version.Activities = append([]Activity(nil), version.Activities...) + for index := range version.Activities { + metadata := map[string]any{} + for key, value := range version.Activities[index].Metadata { + metadata[key] = value + } + metadata["workspaceProjectionVolumes"] = volumes[version.Activities[index].ID] + version.Activities[index].Metadata = metadata + } + return version, nil +} + +func loadWorkspaceProjectionModels(version WorkflowVersion, resourceTypes map[string]string) (map[string]WorkspaceProjection, error) { + models := map[string]WorkspaceProjection{} + for _, activity := range version.Activities { + raw := activity.Metadata["workspaceProjection"] + if raw == nil { + return nil, fmt.Errorf("workspace projection missing for activity %q", activity.ID) + } + encoded, err := json.Marshal(raw) + if err != nil { + return nil, err + } + var model WorkspaceProjection + if err = json.Unmarshal(encoded, &model); err != nil { + return nil, fmt.Errorf("workspace projection %q: %w", activity.ID, err) + } + if model.Contract != "whole-workspace-v1" || strings.TrimSpace(model.Source) == "" { + return nil, fmt.Errorf("workspace projection %q requires whole-workspace-v1 and provenance", activity.ID) + } + if model.TerminalObservationBytes != nil && *model.TerminalObservationBytes < 0 { + return nil, fmt.Errorf("negative terminal observation bytes for %q", activity.ID) + } + if model.PublicationCleanupBytes != nil && *model.PublicationCleanupBytes < 0 { + return nil, fmt.Errorf("negative publication cleanup bytes for %q", activity.ID) + } + if resourceTypes != nil && len(model.InputsByResourceType) > 0 { + resourceType, exists := resourceTypes[activity.ID] + if !exists { + return nil, fmt.Errorf("workspace projection resource type missing for %q", activity.ID) + } + inputs, exists := model.InputsByResourceType[resourceType] + if !exists { + return nil, fmt.Errorf("workspace projection has no %q runtime inputs for %q", resourceType, activity.ID) + } + model.Inputs = inputs + } + if _, exists := models[activity.ID]; exists { + return nil, fmt.Errorf("duplicate activity %q", activity.ID) + } + models[activity.ID] = model + } + return models, nil +} + +func mergeProjectedFile(manifest map[string]ProjectedFile, activityID string, file ProjectedFile, replace bool) error { + if file.Path == "" || strings.ContainsRune( + file.Path, + '\x00', + ) || path.IsAbs( + file.Path, + ) || path.Clean( + file.Path, + ) != file.Path || file.Path == "." || file.Path == ".." || strings.HasPrefix( + file.Path, + "../", + ) || strings.Contains( + file.Path, + "\\", + ) || file.SizeBytes < 0 || file.ContentID == "" { + return fmt.Errorf("invalid projected file %q in %q", file.Path, activityID) + } + if previous, exists := manifest[file.Path]; exists && !replace && (previous.ContentID != file.ContentID || previous.SizeBytes != file.SizeBytes) { + return fmt.Errorf("projected workspace conflict in %q at %q", activityID, file.Path) + } + manifest[file.Path] = file + return nil +} + +func finalizeProjectedWorkspace(activityID string, model WorkspaceProjection, manifest map[string]ProjectedFile, volume *WorkspaceProjectionVolumes) error { + merge := func(file ProjectedFile, replace bool) error { + return mergeProjectedFile(manifest, activityID, file, replace) + } + for _, removed := range model.RemovedPaths { + if _, exists := manifest[removed]; !exists { + return fmt.Errorf("removed path %q absent in %q", removed, activityID) + } + delete(manifest, removed) + } + outputPaths := map[string]bool{} + for _, file := range model.Outputs { + if file.SizeBytes > math.MaxInt64-volume.CreatedOutputBytes || file.SizeBytes < 0 { + return fmt.Errorf("workspace created output size overflow for %q", activityID) + } + volume.CreatedOutputBytes += file.SizeBytes + if outputPaths[file.Path] { + return fmt.Errorf("duplicate projected output %q in %q", file.Path, activityID) + } + outputPaths[file.Path] = true + if err := merge(file, true); err != nil { + return err + } + } + for _, file := range manifest { + if file.SizeBytes > math.MaxInt64-volume.OutputBytes { + return fmt.Errorf("workspace output size overflow for %q", activityID) + } + volume.OutputBytes += file.SizeBytes + } + volume.OutputFiles = len(manifest) + volume.TerminalObservationBytes = volume.OutputBytes + if model.PublicationCleanupBytes != nil { + volume.PublicationCleanupBytes = *model.PublicationCleanupBytes + } + if model.TerminalObservationBytes != nil { + volume.TerminalObservationBytes = *model.TerminalObservationBytes + } + return nil +} + +func projectActivityWorkspace(activityID string, model WorkspaceProjection, parents []string, manifests map[string]map[string]ProjectedFile) (map[string]ProjectedFile, WorkspaceProjectionVolumes, error) { + manifest := map[string]ProjectedFile{} + volume := WorkspaceProjectionVolumes{} + merge := func(file ProjectedFile, replace bool) error { + return mergeProjectedFile(manifest, activityID, file, replace) + } + for _, producer := range parents { + for _, file := range manifests[producer] { + if file.SizeBytes > math.MaxInt64-volume.InputTotalBytes { + return nil, WorkspaceProjectionVolumes{}, fmt.Errorf("workspace input size overflow for %q", activityID) + } + volume.InputTotalBytes += file.SizeBytes + if err := merge(file, false); err != nil { + return nil, WorkspaceProjectionVolumes{}, err + } + } + } + for _, file := range manifest { + if file.SizeBytes > math.MaxInt64-volume.InputUniqueBytes { + return nil, WorkspaceProjectionVolumes{}, fmt.Errorf("workspace input union overflow for %q", activityID) + } + volume.InputUniqueBytes += file.SizeBytes + } + volume.InputFiles = len(manifest) + for _, file := range model.Inputs { + if err := merge(file, false); err != nil { + return nil, WorkspaceProjectionVolumes{}, err + } + } + for _, file := range manifest { + if file.SizeBytes > math.MaxInt64-volume.PreExecutionBytes { + return nil, WorkspaceProjectionVolumes{}, fmt.Errorf("workspace preparation size overflow for %q", activityID) + } + volume.PreExecutionBytes += file.SizeBytes + } + if err := finalizeProjectedWorkspace(activityID, model, manifest, &volume); err != nil { + return nil, WorkspaceProjectionVolumes{}, err + } + return manifest, volume, nil +} diff --git a/internal/domain/workflow/workspace_projection_test.go b/internal/domain/workflow/workspace_projection_test.go new file mode 100644 index 00000000..4b054eb6 --- /dev/null +++ b/internal/domain/workflow/workspace_projection_test.go @@ -0,0 +1,139 @@ +package workflow + +import ( + "encoding/json" + "github.com/stretchr/testify/require" + "os" + "testing" +) + +func TestWorkspaceProjectionReconstructsAuditedMontageBaseline(t *testing.T) { + payload, err := os.ReadFile("../../../experiments/montage-micro-hpc-e2/workspace-projection-v2.json") + require.NoError(t, err) + var evidence struct { + ActivityModels map[string]WorkspaceProjection `json:"activityModels"` + Dependencies []ActivityDependency `json:"dependencies"` + ObservedBytes map[string]int64 `json:"observedProducerWorkspaceBytes"` + ResourceTypes map[string]string `json:"baselineAssignedResourceTypes"` + } + require.NoError(t, json.Unmarshal(payload, &evidence)) + require.Len(t, evidence.ActivityModels, 58) + version := WorkflowVersion{Dependencies: evidence.Dependencies} + for id, model := range evidence.ActivityModels { + version.Activities = append(version.Activities, Activity{ID: id, Metadata: map[string]any{"workspaceProjection": model}}) + } + projected, err := ProjectWorkspaceDependenciesForResourceTypes(version, evidence.ResourceTypes) + require.NoError(t, err) + require.Len(t, projected.DataDependencies, 114) + for _, edge := range projected.DataDependencies { + require.Equal(t, evidence.ObservedBytes[edge.ProducerActivityID], edge.SizeBytes, "producer %s consumer %s", edge.ProducerActivityID, edge.ConsumerActivityID) + } + // Change a root's future placement: inherited image packaging must change + // the downstream predictions, not replay the previous assignment's bytes. + for id, model := range evidence.ActivityModels { + if len(model.InputsByResourceType) == 0 || evidence.ResourceTypes[id] != "hpc_machine" { + continue + } + evidence.ResourceTypes[id] = "cloud_vm" + swapped, err := ProjectWorkspaceDependenciesForResourceTypes(version, evidence.ResourceTypes) + require.NoError(t, err) + for _, edge := range swapped.DataDependencies { + if edge.ProducerActivityID == id { + require.Greater(t, edge.SizeBytes, evidence.ObservedBytes[id]) + } + } + break + } +} + +func projectedActivity(id string, inputs, outputs []ProjectedFile) Activity { + return Activity{ID: id, Metadata: map[string]any{"workspaceProjection": WorkspaceProjection{Contract: "whole-workspace-v1", Source: "completed-baseline", Inputs: inputs, Outputs: outputs}}} +} + +func TestWorkspaceProjectionInheritedDiamond(t *testing.T) { + input := ProjectedFile{Path: "input.fits", SizeBytes: 100, ContentID: "input-hash"} + version := WorkflowVersion{Activities: []Activity{ + projectedActivity("a", []ProjectedFile{input}, nil), + projectedActivity("b", nil, []ProjectedFile{{Path: "b.fits", SizeBytes: 20, ContentID: "expected-b"}}), + projectedActivity("c", nil, []ProjectedFile{{Path: "c.fits", SizeBytes: 30, ContentID: "expected-c"}}), + projectedActivity("d", nil, nil), projectedActivity("e", nil, nil), + }, Dependencies: []ActivityDependency{ + {ActivityID: "b", DependsOnActivityID: "a"}, + {ActivityID: "c", DependsOnActivityID: "a"}, + {ActivityID: "d", DependsOnActivityID: "b"}, + {ActivityID: "d", DependsOnActivityID: "c"}, + {ActivityID: "e", DependsOnActivityID: "d"}, + }} + projected, err := ProjectWorkspaceDependencies(version) + require.NoError(t, err) + var sizes []int64 + for _, edge := range projected.DataDependencies { + sizes = append(sizes, edge.SizeBytes) + } + // Both parent snapshots include input; the merged child has it only once. + require.Equal(t, []int64{100, 100, 120, 130, 150}, sizes) + for _, activity := range projected.Activities { + if activity.ID == "d" { + volume := activity.Metadata["workspaceProjectionVolumes"].(WorkspaceProjectionVolumes) + require.EqualValues(t, 250, volume.InputTotalBytes) + require.EqualValues(t, 150, volume.InputUniqueBytes) + require.Equal(t, 3, volume.InputFiles) + } + } + for _, activity := range version.Activities { + require.NotContains(t, activity.Metadata, "workspaceProjectionVolumes") + } + require.Empty(t, version.DataDependencies, "must not mutate the catalog snapshot") + second, err := ProjectWorkspaceDependencies(projected) + require.NoError(t, err) + require.Equal(t, projected.DataDependencies, second.DataDependencies) +} + +func TestWorkspaceProjectionRejectsConflictingParents(t *testing.T) { + version := WorkflowVersion{ + Activities: []Activity{ + projectedActivity("a", nil, []ProjectedFile{{Path: "same", SizeBytes: 1, ContentID: "a"}}), + projectedActivity("b", nil, []ProjectedFile{{Path: "same", SizeBytes: 1, ContentID: "b"}}), + projectedActivity("c", nil, nil), + }, + Dependencies: []ActivityDependency{{ActivityID: "c", DependsOnActivityID: "a"}, {ActivityID: "c", DependsOnActivityID: "b"}}, + } + _, err := ProjectWorkspaceDependencies(version) + require.ErrorContains(t, err, "conflict") +} + +func TestWorkspaceProjectionFailsClosed(t *testing.T) { + for _, file := range []ProjectedFile{{Path: "../secret", SizeBytes: 1, ContentID: "x"}, {Path: "/absolute", SizeBytes: 1, ContentID: "x"}, {Path: "valid", SizeBytes: -1, ContentID: "x"}, {Path: "valid", SizeBytes: 1}} { + _, err := ProjectWorkspaceDependencies(WorkflowVersion{Activities: []Activity{projectedActivity("a", []ProjectedFile{file}, nil)}}) + require.Error(t, err) + } + _, err := ProjectWorkspaceDependencies(WorkflowVersion{Activities: []Activity{projectedActivity("a", nil, nil), {ID: "b"}}}) + require.ErrorContains(t, err, "missing") + _, err = ProjectWorkspaceDependencies(WorkflowVersion{Activities: []Activity{projectedActivity("a", nil, nil)}, Dependencies: []ActivityDependency{{ActivityID: "a", DependsOnActivityID: "a"}}}) + require.ErrorContains(t, err, "cycle") +} + +func TestWorkspaceProjectionLegacyUnchanged(t *testing.T) { + version := WorkflowVersion{Activities: []Activity{{ID: "a"}}, DataDependencies: []ActivityDataDependency{{SizeBytes: 42}}} + actual, err := ProjectWorkspaceDependencies(version) + require.NoError(t, err) + require.Equal(t, version, actual) +} + +func TestWorkspaceProjectionRuntimePackagingUsesAssignedType(t *testing.T) { + output := ProjectedFile{Path: "projected.fits", SizeBytes: 100, ContentID: "science-output"} + asset := ProjectedFile{Path: "image-input.fits", SizeBytes: 90, ContentID: "image-input"} + a := projectedActivity("a", []ProjectedFile{asset}, []ProjectedFile{output}) + model := a.Metadata["workspaceProjection"].(WorkspaceProjection) + model.InputsByResourceType = map[string][]ProjectedFile{"cloud_vm": {asset}, "hpc_machine": {}} + a.Metadata["workspaceProjection"] = model + version := WorkflowVersion{Activities: []Activity{a, projectedActivity("b", nil, nil)}, Dependencies: []ActivityDependency{{ActivityID: "b", DependsOnActivityID: "a"}}} + search, err := ProjectWorkspaceDependencies(version) + require.NoError(t, err) + require.EqualValues(t, 190, search.DataDependencies[0].SizeBytes) + hpc, err := ProjectWorkspaceDependenciesForResourceTypes(version, map[string]string{"a": "hpc_machine", "b": "cloud_vm"}) + require.NoError(t, err) + require.EqualValues(t, 100, hpc.DataDependencies[0].SizeBytes) + _, err = ProjectWorkspaceDependenciesForResourceTypes(version, map[string]string{"a": "unknown"}) + require.ErrorContains(t, err, "runtime inputs") +} diff --git a/internal/infrastructure/database/cloud/lifecycle_attempt_test.go b/internal/infrastructure/database/cloud/lifecycle_attempt_test.go new file mode 100644 index 00000000..03cd20c2 --- /dev/null +++ b/internal/infrastructure/database/cloud/lifecycle_attempt_test.go @@ -0,0 +1,81 @@ +package cloud_test + +import ( + "context" + "path/filepath" + "testing" + + "github.com/UFFeScience/akoflow/internal/infrastructure/database" + dbcloud "github.com/UFFeScience/akoflow/internal/infrastructure/database/cloud" +) + +func TestLifecycleSupersededAttemptStillProtectsLiveWork(t *testing.T) { + db, err := database.Open(filepath.Join(t.TempDir(), "guard.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + _, err = db.Exec(` + CREATE TABLE task_executions(execution_run_id TEXT, activity_id TEXT, attempt INTEGER, status TEXT, metadata TEXT); + CREATE TABLE activity_handles(resource_id TEXT, status TEXT, metadata TEXT); + CREATE TABLE cloud_provisioned_instances(id TEXT, capacity_target_id TEXT); + CREATE TABLE transfer_runs(status TEXT, route TEXT); + CREATE TABLE artifact_materializations(run_id TEXT, resource_id TEXT, status TEXT); + CREATE TABLE cloud_operation_runs(id TEXT, instance_id TEXT, execution_run_id TEXT, status TEXT); + CREATE TABLE console_sessions(resource_id TEXT, status TEXT); + CREATE TABLE activity_workspaces(id TEXT, state TEXT, uri TEXT); + CREATE TABLE workspace_leases(workspace_id TEXT, released INTEGER); + INSERT INTO cloud_provisioned_instances VALUES('vm','target'); + INSERT INTO task_executions VALUES('run','activity',1,'running','{"cloudInstanceId":"vm"}');`) + if err != nil { + t.Fatal(err) + } + r := dbcloud.New(db) + ctx := context.Background() + if err := r.CheckCloudLifecycle(ctx, "vm", "destroy"); err == nil { + t.Fatal("live task must block cleanup") + } + if _, err := db.Exec(`INSERT INTO task_executions VALUES('run','activity',2,'completed','{"cloudInstanceId":"vm"}')`); err != nil { + t.Fatal(err) + } + if err := r.CheckCloudLifecycle(ctx, "vm", "destroy"); err != nil { + t.Fatalf("completed replacement: %v", err) + } + if _, err := db.Exec(`INSERT INTO activity_workspaces VALUES('producer','sealed','cloud://target/path?cloudInstanceId=vm&runtime=x'); + INSERT INTO workspace_leases VALUES('producer',0)`); err != nil { + t.Fatal(err) + } + if err := r.CheckCloudLifecycle(ctx, "vm", "destroy"); err == nil { + t.Fatal("completed producer with pending consumers must block cleanup") + } + if _, err := db.Exec(`UPDATE workspace_leases SET released=1`); err != nil { + t.Fatal(err) + } + if err := r.CheckCloudLifecycle(ctx, "vm", "destroy"); err != nil { + t.Fatalf("released consumer lease: %v", err) + } + if _, err := db.Exec(`INSERT INTO activity_handles VALUES('target','running','{"cloudInstanceId":"old-vm"}')`); err != nil { + t.Fatal(err) + } + if err := r.CheckCloudLifecycle(ctx, "vm", "destroy"); err != nil { + t.Fatalf("another instance's handle must not block cleanup: %v", err) + } + if _, err := db.Exec(`INSERT INTO activity_handles VALUES('target','running','{"cloudInstanceId":"vm"}')`); err != nil { + t.Fatal(err) + } + if err := r.CheckCloudLifecycle(ctx, "vm", "destroy"); err == nil { + t.Fatal("live handle must block even with completed replacement") + } + if _, err := db.Exec(`UPDATE activity_handles SET status='stopped'; INSERT INTO transfer_runs VALUES('running','{"sourceCloudInstanceId":"vm"}')`); err != nil { + t.Fatal(err) + } + if err := r.CheckCloudLifecycle(ctx, "vm", "destroy"); err == nil { + t.Fatal("active transfer must block cleanup") + } + if _, err := db.Exec(`UPDATE transfer_runs SET status='completed'; INSERT INTO task_executions VALUES('run','activity',3,'preparing','{"cloudInstanceId":"vm"}')`); err != nil { + t.Fatal(err) + } + if err := r.CheckCloudLifecycle(ctx, "vm", "destroy"); err == nil { + t.Fatal("newer preparation must still block cleanup") + } +} diff --git a/internal/infrastructure/database/cloud/repository.go b/internal/infrastructure/database/cloud/repository.go index 8d05cff5..b0a52990 100644 --- a/internal/infrastructure/database/cloud/repository.go +++ b/internal/infrastructure/database/cloud/repository.go @@ -337,7 +337,24 @@ func (r *Repository) CreateCapacityTarget(ctx context.Context, value domain.Clou id,environment_id,name,provider,provider_machine_type,region,zone_policy,fixed_zone, image_reference,architecture,vcpu,memory_mib,provisioning_mode,maximum_instances, lifecycle_policy,configuration,enabled,created_at - ) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, value.ID, value.EnvironmentID, value.Name, + ) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + ON CONFLICT(id) DO UPDATE SET + environment_id=excluded.environment_id, + name=excluded.name, + provider=excluded.provider, + provider_machine_type=excluded.provider_machine_type, + region=excluded.region, + zone_policy=excluded.zone_policy, + fixed_zone=excluded.fixed_zone, + image_reference=excluded.image_reference, + architecture=excluded.architecture, + vcpu=excluded.vcpu, + memory_mib=excluded.memory_mib, + provisioning_mode=excluded.provisioning_mode, + maximum_instances=excluded.maximum_instances, + lifecycle_policy=excluded.lifecycle_policy, + configuration=excluded.configuration, + enabled=excluded.enabled`, value.ID, value.EnvironmentID, value.Name, value.Provider, value.ProviderMachineType, value.Region, value.ZonePolicy, value.FixedZone, value.ImageReference, value.Architecture, value.VCPU, value.MemoryMiB, value.ProvisioningMode, value.MaximumInstances, value.LifecyclePolicy, configuration, @@ -345,29 +362,12 @@ func (r *Repository) CreateCapacityTarget(ctx context.Context, value domain.Clou if err != nil { return err } - configurations := value.MachineConfigurations - hasDefault := false - for _, configuration := range configurations { - if configuration.ConfigurationVersionID == domain.DefaultMachineConfigurationVersionID { - hasDefault = true - break - } - } - if !hasDefault { - configurations = append([]domain.CloudTargetConfiguration{{ - ConfigurationVersionID: domain.DefaultMachineConfigurationVersionID, - ExecutionOrder: 0, Required: true, Enabled: true, - }}, configurations...) + if _, err = tx.ExecContext(ctx, `DELETE FROM cloud_capacity_target_configurations + WHERE capacity_target_id=?`, value.ID); err != nil { + return err } - for _, item := range configurations { - variables, _ := json.Marshal(item.Variables) - _, err = tx.ExecContext(ctx, `INSERT INTO cloud_capacity_target_configurations( - capacity_target_id,configuration_version_id,execution_order,variables,required,enabled - ) VALUES(?,?,?,?,?,?)`, value.ID, item.ConfigurationVersionID, item.ExecutionOrder, - variables, item.Required, item.Enabled) - if err != nil { - return err - } + if err := persistCapacityTargetConfigurations(ctx, tx, value); err != nil { + return err } return tx.Commit() } @@ -671,9 +671,32 @@ func (r *Repository) CheckCloudLifecycle(ctx context.Context, instanceID, exclud args []any }{ { - query: `SELECT COUNT(*) FROM task_executions WHERE json_extract(metadata,'$.cloudInstanceId')=? AND status IN ('blocked','ready','preparing','running')`, + query: `SELECT COUNT(*) FROM activity_workspaces w + JOIN workspace_leases l ON l.workspace_id=w.id + WHERE w.state<>'released' AND l.released=0 + AND instr('&'||substr(w.uri,instr(w.uri,'?')+1)||'&', + '&cloudInstanceId='||?||'&')>0`, + message: "producer workspaces still have pending consumers", args: []any{instanceID}, + }, + { + query: `SELECT COUNT(*) FROM task_executions t + WHERE json_extract(t.metadata,'$.cloudInstanceId')=? + AND t.status IN ('blocked','ready','preparing','running') + AND NOT EXISTS ( + SELECT 1 FROM task_executions newer + WHERE newer.execution_run_id=t.execution_run_id + AND newer.activity_id=t.activity_id AND newer.attempt>t.attempt + AND newer.status='completed')`, message: "workflow activities are still active", args: []any{instanceID}, }, + { + query: `SELECT COUNT(*) FROM activity_handles h + JOIN cloud_provisioned_instances i ON i.capacity_target_id=h.resource_id + WHERE i.id=? AND h.status IN ('starting','running') + AND (json_extract(h.metadata,'$.cloudInstanceId')=i.id + OR COALESCE(json_extract(h.metadata,'$.cloudInstanceId'),'')='')`, + message: "workflow execution handles are still active", args: []any{instanceID}, + }, { query: `SELECT COUNT(*) FROM transfer_runs WHERE status IN ('planned','running') AND (json_extract(route,'$.sourceCloudInstanceId')=? OR json_extract(route,'$.targetCloudInstanceId')=?)`, message: "data transfers are still active", args: []any{instanceID, instanceID}, @@ -703,3 +726,47 @@ func (r *Repository) CheckCloudLifecycle(ctx context.Context, instanceID, exclud } return nil } + +func persistCapacityTargetConfigurations(ctx context.Context, tx *sql.Tx, value domain.CloudCapacityTarget) error { + var err error + configurations := append([]domain.CloudTargetConfiguration(nil), value.MachineConfigurations...) + hasDefault := false + for index, configuration := range configurations { + // A published required version of the worker baseline replaces the + // default version, not the safety requirement. This supports immutable + // preinstalled images without silently appending the installation recipe. + var configurationID, status string + if err := tx.QueryRowContext(ctx, `SELECT machine_configuration_id,status + FROM machine_configuration_versions WHERE id=?`, configuration.ConfigurationVersionID).Scan(&configurationID, &status); err != nil { + return fmt.Errorf("load target configuration version: %w", err) + } + if configuration.ConfigurationVersionID == domain.DefaultMachineConfigurationVersionID { + configurations[index].Required, configurations[index].Enabled = true, true + hasDefault = true + } else if configurationID == domain.DefaultMachineConfigurationID && status == "published" && configuration.Required && configuration.Enabled { + hasDefault = true + } + } + if !hasDefault { + // Reserve nonnegative order zero for the mandatory baseline while + // preserving the relative order of explicitly supplied configurations. + for index := range configurations { + configurations[index].ExecutionOrder++ + } + configurations = append([]domain.CloudTargetConfiguration{{ + ConfigurationVersionID: domain.DefaultMachineConfigurationVersionID, + ExecutionOrder: 0, Required: true, Enabled: true, + }}, configurations...) + } + for _, item := range configurations { + variables, _ := json.Marshal(item.Variables) + _, err = tx.ExecContext(ctx, `INSERT INTO cloud_capacity_target_configurations( + capacity_target_id,configuration_version_id,execution_order,variables,required,enabled + ) VALUES(?,?,?,?,?,?)`, value.ID, item.ConfigurationVersionID, item.ExecutionOrder, + variables, item.Required, item.Enabled) + if err != nil { + return err + } + } + return nil +} diff --git a/internal/infrastructure/database/cloud/repository_test.go b/internal/infrastructure/database/cloud/repository_test.go index c2ca2a4a..953760d5 100644 --- a/internal/infrastructure/database/cloud/repository_test.go +++ b/internal/infrastructure/database/cloud/repository_test.go @@ -58,11 +58,65 @@ func TestRepositoryStoresConfigurationsAndTargets(t *testing.T) { t.Fatalf("removed target remains visible: %#v", targets) } if err := repository.CreateCapacityTarget(ctx, domain.CloudCapacityTarget{ - ID: "cpu-8-replacement", EnvironmentID: "gcp-env", Name: "CPU 8", Provider: "gcp", + ID: "cpu-8", EnvironmentID: "gcp-env", Name: "CPU 8 replacement", Provider: "gcp", ProviderMachineType: "c3-standard-8", Region: "us-central1", ImageReference: "ubuntu-2404", VCPU: 8, MemoryMiB: 32768, Enabled: true, + Configuration: map[string]any{"diskType": "hyperdisk-balanced"}, }); err != nil { - t.Fatalf("reuse removed target name: %v", err) + t.Fatalf("restore removed target id: %v", err) + } + targets, err = repository.ListCapacityTargets(ctx, "gcp-env") + if err != nil { + t.Fatal(err) + } + if len(targets) != 1 || targets[0].ID != "cpu-8" || targets[0].Name != "CPU 8 replacement" || targets[0].Configuration["diskType"] != "hyperdisk-balanced" { + t.Fatalf("restored target = %#v", targets) + } +} + +func TestCapacityTargetHonorsPublishedRequiredWorkerVersion(t *testing.T) { + ctx := context.Background() + db, err := database.Open(filepath.Join(t.TempDir(), "cloud.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := database.Bootstrap(ctx, db); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO environments(id,name) VALUES('gcp-env','GCP')`); err != nil { + t.Fatal(err) + } + repository := dbcloud.New(db) + if err := repository.EnsureDefaults(ctx); err != nil { + t.Fatal(err) + } + version := domain.MachineConfigurationVersion{ID: "preinstalled-v8", MachineConfigurationID: domain.DefaultMachineConfigurationID, + Version: 8, Status: "published", PlaybookYAML: "---\n- hosts: all\n tasks:\n - name: Verify Docker\n ansible.builtin.command: docker --version\n changed_when: false\n"} + if err := repository.CreateMachineConfigurationVersion(ctx, version); err != nil { + t.Fatal(err) + } + for _, required := range []bool{true, false} { + target := domain.CloudCapacityTarget{ID: "preinstalled", EnvironmentID: "gcp-env", Name: "Preinstalled worker", + Provider: "gcp", ProviderMachineType: "e2-micro", ImageReference: "worker-image", VCPU: 1, MemoryMiB: 1024, Enabled: true, + MachineConfigurations: []domain.CloudTargetConfiguration{{ConfigurationVersionID: version.ID, ExecutionOrder: 0, Required: required, Enabled: true}}} + if err := repository.CreateCapacityTarget(ctx, target); err != nil { + t.Fatal(err) + } + stored, err := repository.ListCapacityTargets(ctx, "gcp-env") + if err != nil { + t.Fatal(err) + } + configs := stored[0].MachineConfigurations + if required { + if len(configs) != 1 || configs[0].ConfigurationVersionID != version.ID { + t.Fatalf("explicit baseline not honored: %#v", configs) + } + } else { + if len(configs) != 2 || configs[0].ConfigurationVersionID != domain.DefaultMachineConfigurationVersionID || configs[0].ExecutionOrder >= configs[1].ExecutionOrder { + t.Fatalf("optional version bypassed baseline or collided: %#v", configs) + } + } } } diff --git a/internal/infrastructure/database/data/repository_test.go b/internal/infrastructure/database/data/repository_test.go index fab5014b..98d883b2 100644 --- a/internal/infrastructure/database/data/repository_test.go +++ b/internal/infrastructure/database/data/repository_test.go @@ -210,10 +210,15 @@ func TestMaterializationAndTransferLifecycle(t *testing.T) { } workspaceTransfer := domain.DataTransferRun{ID: "workspace-route", PlanID: "workspace-plan", ExecutionRunID: "run", ActivityID: "activity", Strategy: domain.TransferGateway, - Status: domain.TransferCompleted} + Status: domain.TransferCompleted, Route: domain.TransferRoute{ArchiveBytes: 10240, + Phases: []domain.TransferPhase{{Name: "archive-upload", StartedAt: 1, FinishedAt: 3, DurationSeconds: 2}}}} if err := repository.SaveTransferRun(ctx, workspaceTransfer); err != nil { t.Fatal(err) } + workspaceFound, err := repository.FindTransferRun(ctx, workspaceTransfer.ID) + if err != nil || workspaceFound == nil || workspaceFound.Route.ArchiveBytes != 10240 || len(workspaceFound.Route.Phases) != 1 || workspaceFound.Route.Phases[0].DurationSeconds != 2 { + t.Fatalf("phase persistence = %#v, %v", workspaceFound, err) + } chunk := domain.TransferChunkRun{TransferRunID: transfer.ID, Index: 1, Offset: 50, SizeBytes: 50, Digest: digest, Status: domain.TransferCompleted, Attempts: 2} if err := repository.SaveTransferChunkRun(ctx, chunk); err != nil { t.Fatal(err) diff --git a/internal/infrastructure/database/execution/preservation.go b/internal/infrastructure/database/execution/preservation.go new file mode 100644 index 00000000..e58a47f4 --- /dev/null +++ b/internal/infrastructure/database/execution/preservation.go @@ -0,0 +1,68 @@ +package execution + +import ( + "context" + "fmt" + "net/url" + "strings" + "time" +) + +// ReleasePreservedWorkspaces records durable export and releases leases in one +// transaction. Cancellation and absence of live handles are rechecked here. +func (r *Repository) ReleasePreservedWorkspaces(ctx context.Context, runID, instanceID string, ids []string, archive string) error { + if len(ids) == 0 || !strings.HasPrefix(archive, "storage/preserved-workspaces/") || !strings.Contains(archive, " sha256:") { + return fmt.Errorf("verified archive required") + } + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + var status string + if err := tx.QueryRowContext(ctx, `SELECT status FROM execution_runs WHERE id=?`, runID).Scan(&status); err != nil { + return err + } + if status != "cancelled" { + return fmt.Errorf("run is not cancelled") + } + var active int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM activity_handles WHERE execution_run_id=? AND status NOT IN ('completed','failed','stopped')`, runID).Scan(&active); err != nil { + return err + } + if active != 0 { + return fmt.Errorf("run has active handles") + } + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM transfer_runs WHERE execution_run_id=? AND status NOT IN ('completed','failed','cancelled','stopped')`, runID).Scan(&active); err != nil { + return err + } + if active != 0 { + return fmt.Errorf("run has active transfers") + } + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM cloud_operation_runs WHERE instance_id=? AND status IN ('queued','running')`, instanceID).Scan(&active); err != nil { + return err + } + if active != 0 { + return fmt.Errorf("cloud generation has active infrastructure operations") + } + now := time.Now().UTC() + reason := "cancelled consumers; verified relay preservation: " + archive + for _, id := range ids { + var uri string + var pinned bool + if err := tx.QueryRowContext(ctx, `SELECT uri,pinned FROM activity_workspaces WHERE id=? AND execution_run_id=?`, id, runID).Scan(&uri, &pinned); err != nil { + return err + } + location, err := url.Parse(uri) + if err != nil || pinned || location.Query().Get("cloudInstanceId") != instanceID { + return fmt.Errorf("workspace identity or retention changed") + } + if _, err := tx.ExecContext(ctx, `UPDATE workspace_leases SET released=1,released_at=?,release_reason=? WHERE workspace_id=? AND execution_run_id=? AND released=0`, now, reason, id, runID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `UPDATE activity_workspaces SET state='released',released_at=?,release_reason=? WHERE id=? AND execution_run_id=?`, now, reason, id, runID); err != nil { + return err + } + } + return tx.Commit() +} diff --git a/internal/infrastructure/database/execution/preservation_test.go b/internal/infrastructure/database/execution/preservation_test.go new file mode 100644 index 00000000..aafad127 --- /dev/null +++ b/internal/infrastructure/database/execution/preservation_test.go @@ -0,0 +1,60 @@ +package execution + +import ( + "context" + "path/filepath" + "testing" + + "github.com/UFFeScience/akoflow/internal/infrastructure/database" +) + +func TestPreservedWorkspaceReleaseRequiresCancelledIdleExactGeneration(t *testing.T) { + db, err := database.Open(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + _, err = db.Exec(`CREATE TABLE execution_runs(id TEXT,status TEXT); +CREATE TABLE activity_handles(execution_run_id TEXT,status TEXT); +CREATE TABLE transfer_runs(execution_run_id TEXT,status TEXT); +CREATE TABLE cloud_operation_runs(instance_id TEXT,status TEXT); +CREATE TABLE activity_workspaces(id TEXT,execution_run_id TEXT,uri TEXT,pinned BOOLEAN,state TEXT,released_at DATETIME,release_reason TEXT); +CREATE TABLE workspace_leases(workspace_id TEXT,execution_run_id TEXT,released BOOLEAN,released_at DATETIME,release_reason TEXT); +INSERT INTO execution_runs VALUES('run','running'); +INSERT INTO activity_workspaces VALUES('workspace','run','ssh://worker/path?cloudInstanceId=vm',0,'sealed',NULL,''); +INSERT INTO workspace_leases VALUES('workspace','run',0,NULL,'');`) + if err != nil { + t.Fatal(err) + } + repo := New(db) + ctx := context.Background() + archive := "storage/preserved-workspaces/digest.tar.gz sha256:digest" + if err := repo.ReleasePreservedWorkspaces(ctx, "run", "vm", []string{"workspace"}, archive); err == nil { + t.Fatal("live run released") + } + if _, err := db.Exec(`UPDATE execution_runs SET status='cancelled'; INSERT INTO activity_handles VALUES('run','running')`); err != nil { + t.Fatal(err) + } + if err := repo.ReleasePreservedWorkspaces(ctx, "run", "vm", []string{"workspace"}, archive); err == nil { + t.Fatal("live handles released") + } + if _, err := db.Exec(`UPDATE activity_handles SET status='stopped'`); err != nil { + t.Fatal(err) + } + if err := repo.ReleasePreservedWorkspaces(ctx, "run", "another", []string{"workspace"}, archive); err == nil { + t.Fatal("different generation released") + } + if err := repo.ReleasePreservedWorkspaces(ctx, "run", "vm", []string{"workspace", "missing"}, archive); err == nil { + t.Fatal("partial identity set accepted") + } + var released bool + if err := db.QueryRow(`SELECT released FROM workspace_leases`).Scan(&released); err != nil || released { + t.Fatalf("transaction did not roll back: %v", err) + } + if err := repo.ReleasePreservedWorkspaces(ctx, "run", "vm", []string{"workspace"}, archive); err != nil { + t.Fatal(err) + } + if err := db.QueryRow(`SELECT released FROM workspace_leases`).Scan(&released); err != nil || !released { + t.Fatalf("preserved lease not released: %v", err) + } +} diff --git a/internal/infrastructure/database/execution/repository.go b/internal/infrastructure/database/execution/repository.go index 1b6c2879..6d9b63e0 100644 --- a/internal/infrastructure/database/execution/repository.go +++ b/internal/infrastructure/database/execution/repository.go @@ -222,7 +222,7 @@ func (r *Repository) ListRuns(ctx context.Context) ([]domain.ExecutionRun, error return page.Items, err } -func (r *Repository) ListRunsPage(ctx context.Context, page, pageSize int, kind, mode, status string) (domain.ExecutionRunPage, error) { +func (r *Repository) ListRunsPage(ctx context.Context, page, pageSize int, kind, mode, status string, workflowVersionIDs ...string) (domain.ExecutionRunPage, error) { if page < 1 { page = 1 } @@ -230,6 +230,15 @@ func (r *Repository) ListRunsPage(ctx context.Context, page, pageSize int, kind, pageSize = 20 } conditions, arguments := runFeedFilters(kind, mode, status) + if len(workflowVersionIDs) > 0 && strings.TrimSpace(workflowVersionIDs[0]) != "" { + if conditions == "" { + conditions = " WHERE " + } else { + conditions += " AND " + } + conditions += "schedule_plan_id IN (SELECT id FROM schedule_plans WHERE workflow_version_id=?)" + arguments = append(arguments, strings.TrimSpace(workflowVersionIDs[0])) + } filteredFeed := `SELECT * FROM (` + runFeedSelect + `)` + conditions var total int if err := r.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM (`+filteredFeed+`)`, arguments...).Scan(&total); err != nil { @@ -339,7 +348,7 @@ func (r *Repository) ListTasks(ctx context.Context, runID string) ([]domain.Task t.metadata FROM task_executions t LEFT JOIN activity_handles h ON h.execution_run_id=t.execution_run_id - AND h.activity_id=t.activity_id AND h.status IN ('failed', 'stopped') + AND h.activity_id=t.activity_id AND h.id=t.id AND h.status IN ('failed', 'stopped') WHERE t.execution_run_id=? ORDER BY t.started_at, t.activity_id`, runID) if err != nil { return nil, err diff --git a/internal/infrastructure/database/execution/repository_test.go b/internal/infrastructure/database/execution/repository_test.go index 2adf43a3..d1cb0da4 100644 --- a/internal/infrastructure/database/execution/repository_test.go +++ b/internal/infrastructure/database/execution/repository_test.go @@ -139,6 +139,44 @@ func TestRepositoryReturnsNilForMissingRecords(t *testing.T) { } } +func TestListRunsPageFiltersWorkflowBeforePagination(t *testing.T) { + repository := setup(t) + ctx := context.Background() + for _, query := range []string{ + `INSERT INTO workflow_versions(id, workflow_id, version, definition_hash) VALUES ('other-version', 'workflow', 2, 'other')`, + `INSERT INTO schedule_plans(id, workflow_version_id, execution_scope_id, source, algorithm) VALUES ('other-plan', 'other-version', 'scope', 'plugin', 'test')`, + } { + if _, err := repository.db.ExecContext(ctx, query); err != nil { + t.Fatal(err) + } + } + for _, run := range []domain.ExecutionRun{ + {ID: "a", SchedulePlanID: "plan", Mode: domain.ExecutionModeReal, Status: domain.ExecutionRunCompleted}, + {ID: "b", SchedulePlanID: "plan", Mode: domain.ExecutionModeReal, Status: domain.ExecutionRunCompleted}, + {ID: "z-unrelated", SchedulePlanID: "other-plan", Mode: domain.ExecutionModeReal, Status: domain.ExecutionRunCompleted}, + } { + if err := repository.CreateRun(ctx, run); err != nil { + t.Fatal(err) + } + } + first, err := repository.ListRunsPage(ctx, 1, 1, "", "", "", "workflow-version") + if err != nil || first.Total != 2 || !first.HasNext || len(first.Items) != 1 || first.Items[0].ID != "b" { + t.Fatalf("first=%+v err=%v", first, err) + } + second, err := repository.ListRunsPage(ctx, 2, 1, "", "", "", "workflow-version") + if err != nil || second.HasNext || len(second.Items) != 1 || second.Items[0].ID != "a" { + t.Fatalf("second=%+v err=%v", second, err) + } + empty, err := repository.ListRunsPage(ctx, 1, 20, "", "", "", "missing") + if err != nil || empty.Total != 0 || len(empty.Items) != 0 || empty.HasNext { + t.Fatalf("empty=%+v err=%v", empty, err) + } + standalone, err := repository.ListRunsPage(ctx, 1, 20, "standalone", "", "", "workflow-version") + if err != nil || standalone.Total != 0 { + t.Fatalf("standalone=%+v err=%v", standalone, err) + } +} + func TestListRunsPageCombinesWorkflowInteractiveAndStandaloneRuns(t *testing.T) { repository := setup(t) ctx := context.Background() @@ -262,11 +300,11 @@ func TestFailRunAndStoppedHandleOverrideRunningTask(t *testing.T) { if err := repository.CreateRun(ctx, domain.ExecutionRun{ID: "run", SchedulePlanID: "plan", Mode: domain.ExecutionModeReal, Status: domain.ExecutionRunRunning}); err != nil { t.Fatal(err) } - task := domain.TaskExecution{ID: "task", ExecutionRunID: "run", PlanAssignmentID: "assignment", ActivityID: "activity", PlannedResourceID: "resource", Attempt: 1, Status: domain.TaskRunning} + task := domain.TaskExecution{ID: "run:activity", ExecutionRunID: "run", PlanAssignmentID: "assignment", ActivityID: "activity", PlannedResourceID: "resource", Attempt: 1, Status: domain.TaskRunning} if err := repository.SaveTask(ctx, task); err != nil { t.Fatal(err) } - handle := domain.ActivityHandle{ID: "handle", RunID: "run", ActivityID: "activity", ResourceID: "resource", RuntimeID: "local", Status: domain.HandleStopped, Failure: "cancelled", Metadata: map[string]any{}} + handle := domain.ActivityHandle{ID: "run:activity", RunID: "run", ActivityID: "activity", ResourceID: "resource", RuntimeID: "local", Status: domain.HandleStopped, Failure: "cancelled", Metadata: map[string]any{}} if err := repository.Save(ctx, handle); err != nil { t.Fatal(err) } diff --git a/internal/infrastructure/database/execution/task_attempt_observation_test.go b/internal/infrastructure/database/execution/task_attempt_observation_test.go new file mode 100644 index 00000000..8cca6720 --- /dev/null +++ b/internal/infrastructure/database/execution/task_attempt_observation_test.go @@ -0,0 +1,40 @@ +package execution + +import ( + "context" + "github.com/UFFeScience/akoflow/internal/domain" + "testing" +) + +func TestOldStoppedHandleDoesNotOverrideRecoveredRunningTask(t *testing.T) { + r := setup(t) + ctx := context.Background() + if err := r.CreateRun(ctx, domain.ExecutionRun{ID: "run", SchedulePlanID: "plan", Mode: domain.ExecutionModeReal, Status: domain.ExecutionRunRunning}); err != nil { + t.Fatal(err) + } + for attempt, id := range []string{"run:activity", "run:activity:attempt-2"} { + task := domain.TaskExecution{ID: id, ExecutionRunID: "run", PlanAssignmentID: "assignment", ActivityID: "activity", PlannedResourceID: "resource", Attempt: attempt + 1, Status: domain.TaskRunning, StartedAt: float64(10 + 10*attempt)} + if err := r.SaveTask(ctx, task); err != nil { + t.Fatal(err) + } + handle := domain.ActivityHandle{ID: id, RunID: "run", ActivityID: "activity", ResourceID: "resource", RuntimeID: "local", Status: domain.HandleRunning} + if attempt == 0 { + handle.Status = domain.HandleStopped + handle.Failure = "old attempt cancelled" + handle.FinishedAt = 12 + } + if err := r.Save(ctx, handle); err != nil { + t.Fatal(err) + } + } + tasks, err := r.ListTasks(ctx, "run") + if err != nil || len(tasks) != 2 { + t.Fatalf("tasks=%+v err=%v", tasks, err) + } + if tasks[0].Status != domain.TaskFailed || tasks[0].FailureReason != "old attempt cancelled" { + t.Fatalf("old attempt: %+v", tasks[0]) + } + if tasks[1].Status != domain.TaskRunning || tasks[1].FailureReason != "" || tasks[1].FinishedAt != 0 { + t.Fatalf("recovered attempt contaminated: %+v", tasks[1]) + } +} diff --git a/internal/infrastructure/database/queue/lease_attempt_limit_test.go b/internal/infrastructure/database/queue/lease_attempt_limit_test.go new file mode 100644 index 00000000..98c4c196 --- /dev/null +++ b/internal/infrastructure/database/queue/lease_attempt_limit_test.go @@ -0,0 +1,37 @@ +package queue + +import ( + "context" + "strings" + "testing" + "time" + + domainqueue "github.com/UFFeScience/akoflow/internal/domain/queue" +) + +func TestExpiredLeaseCannotReplaySingleAttemptExecution(t *testing.T) { + repository, db := setup(t) + defer db.Close() + ctx := context.Background() + job := executionJob(t, "real-no-replay", "real", "hpc") + job.MaxAttempts = 1 + stored, err := repository.Publish(ctx, job) + if err != nil { + t.Fatal(err) + } + leased, err := repository.Lease(ctx, "worker", nil, 1, time.Millisecond) + if err != nil || len(leased) != 1 { + t.Fatalf("initial lease: %v %v", leased, err) + } + if _, err := repository.ReleaseExpired(ctx, time.Now().Add(time.Second)); err != nil { + t.Fatal(err) + } + terminal, err := repository.FindByID(ctx, stored.ID) + if err != nil || terminal.Status != domainqueue.StatusFailed || terminal.CompletedAt == nil || !strings.Contains(terminal.LastError, "explicit recovery required") { + t.Fatalf("expired execution not terminal: job=%+v err=%v", terminal, err) + } + replayed, err := repository.Lease(ctx, "another-worker", nil, 1, time.Minute) + if err != nil || len(replayed) != 0 { + t.Fatalf("execution replayed after attempt limit: %v %v", replayed, err) + } +} diff --git a/internal/infrastructure/database/queue/lease_diagnostics_test.go b/internal/infrastructure/database/queue/lease_diagnostics_test.go new file mode 100644 index 00000000..0e5a1ca3 --- /dev/null +++ b/internal/infrastructure/database/queue/lease_diagnostics_test.go @@ -0,0 +1,30 @@ +package queue + +import ( + "context" + "strings" + "testing" + "time" +) + +func TestRenewLeaseReportsExpiredTerminalState(t *testing.T) { + repo, db := setup(t) + defer db.Close() + ctx := context.Background() + job := executionJob(t, "lease-diagnostic", "simulation") + job.MaxAttempts = 1 + if _, err := repo.Publish(ctx, job); err != nil { + t.Fatal(err) + } + leased, err := repo.Lease(ctx, "worker", nil, 1, time.Second) + if err != nil || len(leased) != 1 { + t.Fatalf("lease: %v %v", leased, err) + } + if _, err := repo.ReleaseExpired(ctx, time.Now().UTC().Add(2*time.Second)); err != nil { + t.Fatal(err) + } + err = repo.RenewLease(ctx, job.ID, "worker", time.Now().UTC().Add(time.Minute)) + if err == nil || !strings.Contains(err.Error(), "status=failed") || !strings.Contains(err.Error(), "ownerMatches=false") || !strings.Contains(err.Error(), "explicit recovery required") { + t.Fatalf("missing diagnostic: %v", err) + } +} diff --git a/internal/infrastructure/database/queue/repository.go b/internal/infrastructure/database/queue/repository.go index c6151ee5..7cfbde5e 100644 --- a/internal/infrastructure/database/queue/repository.go +++ b/internal/infrastructure/database/queue/repository.go @@ -305,7 +305,15 @@ func (r *Repository) RenewLease(ctx context.Context, id, owner string, expiresAt if err != nil { return err } - return requireOne(result, "renew queue job lease") + if err := requireOne(result, "renew queue job lease"); err != nil { + var status, lastError string + var sameOwner bool + if diagnosticErr := r.db.QueryRowContext(ctx, `SELECT status, lease_owner = ?, last_error FROM queue_jobs WHERE id = ?`, owner, id).Scan(&status, &sameOwner, &lastError); diagnosticErr != nil { + return fmt.Errorf("%w; lease state unavailable: %v", err, diagnosticErr) + } + return fmt.Errorf("%w; status=%s ownerMatches=%t lastError=%q", err, status, sameOwner, lastError) + } + return nil } func (r *Repository) Retry(ctx context.Context, id, owner string, cause error, availableAt time.Time) error { @@ -341,8 +349,15 @@ func (r *Repository) CancelByAggregate(ctx context.Context, aggregateType, aggre } func (r *Repository) ReleaseExpired(ctx context.Context, now time.Time) (int64, error) { - result, err := r.db.ExecContext(ctx, `UPDATE queue_jobs SET status='pending', lease_owner='', lease_expires_at=NULL, - available_at=? WHERE status='leased' AND lease_expires_at <= ?`, now, now) + now = now.UTC() + result, err := r.db.ExecContext(ctx, `UPDATE queue_jobs SET + status=CASE WHEN attempts >= max_attempts THEN 'failed' ELSE 'pending' END, + lease_owner='', lease_expires_at=NULL, + completed_at=CASE WHEN attempts >= max_attempts THEN ? ELSE NULL END, + last_error=CASE WHEN attempts >= max_attempts + THEN 'queue lease expired after maximum attempts; explicit recovery required' + ELSE 'queue lease expired; handler ownership lost' END, + available_at=? WHERE status='leased' AND lease_expires_at <= ?`, now, now, now) if err != nil { return 0, err } diff --git a/internal/infrastructure/database/storage/repository.go b/internal/infrastructure/database/storage/repository.go index adc10ec2..adc785f1 100644 --- a/internal/infrastructure/database/storage/repository.go +++ b/internal/infrastructure/database/storage/repository.go @@ -18,6 +18,35 @@ type Repository struct{ db *sql.DB } var sha256Digest = regexp.MustCompile(`^sha256:[a-fA-F0-9]{64}$`) func New(db *sql.DB) *Repository { return &Repository{db} } + +func (r *Repository) EnsureCloudWorkspaceStorage(ctx context.Context, environmentID string, storage domain.StorageResource) error { + if !strings.HasPrefix(storage.ID, "cloud-workspace:") || storage.Type != domain.StorageSSH || storage.Endpoint != "/akoflow/workspace" { + return fmt.Errorf("invalid concrete cloud workspace storage") + } + configuration, err := json.Marshal(map[string]any{"browseRoots": storage.BrowseRoots}) + if err != nil { + return err + } + metadata, err := json.Marshal(storage.Metadata) + if err != nil { + return err + } + result, err := r.db.ExecContext(ctx, `INSERT INTO storage_resources + (id,environment_version_id,name,type,endpoint,configuration,metadata) + SELECT ?,id,?,'ssh-filesystem',?,?,? FROM environment_versions + WHERE environment_id=? ORDER BY (status='published') DESC,version DESC LIMIT 1 + ON CONFLICT(id) DO NOTHING`, storage.ID, storage.ID, storage.Endpoint, string(configuration), string(metadata), environmentID) + if err != nil { + return err + } + if count, _ := result.RowsAffected(); count == 0 { + existing, err := r.FindStorage(ctx, storage.ID) + if err != nil || existing == nil || existing.Type != storage.Type || existing.Endpoint != storage.Endpoint || existing.Metadata["cloudInstanceId"] != storage.Metadata["cloudInstanceId"] { + return fmt.Errorf("cloud storage identity conflict or missing environment version") + } + } + return nil +} func (r *Repository) ListEnvironmentStorages(ctx context.Context, environmentID string) ([]domain.StorageResource, error) { const query = `SELECT s.id,s.environment_version_id,s.name,s.type,s.endpoint, s.capacity_bytes,s.shared,s.read_only,s.configuration,s.credential_reference,s.metadata diff --git a/internal/infrastructure/database/storage/repository_test.go b/internal/infrastructure/database/storage/repository_test.go index dbcd0598..6630390e 100644 --- a/internal/infrastructure/database/storage/repository_test.go +++ b/internal/infrastructure/database/storage/repository_test.go @@ -60,6 +60,21 @@ func TestRepositoryListsFindsAndScansStorageConfiguration(t *testing.T) { } } +func TestConcreteCloudWorkspaceStoragePersistsDownloadForeignKey(t *testing.T) { + r := testRepository(t) + ctx := context.Background() + storage := domain.StorageResource{ID: "cloud-workspace:vm", Type: domain.StorageSSH, Endpoint: "/akoflow/workspace", + BrowseRoots: []domain.StorageBrowseRoot{{Path: "/akoflow/workspace"}}, Metadata: map[string]any{"cloudInstanceId": "vm"}} + for i := 0; i < 2; i++ { + if err := r.EnsureCloudWorkspaceStorage(ctx, "environment", storage); err != nil { + t.Fatal(err) + } + } + if err := r.SaveDownload(ctx, domain.DownloadRun{ID: "archive", StorageID: storage.ID, Path: "runs/run", Status: domain.DownloadQueued, Strategy: "archive", CreatedAt: time.Now(), UpdatedAt: time.Now()}); err != nil { + t.Fatalf("concrete storage download foreign key: %v", err) + } +} + func TestRepositoryPersistsDownloadLifecycle(t *testing.T) { repository, ctx := testRepository(t), context.Background() now := time.Now().UTC().Truncate(time.Second) diff --git a/internal/infrastructure/transfer/endpoint_resolver.go b/internal/infrastructure/transfer/endpoint_resolver.go index ef447eaf..e09c0ab5 100644 --- a/internal/infrastructure/transfer/endpoint_resolver.go +++ b/internal/infrastructure/transfer/endpoint_resolver.go @@ -19,7 +19,9 @@ type EnvironmentEndpointResolver struct { func (resolver EnvironmentEndpointResolver) ResolveTransferEndpoint(ctx context.Context, location domain.TransferLocation) (domain.TransferEndpoint, error) { endpoint := domain.TransferEndpoint{ - URI: location.URI, ResourceID: location.ResourceID, EnvironmentID: location.EnvironmentID, + WorkspaceSnapshotID: location.WorkspaceSnapshotID, + WorkspaceSnapshotEntries: append([]domain.WorkspaceEntry(nil), location.WorkspaceSnapshotEntries...), + URI: location.URI, ResourceID: location.ResourceID, EnvironmentID: location.EnvironmentID, RuntimeID: location.RuntimeID, ConnectionID: location.ConnectionID, CloudInstanceID: location.CloudInstanceID, NetworkDomain: location.NetworkDomain, } @@ -104,6 +106,9 @@ func (resolver EnvironmentEndpointResolver) resolveConnectionEndpoint(endpoint d knownHosts = "storage/credentials/ssh/known_hosts" } query.Set("knownHostsFile", knownHosts) + if strict, _ := connection.Configuration["strictHostKeyChecking"].(bool); strict { + query.Set("strictHostKeyChecking", "true") + } if value, _ := connection.Configuration["acceptNewHostKey"].(bool); value { query.Set("acceptNewHostKey", "true") } @@ -158,13 +163,17 @@ func (resolver EnvironmentEndpointResolver) cloudConnection( address = selected.PrivateAddress } networkDomain, _ := selected.TerraformOutput["network_domain"].(string) + alias, knownHosts, err := selected.VerifiedSSHTrust() + if err != nil { + return nil, fmt.Errorf("cloud workspace SSH trust: %w", err) + } return &domain.EnvironmentConnection{ ID: connection.ID + "-" + selected.ID, EnvironmentID: connection.EnvironmentID, Name: selected.Name, Type: domain.ConnectionSSH, Endpoint: address, Username: selected.SSHUsername, CredentialRef: selected.SSHCredentialRef, Configuration: map[string]any{ - "port": 22, "acceptNewHostKey": true, - "hostKeyAlias": selected.ID, "networkDomain": networkDomain, + "port": 22, "acceptNewHostKey": false, "strictHostKeyChecking": true, + "hostKeyAlias": alias, "knownHostsFile": knownHosts, "networkDomain": networkDomain, "directAddress": selected.PrivateAddress, }, }, nil diff --git a/internal/infrastructure/transfer/endpoint_resolver_test.go b/internal/infrastructure/transfer/endpoint_resolver_test.go index bf7e7853..a357b585 100644 --- a/internal/infrastructure/transfer/endpoint_resolver_test.go +++ b/internal/infrastructure/transfer/endpoint_resolver_test.go @@ -6,9 +6,59 @@ import ( "strings" "testing" + "github.com/UFFeScience/akoflow/internal/application/ports" "github.com/UFFeScience/akoflow/internal/domain" ) +type verifiedCloudStore struct { + ports.CloudConfigurationStore + instance domain.CloudProvisionedInstance +} + +func (s verifiedCloudStore) ListProvisionedInstances(context.Context, string) ([]domain.CloudProvisionedInstance, error) { + return []domain.CloudProvisionedInstance{s.instance}, nil +} + +func TestCloudWorkspaceEndpointUsesVerifiedGenerationTrust(t *testing.T) { + instance := domain.CloudProvisionedInstance{ID: "vm", Status: "ready", CapacityTargetID: "target", ProviderID: "generation", PublicAddress: "cloud.example", TerraformOutput: map[string]any{ + "verifiedHostGeneration": "generation", "verifiedHostKeyAlias": "verified-alias", "verifiedKnownHostsFile": "/verified/known_hosts", + }} + connection := domain.EnvironmentConnection{ID: "cloud", Type: domain.ConnectionCloud} + location := domain.TransferLocation{URI: "file:///workspace?connectionId=cloud", ResourceID: "target", CloudInstanceID: "vm", + WorkspaceSnapshotID: "sealed-manifest", WorkspaceSnapshotEntries: []domain.WorkspaceEntry{{Path: "input.fits", SizeBytes: 10, Digest: "sha256:content"}}} + resolve := func() (domain.TransferEndpoint, error) { + return (EnvironmentEndpointResolver{Connections: connectionStoreStub{connection: connection}, Cloud: verifiedCloudStore{instance: instance}}).ResolveTransferEndpoint(context.Background(), location) + } + endpoint, err := resolve() + if err != nil { + t.Fatal(err) + } + if endpoint.WorkspaceSnapshotID != location.WorkspaceSnapshotID || len(endpoint.WorkspaceSnapshotEntries) != 1 { + t.Fatal("generation resolution discarded sealed snapshot identity") + } + endpoint.WorkspaceSnapshotEntries[0].Path = "changed" + if location.WorkspaceSnapshotEntries[0].Path != "input.fits" { + t.Fatal("resolved endpoint aliases producer manifest entries") + } + args := strings.Join(sshArgs(endpoint), " ") + for _, want := range []string{"StrictHostKeyChecking=yes", "HostKeyAlgorithms=ssh-ed25519", "HostKeyAlias=verified-alias", "UserKnownHostsFile=/verified/known_hosts"} { + if !strings.Contains(args, want) { + t.Fatalf("missing %s in %s", want, args) + } + } + if strings.Contains(args, "accept-new") { + t.Fatal("must not enroll via SSH") + } + instance.ProviderID = "replaced" + if _, err := resolve(); err == nil { + t.Fatal("stale generation accepted") + } + instance.TerraformOutput = nil + if _, err := resolve(); err == nil { + t.Fatal("missing verification accepted") + } +} + type connectionStoreStub struct { connection domain.EnvironmentConnection err error diff --git a/internal/infrastructure/transfer/peer_snapshot_build.go b/internal/infrastructure/transfer/peer_snapshot_build.go new file mode 100644 index 00000000..6632da16 --- /dev/null +++ b/internal/infrastructure/transfer/peer_snapshot_build.go @@ -0,0 +1,178 @@ +package transfer + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "math" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func peerSnapshotKey(source domain.TransferEndpoint) string { + if !peerArchiveDigest.MatchString(source.WorkspaceSnapshotID) || source.CloudInstanceID == "" || len(source.WorkspaceSnapshotEntries) == 0 { + return "" + } + // Include verified connection URI (host alias, known-host generation and + // workspace path), not a capacity target or an address alone. + manifest, err := json.Marshal(source.WorkspaceSnapshotEntries) + if err != nil { + return "" + } + digest := sha256.Sum256([]byte(source.CloudInstanceID + "\x00" + source.ConnectionID + "\x00" + source.URI + "\x00" + source.WorkspaceSnapshotID + "\x00" + string(manifest))) + return fmt.Sprintf("%x", digest) +} + +func verifyPeerSnapshotSource(ctx context.Context, source domain.TransferEndpoint) error { + host, origin, err := sshTarget(source, "") + if err != nil { + return err + } + manifest, err := json.Marshal(source.WorkspaceSnapshotEntries) + if err != nil { + return err + } + command := "python3 -c " + shell(peerSnapshotManifestScript) + " " + shell(origin) + " " + shell(base64.StdEncoding.EncodeToString(manifest)) + if _, err := runSSHCombinedOutput(ctx, source, append(sshArgs(source), host, command), nil); err != nil { + return fmt.Errorf("reused peer snapshot source differs from sealed manifest: %w", err) + } + return nil +} + +func buildPeerSnapshot(ctx context.Context, source domain.TransferEndpoint) (_ *peerSnapshot, err error) { + host, origin, err := sshTarget(source, "") + if err != nil { + return nil, err + } + var sshWait, commandSeconds float64 + output, err := runSSHProtocolMeasured(ctx, source, append(sshArgs(source), host, "mktemp -d /tmp/akoflow-peer-snapshot-XXXXXX"), &sshWait, &commandSeconds) + if err != nil { + return nil, err + } + root := strings.TrimSpace(string(output)) + if !filepath.IsAbs(root) || filepath.Dir(root) != "/tmp" || !strings.HasPrefix(filepath.Base(root), "akoflow-peer-snapshot-") { + return nil, fmt.Errorf("invalid peer snapshot directory") + } + cleanup := func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + _, _ = runSSHCombinedOutput(cleanupCtx, source, append(sshArgs(source), host, "rm -rf -- "+shell(root)), nil) + } + defer func() { + if err != nil { + cleanup() + } + }() + verify := "" + if peerSnapshotKey(source) != "" { + manifest, marshalErr := json.Marshal(source.WorkspaceSnapshotEntries) + if marshalErr != nil { + return nil, marshalErr + } + verify = "python3 -c " + shell(peerSnapshotManifestScript) + " " + shell(origin) + " " + shell(base64.StdEncoding.EncodeToString(manifest)) + "; " + } + archive := filepath.Join(root, "workspace.tar.gz") + command := "set -eu; umask 077; " + verify + "bad=$(find " + shell(origin) + " ! -type d ! -type f -print -quit); test -z \"$bad\"; " + + "bash -o pipefail -c " + shell("tar --hard-dereference -C \"$1\" -cf - . | gzip -1 > \"$2\"") + " archive " + shell(origin) + " " + shell(archive) + "; " + verify + + "sha256sum -- " + shell(archive) + "; wc -c < " + shell(archive) + command = "python3 -c " + shell(peerSnapshotTimingScript) + " " + shell(command) + metadata, err := runSSHProtocolMeasured(ctx, source, append(sshArgs(source), host, command), &sshWait, &commandSeconds) + if err != nil { + return nil, fmt.Errorf("build sealed peer snapshot: %w", err) + } + lines := strings.Split(strings.TrimSpace(string(metadata)), "\n") + if (len(lines) != 2 && len(lines) != 3) || len(strings.Fields(lines[0])) < 1 { + return nil, fmt.Errorf("invalid peer snapshot metadata") + } + digest := strings.Fields(lines[0])[0] + size, parseErr := strconv.ParseInt(strings.TrimSpace(lines[1]), 10, 64) + if parseErr != nil || size <= 0 || !peerArchiveDigest.MatchString(digest) { + return nil, fmt.Errorf("invalid peer snapshot digest or size") + } + var service *peerSnapshotServiceTiming + if len(lines) == 3 { + service, err = parsePeerSnapshotServiceTiming(lines[2]) + if err != nil { + return nil, err + } + } + return &peerSnapshot{root: root, digest: digest, size: size, cleanup: cleanup, buildSSHWaitSeconds: sshWait, buildCommandSeconds: commandSeconds, serviceTiming: service}, nil +} + +type peerSnapshotServiceTiming struct { + WallSeconds float64 `json:"wallSeconds"` + CPUSeconds float64 `json:"cpuSeconds"` +} + +func parsePeerSnapshotServiceTiming(line string) (*peerSnapshotServiceTiming, error) { + var timing struct { + WallSeconds *float64 `json:"wallSeconds"` + CPUSeconds *float64 `json:"cpuSeconds"` + } + if err := json.Unmarshal([]byte(line), &timing); err != nil || timing.WallSeconds == nil || timing.CPUSeconds == nil { + return nil, fmt.Errorf("invalid peer snapshot service timing") + } + for _, value := range []float64{*timing.WallSeconds, *timing.CPUSeconds} { + if value < 0 || math.IsNaN(value) || math.IsInf(value, 0) { + return nil, fmt.Errorf("invalid peer snapshot service timing") + } + } + return &peerSnapshotServiceTiming{WallSeconds: *timing.WallSeconds, CPUSeconds: *timing.CPUSeconds}, nil +} + +// Measured on the source: excludes local SSH admission and handshake. Wall +// still includes remote scheduling and IO contention; CPU is child-process +// user + system time, not a claim that the source was exclusively occupied. +const peerSnapshotTimingScript = ` +import json, os, subprocess, sys, time +before = os.times() +started = time.monotonic() +result = subprocess.run(['sh', '-c', sys.argv[1]]) +elapsed = time.monotonic() - started +after = os.times() +if result.returncode: + sys.exit(result.returncode) +print(json.dumps({'wallSeconds': elapsed, 'cpuSeconds': after.children_user + after.children_system - before.children_user - before.children_system}), flush=True) +` + +// Validate the entire file set before AND after snapshot creation. A sealed +// manifest is not permission to reuse bytes from a workspace that has mutated. +const peerSnapshotManifestScript = ` +import os, sys, json, base64, hashlib, stat +root = sys.argv[1] +entries = json.loads(base64.b64decode(sys.argv[2], validate=True)) +expected = {} +for entry in entries: + name = entry['path'] + if not name or name == '.' or name.startswith('/') or os.path.normpath(name) != name or name.startswith('../') or name in expected: + raise RuntimeError('invalid sealed snapshot path') + expected[name] = entry +actual = set() +for directory, dirs, files in os.walk(root, followlinks=False): + for name in dirs: + if not stat.S_ISDIR(os.lstat(os.path.join(directory, name)).st_mode): + raise RuntimeError('non-directory snapshot entry') + for name in files: + file = os.path.join(directory, name) + if not stat.S_ISREG(os.lstat(file).st_mode): + raise RuntimeError('non-regular snapshot entry') + rel = os.path.relpath(file, root) + entry = expected.get(rel) + if entry is None or os.path.getsize(file) != entry['sizeBytes']: + raise RuntimeError('sealed snapshot file set or size changed') + digest = hashlib.sha256() + with open(file, 'rb') as stream: + for block in iter(lambda: stream.read(1048576), b''): + digest.update(block) + if 'sha256:' + digest.hexdigest() != entry['digest']: + raise RuntimeError('sealed snapshot content changed') + actual.add(rel) +if actual != set(expected): + raise RuntimeError('sealed snapshot file missing') +` diff --git a/internal/infrastructure/transfer/peer_snapshot_build_test.go b/internal/infrastructure/transfer/peer_snapshot_build_test.go new file mode 100644 index 00000000..261fba22 --- /dev/null +++ b/internal/infrastructure/transfer/peer_snapshot_build_test.go @@ -0,0 +1,80 @@ +package transfer + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestPeerSnapshotManifestRejectsMutableOrIncompleteContents(t *testing.T) { + python, err := exec.LookPath("python3") + if err != nil { + t.Fatal(err) + } + for _, kind := range []string{"valid", "changed", "missing", "extra", "symlink"} { + t.Run(kind, func(t *testing.T) { + root := t.TempDir() + data := []byte("original") + entry := domain.WorkspaceEntry{Path: "file", SizeBytes: int64(len(data)), Digest: fmt.Sprintf("sha256:%x", sha256.Sum256(data))} + if kind == "changed" { + data = []byte("modified") + } + if kind != "missing" { + if err := os.WriteFile(filepath.Join(root, "file"), data, 0600); err != nil { + t.Fatal(err) + } + } + if kind == "extra" { + if err := os.WriteFile(filepath.Join(root, "extra"), []byte("x"), 0600); err != nil { + t.Fatal(err) + } + } + if kind == "symlink" { + if err := os.Symlink("file", filepath.Join(root, "link")); err != nil { + t.Fatal(err) + } + } + manifest, _ := json.Marshal([]domain.WorkspaceEntry{entry}) + output, err := exec.Command(python, "-c", peerSnapshotManifestScript, root, base64.StdEncoding.EncodeToString(manifest)).CombinedOutput() + if (kind == "valid") != (err == nil) { + t.Fatalf("validation %s: %v %s", kind, err, output) + } + }) + } +} + +func TestPeerSnapshotKeyInvalidatesGenerationPathAndManifest(t *testing.T) { + source := domain.TransferEndpoint{URI: "ssh://worker/workspace?hostKeyAlias=generation", ConnectionID: "ssh-vm", CloudInstanceID: "vm", WorkspaceSnapshotID: strings.Repeat("a", 64), WorkspaceSnapshotEntries: []domain.WorkspaceEntry{ + {Path: "file", Digest: "sha256:" + strings.Repeat("b", 64), SizeBytes: 1}, + }} + key := peerSnapshotKey(source) + if key == "" { + t.Fatal("complete snapshot has no identity") + } + for _, change := range []func(*domain.TransferEndpoint){ + func(s *domain.TransferEndpoint) { s.CloudInstanceID = "other" }, + func(s *domain.TransferEndpoint) { s.URI += "-other" }, + func(s *domain.TransferEndpoint) { s.WorkspaceSnapshotID = strings.Repeat("c", 64) }, + func(s *domain.TransferEndpoint) { + s.WorkspaceSnapshotEntries = []domain.WorkspaceEntry{{Path: "changed"}} + }, + } { + copy := source + change(©) + if peerSnapshotKey(copy) == key { + t.Fatal("changed identity reused") + } + } + source.WorkspaceSnapshotID = "" + if peerSnapshotKey(source) != "" { + t.Fatal("unknown seal cached") + } +} diff --git a/internal/infrastructure/transfer/peer_snapshot_cache.go b/internal/infrastructure/transfer/peer_snapshot_cache.go new file mode 100644 index 00000000..8cfb90ab --- /dev/null +++ b/internal/infrastructure/transfer/peer_snapshot_cache.go @@ -0,0 +1,136 @@ +package transfer + +import ( + "context" + "fmt" + "sync" + "time" +) + +type peerSnapshot struct { + root, digest string + size int64 + cleanup func() + buildWallSeconds float64 + buildSSHWaitSeconds float64 + buildCommandSeconds float64 + serviceTiming *peerSnapshotServiceTiming +} + +type peerSnapshotEntry struct { + done chan struct{} + cancel context.CancelFunc + refs int + snapshot *peerSnapshot + err error + timer *time.Timer +} + +// The cache owns remote scratch directories, never producer workspaces. Active +// readers prevent eviction. Idle entries expire; failed builds are not cached. +type peerSnapshotCache struct { + mu sync.Mutex + entries map[string]*peerSnapshotEntry + idleTTL time.Duration +} + +var sharedPeerSnapshots = peerSnapshotCache{entries: make(map[string]*peerSnapshotEntry), idleTTL: time.Minute} + +func (c *peerSnapshotCache) acquire(ctx context.Context, key string, build func(context.Context) (*peerSnapshot, error)) (*peerSnapshot, bool, func(), error) { + c.mu.Lock() + if c.entries == nil { + c.entries = make(map[string]*peerSnapshotEntry) + } + e := c.entries[key] + reused := e != nil && key != "" + if key == "" { + e = nil + } + if e == nil { + if key != "" && len(c.entries) >= 64 { + c.mu.Unlock() + return nil, false, nil, fmt.Errorf("peer snapshot cache capacity exhausted") + } + buildCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Minute) + e = &peerSnapshotEntry{done: make(chan struct{}), cancel: cancel} + if key != "" { + c.entries[key] = e + } + go func() { + started := time.Now() + snapshot, err := build(buildCtx) + if snapshot != nil { + snapshot.buildWallSeconds = time.Since(started).Seconds() + } + cancel() + c.mu.Lock() + e.snapshot, e.err = snapshot, err + close(e.done) + if err != nil && c.entries[key] == e { + delete(c.entries, key) + } + orphaned := e.refs == 0 + c.mu.Unlock() + if orphaned && snapshot != nil && snapshot.cleanup != nil { + snapshot.cleanup() + } + }() + } + e.refs++ + if e.timer != nil { + e.timer.Stop() + e.timer = nil + } + c.mu.Unlock() + var once sync.Once + release := func() { once.Do(func() { c.release(key, e) }) } + select { + case <-ctx.Done(): + release() + return nil, reused, nil, ctx.Err() + case <-e.done: + if e.err != nil { + release() + return nil, reused, nil, e.err + } + return e.snapshot, reused, release, nil + } +} + +func (c *peerSnapshotCache) release(key string, e *peerSnapshotEntry) { + c.mu.Lock() + e.refs-- + if e.refs != 0 { + c.mu.Unlock() + return + } + select { + case <-e.done: + if key != "" && e.err == nil && c.entries[key] == e { + e.timer = time.AfterFunc(c.idleTTL, func() { + c.mu.Lock() + if e.refs != 0 || c.entries[key] != e { + c.mu.Unlock() + return + } + delete(c.entries, key) + c.mu.Unlock() + if e.snapshot != nil && e.snapshot.cleanup != nil { + e.snapshot.cleanup() + } + }) + c.mu.Unlock() + return + } + c.mu.Unlock() + if e.snapshot != nil && e.snapshot.cleanup != nil { + e.snapshot.cleanup() + } + default: + if c.entries[key] == e { + delete(c.entries, key) + } + e.cancel() + c.mu.Unlock() + } +} diff --git a/internal/infrastructure/transfer/peer_snapshot_cache_test.go b/internal/infrastructure/transfer/peer_snapshot_cache_test.go new file mode 100644 index 00000000..e621bf11 --- /dev/null +++ b/internal/infrastructure/transfer/peer_snapshot_cache_test.go @@ -0,0 +1,151 @@ +package transfer + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestPeerSnapshotCacheSharesBuildAndProtectsReaders(t *testing.T) { + c := &peerSnapshotCache{idleTTL: 10 * time.Millisecond} + var builds atomic.Int32 + cleaned := make(chan struct{}, 1) + build := func(context.Context) (*peerSnapshot, error) { + builds.Add(1) + return &peerSnapshot{root: "owned-scratch", cleanup: func() { cleaned <- struct{}{} }}, nil + } + const readers = 8 + releases := make(chan func(), readers) + var wg sync.WaitGroup + for i := 0; i < readers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + snapshot, _, release, err := c.acquire(context.Background(), "sealed-generation", build) + if err != nil || snapshot.root != "owned-scratch" { + t.Errorf("acquire: %v", err) + return + } + if snapshot.buildWallSeconds <= 0 { + t.Error("shared snapshot must publish measured build wall time before readers receive it") + } + releases <- release + }() + } + wg.Wait() + if builds.Load() != 1 { + t.Fatalf("duplicate builds: %d", builds.Load()) + } + for i := 0; i < readers-1; i++ { + (<-releases)() + } + select { + case <-cleaned: + t.Fatal("deleted while reader active") + case <-time.After(30 * time.Millisecond): + } + last := <-releases + last() + last() // Reader release must be idempotent. + select { + case <-cleaned: + case <-time.After(time.Second): + t.Fatal("idle scratch not cleaned") + } + if len(c.entries) != 0 { + t.Fatal("expired cache entry retained") + } +} + +func TestPeerSnapshotCacheDoesNotCacheFailuresOrUnknownSeals(t *testing.T) { + c := &peerSnapshotCache{idleTTL: time.Millisecond} + var builds int + build := func(context.Context) (*peerSnapshot, error) { builds++; return nil, errors.New("pack failed") } + for i := 0; i < 2; i++ { + if _, _, _, err := c.acquire(context.Background(), "seal", build); err == nil { + t.Fatal("failure suppressed") + } + } + if builds != 2 { + t.Fatal("failed pack was reused") + } + var cleaned atomic.Int32 + for i := 0; i < 2; i++ { + _, reused, release, err := c.acquire(context.Background(), "", func(context.Context) (*peerSnapshot, error) { + return &peerSnapshot{cleanup: func() { cleaned.Add(1) }}, nil + }) + if err != nil || reused { + t.Fatal("unknown seal reused") + } + release() + } + if cleaned.Load() != 2 { + t.Fatal("uncached scratch not released") + } +} + +func TestPeerSnapshotCacheCancelledReaderDoesNotCancelAnother(t *testing.T) { + c := &peerSnapshotCache{idleTTL: time.Millisecond} + ctx, cancel := context.WithCancel(context.Background()) + started := make(chan struct{}) + allowBuild := make(chan struct{}) + cleaned := make(chan struct{}, 1) + build := func(buildCtx context.Context) (*peerSnapshot, error) { + close(started) + select { + case <-allowBuild: + return &peerSnapshot{cleanup: func() { cleaned <- struct{}{} }}, nil + case <-buildCtx.Done(): + return nil, buildCtx.Err() + } + } + first := make(chan error, 1) + go func() { + _, _, release, err := c.acquire(ctx, "seal", build) + if release != nil { + release() + } + first <- err + }() + <-started + second := make(chan error, 1) + go func() { + _, reused, release, err := c.acquire(context.Background(), "seal", build) + if err == nil && !reused { + err = errors.New("did not share build") + } + if release != nil { + release() + } + second <- err + }() + deadline := time.Now().Add(time.Second) + for { + c.mu.Lock() + refs := c.entries["seal"].refs + c.mu.Unlock() + if refs == 2 { + break + } + if time.Now().After(deadline) { + t.Fatal("second reader did not join") + } + time.Sleep(time.Millisecond) + } + cancel() + if err := <-first; !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled reader: %v", err) + } + close(allowBuild) + if err := <-second; err != nil { + t.Fatalf("remaining reader: %v", err) + } + select { + case <-cleaned: + case <-time.After(time.Second): + t.Fatal("cleanup missing") + } +} diff --git a/internal/infrastructure/transfer/peer_snapshot_timing_test.go b/internal/infrastructure/transfer/peer_snapshot_timing_test.go new file mode 100644 index 00000000..2302bc21 --- /dev/null +++ b/internal/infrastructure/transfer/peer_snapshot_timing_test.go @@ -0,0 +1,58 @@ +package transfer + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestPeerProtocolCommandTimeExcludesAdmissionWait(t *testing.T) { + bin := t.TempDir() + if err := os.WriteFile(filepath.Join(bin, "ssh"), []byte("#!/bin/sh\nsleep 0.05\nprintf protocol\n"), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + var wait, execution float64 + output, err := runSSHProtocolMeasured(context.Background(), domain.TransferEndpoint{}, nil, &wait, &execution) + if err != nil || string(output) != "protocol" || execution < .04 || wait >= execution { + t.Fatalf("separate queue/command evidence: %q %g %g %v", output, wait, execution, err) + } +} + +func TestPeerSnapshotSourceServiceTiming(t *testing.T) { + python, err := exec.LookPath("python3") + if err != nil { + t.Skip("python3 unavailable") + } + output, err := exec.Command(python, "-c", peerSnapshotTimingScript, "sleep 0.05; printf 'archive-metadata\\n'").CombinedOutput() + if err != nil { + t.Fatalf("source timing: %s: %v", output, err) + } + lines := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(lines) != 2 || lines[0] != "archive-metadata" { + t.Fatalf("protocol changed: %q", output) + } + timing, err := parsePeerSnapshotServiceTiming(lines[1]) + if err != nil || timing.WallSeconds < .04 || timing.CPUSeconds >= timing.WallSeconds { + t.Fatalf("sleep must not become CPU work: %+v %v", timing, err) + } + if _, err := exec.Command(python, "-c", peerSnapshotTimingScript, "exit 7").Output(); err == nil { + t.Fatal("failed build must not become a valid measurement") + } +} + +func TestPeerSnapshotServiceTimingRejectsIncompleteEvidence(t *testing.T) { + for _, input := range []string{`{}`, `{"wallSeconds":1}`, `{"wallSeconds":-1,"cpuSeconds":0}`, `{"wallSeconds":1,"cpuSeconds":-1}`, `{"wallSeconds":1e999,"cpuSeconds":0}`} { + if _, err := parsePeerSnapshotServiceTiming(input); err == nil { + t.Fatalf("accepted invalid evidence %s", input) + } + } + if timing, err := parsePeerSnapshotServiceTiming(`{"wallSeconds":0,"cpuSeconds":0}`); err != nil || timing == nil { + t.Fatalf("zero measurement must differ from absent legacy evidence: %v", err) + } +} diff --git a/internal/infrastructure/transfer/peer_sshd_test.go b/internal/infrastructure/transfer/peer_sshd_test.go new file mode 100644 index 00000000..f523c0e7 --- /dev/null +++ b/internal/infrastructure/transfer/peer_sshd_test.go @@ -0,0 +1,146 @@ +package transfer + +import ( + "fmt" + "net" + "os" + "os/exec" + "os/user" + "path/filepath" + "strings" + "testing" + "time" +) + +// Run in the Linux worker-compatible container: this exercises sshd's actual +// authorized_keys parser, not only Go's syntax parser. No real user key file +// is touched and the server is bound exclusively to loopback. +func TestScopedPeerCredentialAuthenticatesWithOpenSSHServer(t *testing.T) { + sshd, err := exec.LookPath("sshd") + if err != nil || os.Geteuid() != 0 { + t.Skip("isolated sshd integration requires Linux root container with openssh-server") + } + if err := os.MkdirAll("/run/sshd", 0755); err != nil { + t.Fatal(err) + } + root, err := os.MkdirTemp("/root", ".akoflow-sshd-test-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(root) }) + write := peerSSHFixtureWriter(t, root) + hostPublic, hostPrivate, err := ephemeralSSHKey() + if err != nil { + t.Fatal(err) + } + clientPublic, clientPrivate, err := ephemeralSSHKey() + if err != nil { + t.Fatal(err) + } + hostKey := write("host-key", hostPrivate) + clientKey := write("client-key", clientPrivate) + authorized, err := scopedDirectAuthorizedKey(clientPublic, "integration-token", "printf '%s' AUTHORIZED", time.Now().Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + authorizedPath := write("authorized_keys", []byte(authorized+"\n")) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := listener.Addr().(*net.TCPAddr).Port + listener.Close() + config := fmt.Sprintf( + "ListenAddress 127.0.0.1\nPort %d\nHostKey %s\nAuthorizedKeysFile %s\nPidFile %s\nUsePAM yes\nPasswordAuthentication no\nKbdInteractiveAuthentication no\nPermitRootLogin yes\nStrictModes yes\nLogLevel VERBOSE\n", + port, + hostKey, + authorizedPath, + filepath.Join(root, "pid"), + ) + server := exec.Command(sshd, "-D", "-e", "-f", write("sshd_config", []byte(config))) + log, err := os.Create(filepath.Join(root, "server-log")) + if err != nil { + t.Fatal(err) + } + defer log.Close() + server.Stderr = log + if err := server.Start(); err != nil { + t.Fatal(err) + } + defer func() { server.Process.Kill(); server.Wait() }() + address := fmt.Sprintf("127.0.0.1:%d", port) + awaitLoopbackSSHServer(t, address) + knownHosts := write("known_hosts", []byte("integration-host "+strings.TrimSpace(string(hostPublic))+"\n")) + current, err := user.Current() + if err != nil { + t.Fatal(err) + } + args := scopedLoopbackSSHArguments(port, clientKey, knownHosts, current.Username) + output, err := exec.Command("ssh", args...).CombinedOutput() + if err != nil || string(output) != "AUTHORIZED" { + serverLog, _ := os.ReadFile(filepath.Join(root, "server-log")) + t.Fatalf("scoped authorization rejected or command not forced: %v %s; server: %s", err, output, serverLog) + } + expired, err := scopedDirectAuthorizedKey(clientPublic, "integration-token", "printf '%s' AUTHORIZED", time.Now().Add(-time.Minute)) + if err != nil { + t.Fatal(err) + } + write("authorized_keys", []byte(expired+"\n")) + output, err = exec.Command("ssh", args...).CombinedOutput() + if err == nil || !strings.Contains(string(output), "Permission denied") { + t.Fatalf("expired authorization must be rejected: %v %s", err, output) + } +} + +func peerSSHFixtureWriter(t *testing.T, root string) func(string, []byte) string { + t.Helper() + return func(name string, content []byte) string { + t.Helper() + path := filepath.Join(root, name) + if err := os.WriteFile(path, content, 0600); err != nil { + t.Fatal(err) + } + return path + } +} + +func awaitLoopbackSSHServer(t *testing.T, address string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + conn, err := net.DialTimeout("tcp", address, 100*time.Millisecond) + if err == nil { + conn.Close() + break + } + if time.Now().After(deadline) { + t.Fatal("isolated sshd did not start") + } + time.Sleep(10 * time.Millisecond) + } +} + +func scopedLoopbackSSHArguments(port int, clientKey, knownHosts, username string) []string { + return []string{ + "-F", + "/dev/null", + "-p", + fmt.Sprint(port), + "-i", + clientKey, + "-o", + "BatchMode=yes", + "-o", + "IdentitiesOnly=yes", + "-o", + "StrictHostKeyChecking=yes", + "-o", + "HostKeyAlias=integration-host", + "-o", + "UserKnownHostsFile=" + knownHosts, + "-o", + "ConnectTimeout=3", + username + "@127.0.0.1", + "printf '%s' UNAUTHORIZED", + } +} diff --git a/internal/infrastructure/transfer/rsync_ssh.go b/internal/infrastructure/transfer/rsync_ssh.go index e7512aaa..6b5d6a16 100644 --- a/internal/infrastructure/transfer/rsync_ssh.go +++ b/internal/infrastructure/transfer/rsync_ssh.go @@ -1,14 +1,15 @@ package transfer import ( + "bytes" "context" "crypto/ed25519" "crypto/rand" - "crypto/x509" "encoding/pem" "fmt" "io" "net/url" + "os" "os/exec" "path/filepath" "strconv" @@ -54,7 +55,7 @@ func sshTarget(e domain.TransferEndpoint, name string) (string, string, error) { return host, full, nil } func sshArgs(e domain.TransferEndpoint) []string { - args := []string{} + args := []string{"-o", "BatchMode=yes", "-o", "ConnectionAttempts=1", "-o", "ConnectTimeout=10"} identity, knownHosts, proxy, alias := "", "", "", "" extraOptions := e.Configuration["sshOptions"] port, forward := 0, false @@ -67,6 +68,9 @@ func sshArgs(e domain.TransferEndpoint) []string { } if uri, err := url.Parse(e.URI); err == nil && uri.Scheme == "ssh" && uri.Host != "" { query := uri.Query() + if strict, _ := strconv.ParseBool(query.Get("strictHostKeyChecking")); strict { + args = append(args, "-o", "HostKeyAlgorithms=ssh-ed25519") + } if queryIdentity := query.Get("identityFile"); queryIdentity != "" { identity = queryIdentity args = append(args, "-i", queryIdentity) @@ -104,6 +108,9 @@ func sshArgs(e domain.TransferEndpoint) []string { if connectionID == "" { connectionID = query.Get("connectionId") } + if connectionID == "" { + connectionID = e.ConnectionID + } multiplex, _, err := provider.SSHMultiplexArguments(provider.SSHSessionKey{ ConnectionID: connectionID, Username: username, Host: uri.Hostname(), Port: port, IdentityFile: identity, ProxyCommand: proxy, KnownHostsFile: knownHosts, @@ -121,7 +128,9 @@ func (RsyncSSH) Exists(ctx context.Context, e domain.TransferEndpoint, name stri return false, err } args := append(sshArgs(e), host, "test -f "+shell(path)) - output, err := runSSHCombinedOutput(ctx, e, args, nil) + output, err := retrySSHRead(ctx, time.Second, func() ([]byte, error) { + return runSSHCombinedOutput(ctx, e, args, nil) + }) if err == nil { return true, nil } @@ -181,6 +190,26 @@ func waitSSHCommand(command *exec.Cmd) error { return nil } } + +// VerifyDigest avoids downloading an intact artifact just to validate its cache. +func (RsyncSSH) VerifyDigest(ctx context.Context, e domain.TransferEndpoint, name, expected string) (bool, error) { + host, path, err := sshTarget(e, name) + if err != nil { + return false, err + } + output, err := retrySSHRead(ctx, time.Second, func() ([]byte, error) { + return runSSHCombinedOutput(ctx, e, append(sshArgs(e), host, "sha256sum -- "+shell(path)), nil) + }) + if err != nil { + return false, fmt.Errorf("verify SSH artifact checksum: %w: %s", err, strings.TrimSpace(string(output))) + } + fields := strings.Fields(string(output)) + if len(fields) < 2 || len(fields[0]) != 64 { + return false, fmt.Errorf("invalid remote SHA-256 response") + } + return fields[0] == strings.TrimPrefix(expected, "sha256:"), nil +} + func (connector RsyncSSH) Put(ctx context.Context, e domain.TransferEndpoint, name string, input io.Reader, offset int64) error { host, path, err := sshTarget(e, name) if err != nil { @@ -211,7 +240,7 @@ func (connector RsyncSSH) Put(ctx context.Context, e domain.TransferEndpoint, na closeErr := stdin.Close() waitErr := cmd.Wait() if copyErr != nil { - return fmt.Errorf("stream to SSH staging: %w", copyErr) + return fmt.Errorf("stream to SSH staging: %w (SSH exit: %v): %s", copyErr, waitErr, strings.TrimSpace(stderr.String())) } if closeErr != nil { return fmt.Errorf("close SSH stream: %w", closeErr) @@ -231,8 +260,11 @@ func (RsyncSSH) Commit(ctx context.Context, e domain.TransferEndpoint, partial, return err } args := append(sshArgs(e), host, "mkdir -p -- "+shell(filepath.Dir(f))+" && mv -- "+shell(p)+" "+shell(f)) - _, err = runSSHCombinedOutput(ctx, e, args, nil) - return err + output, err := runSSHCombinedOutput(ctx, e, args, nil) + if err != nil { + return fmt.Errorf("commit SSH artifact: %w: %s", err, strings.TrimSpace(string(output))) + } + return nil } // TransferRoute keeps payload bytes on the runtime side. For different VMs a @@ -269,28 +301,31 @@ func (RsyncSSH) TransferRoute(ctx context.Context, strategy domain.TransferStrat } identity := destination.Configuration["directIdentityFile"] knownHosts := destination.Configuration["directKnownHostsFile"] + writeCommand := "mkdir -p -- " + shell(filepath.Dir(destinationPath)) + " && cat > " + shell(destinationPath) + if offset > 0 { + writeCommand = fmt.Sprintf("test $(wc -c < %s) -eq %d && cat >> %s", shell(destinationPath), offset, shell(destinationPath)) + } cleanup := func() {} if identity == "" || knownHosts == "" { - identity, knownHosts, cleanup, err = prepareDirectCredential(ctx, source, destination) + identity, knownHosts, cleanup, err = prepareDirectCredential(ctx, source, destination, writeCommand) if err != nil { return 0, err } defer cleanup() } - remoteSSH := []string{"ssh", "-i", shell(identity), "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes"} + remoteSSH := []string{"ssh", "-F", "/dev/null", "-i", shell(identity), "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes", "-o", "ForwardAgent=no", "-o", "ConnectTimeout=10", "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2"} remoteSSH = append(remoteSSH, "-o", shell("UserKnownHostsFile="+knownHosts), "-o", "StrictHostKeyChecking=yes") + if uri, parseErr := url.Parse(destination.URI); parseErr == nil && uri.Query().Get("strictHostKeyChecking") == "true" { + remoteSSH = append(remoteSSH, "-o", "HostKeyAlgorithms=ssh-ed25519", "-o", shell("HostKeyAlias="+uri.Query().Get("hostKeyAlias"))) + } if uri, parseErr := url.Parse(destination.URI); parseErr == nil && uri.Port() != "" { remoteSSH = append(remoteSSH, "-p", shell(uri.Port())) } - writeCommand := "mkdir -p -- " + shell(filepath.Dir(destinationPath)) + " && cat > " + shell(destinationPath) - if offset > 0 { - writeCommand = fmt.Sprintf("test $(wc -c < %s) -eq %d && cat >> %s", shell(destinationPath), offset, shell(destinationPath)) - } readCommand := "cat -- " + shell(sourcePath) if offset > 0 { readCommand = fmt.Sprintf("tail -c +%d -- %s", offset+1, shell(sourcePath)) } - remoteCommand = readCommand + " | " + strings.Join(remoteSSH, " ") + " " + shell(destinationHost) + " " + shell(writeCommand) + remoteCommand = "bash -o pipefail -c " + shell(readCommand+" | "+strings.Join(remoteSSH, " ")+" "+shell(destinationHost)+" "+shell(writeCommand)) output, runErr := runSSHCombinedOutput(ctx, source, append(sshArgs(source), sourceHost, remoteCommand), nil) if runErr != nil { return 0, fmt.Errorf("stream directly between cloud instances: %w: %s", runErr, strings.TrimSpace(string(output))) @@ -309,12 +344,16 @@ func destinationUserHost(endpoint domain.TransferEndpoint, address string) strin return address } -func prepareDirectCredential(ctx context.Context, source, destination domain.TransferEndpoint) (string, string, func(), error) { +func prepareDirectCredential(ctx context.Context, source, destination domain.TransferEndpoint, forcedCommand string) (string, string, func(), error) { publicKey, privateKey, err := ephemeralSSHKey() if err != nil { return "", "", func() {}, err } - token := fmt.Sprintf("akoflow-transfer-%d", time.Now().UnixNano()) + var nonce [16]byte + if _, err := rand.Read(nonce[:]); err != nil { + return "", "", func() {}, fmt.Errorf("generate transfer credential identity: %w", err) + } + token := fmt.Sprintf("akoflow-transfer-%x", nonce) identity := "/tmp/" + token knownHosts := identity + ".known_hosts" destinationHost, _, err := sshTarget(destination, "") @@ -325,15 +364,18 @@ func prepareDirectCredential(ctx context.Context, source, destination domain.Tra if err != nil { return "", "", func() {}, err } - authorizedLine := "no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-pty " + strings.TrimSpace(string(publicKey)) + " " + token - installDestination := "umask 077; mkdir -p ~/.ssh; touch ~/.ssh/authorized_keys; printf '%s\\n' " + shell(authorizedLine) + " >> ~/.ssh/authorized_keys" + authorizedLine, err := scopedDirectAuthorizedKey(publicKey, token, forcedCommand, time.Now().Add(30*time.Minute)) + if err != nil { + return "", "", func() {}, err + } + installDestination := "umask 077; mkdir -p ~/.ssh; touch ~/.ssh/authorized_keys; flock ~/.ssh/.akoflow-transfer-lock sh -c " + shell("printf '%s\\n' "+shell(authorizedLine)+" >> ~/.ssh/authorized_keys") if output, installErr := runSSHCombinedOutput(ctx, destination, append(sshArgs(destination), destinationHost, installDestination), nil); installErr != nil { return "", "", func() {}, fmt.Errorf("install temporary destination credential: %w: %s", installErr, strings.TrimSpace(string(output))) } cleanup := func() { cleanupCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() - removeDestination := "grep -v -- " + shell(token) + " ~/.ssh/authorized_keys > ~/.ssh/authorized_keys.akoflow && mv ~/.ssh/authorized_keys.akoflow ~/.ssh/authorized_keys" + removeDestination := directCredentialRemovalCommand(authorizedLine) _, _ = runSSHCombinedOutput(cleanupCtx, destination, append(sshArgs(destination), destinationHost, removeDestination), nil) _, _ = runSSHCombinedOutput(cleanupCtx, source, append(sshArgs(source), sourceHost, "rm -f -- "+shell(identity)+" "+shell(knownHosts)), nil) } @@ -351,6 +393,18 @@ func prepareDirectCredential(ctx context.Context, source, destination domain.Tra if port == "" { port = "22" } + if destinationURL.Query().Get("strictHostKeyChecking") == "true" { + pinned, readErr := os.ReadFile(destinationURL.Query().Get("knownHostsFile")) + if readErr != nil || len(pinned) == 0 { + cleanup() + return "", "", func() {}, fmt.Errorf("read verified direct destination host key: %v", readErr) + } + if writeErr := runSSHInput(ctx, source, sourceHost, "umask 077; cat > "+shell(knownHosts), pinned); writeErr != nil { + cleanup() + return "", "", func() {}, fmt.Errorf("install verified direct destination host key: %w", writeErr) + } + return identity, knownHosts, cleanup, nil + } scan := "ssh-keyscan -p " + shell(port) + " -- " + shell(directAddress) + " > " + shell(knownHosts) if output, scanErr := runSSHCombinedOutput(ctx, source, append(sshArgs(source), sourceHost, scan), nil); scanErr != nil { cleanup() @@ -359,6 +413,33 @@ func prepareDirectCredential(ctx context.Context, source, destination domain.Tra return identity, knownHosts, cleanup, nil } +func scopedDirectAuthorizedKey(publicKey []byte, token, forcedCommand string, expires time.Time) (string, error) { + key, _, _, rest, err := ssh.ParseAuthorizedKey(publicKey) + if err != nil || key.Type() != "ssh-ed25519" || len( + strings.TrimSpace(string(rest)), + ) != 0 || strings.TrimSpace( + forcedCommand, + ) == "" || token == "" || strings.ContainsAny( + forcedCommand, + "\r\n\x00", + ) || strings.ContainsAny( + token, + " \t\r\n\x00", + ) { + return "", fmt.Errorf("invalid scoped direct transfer authorization") + } + return "restrict,command=" + strconv.Quote(forcedCommand) + ",expiry-time=" + strconv.Quote(expires.UTC().Format("20060102150405Z")) + " " + strings.TrimSpace(string(ssh.MarshalAuthorizedKey(key))) + " " + token, nil +} + +// Serialize append/revocation across simultaneous transfers. Remove the exact +// authorization, not every line containing a substring, and keep user keys. +func directCredentialRemovalCommand(authorizedLine string) string { + remove := "umask 077; tmp=$(mktemp ~/.ssh/.akoflow-keys-XXXXXX); trap 'rm -f -- \"$tmp\"' EXIT; " + + "grep -F -x -v -- " + shell(authorizedLine) + " ~/.ssh/authorized_keys > \"$tmp\" || test \"$?\" -eq 1; " + + "chmod 600 \"$tmp\"; mv -- \"$tmp\" ~/.ssh/authorized_keys" + return "flock ~/.ssh/.akoflow-transfer-lock sh -c " + shell("set -eu; "+remove) +} + func ephemeralSSHKey() ([]byte, []byte, error) { public, private, err := ed25519.GenerateKey(rand.Reader) if err != nil { @@ -368,11 +449,11 @@ func ephemeralSSHKey() ([]byte, []byte, error) { if err != nil { return nil, nil, fmt.Errorf("encode temporary transfer public key: %w", err) } - encodedPrivate, err := x509.MarshalPKCS8PrivateKey(private) + encodedPrivate, err := ssh.MarshalPrivateKey(private, "temporary-workspace-transfer") if err != nil { return nil, nil, fmt.Errorf("encode temporary transfer private key: %w", err) } - return ssh.MarshalAuthorizedKey(sshPublic), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: encodedPrivate}), nil + return ssh.MarshalAuthorizedKey(sshPublic), pem.EncodeToMemory(encodedPrivate), nil } func runSSHInput(ctx context.Context, endpoint domain.TransferEndpoint, host, command string, input []byte) error { @@ -384,11 +465,13 @@ func runSSHInput(ctx context.Context, endpoint domain.TransferEndpoint, host, co } func queuedSSHCommand(ctx context.Context, endpoint domain.TransferEndpoint, args []string) (*exec.Cmd, func(), error) { - release, err := provider.AcquireSSHChannel(ctx, sshControlPath(args)) + release, err := provider.AcquireSSHBulkChannel(ctx, sshControlPath(args)) if err != nil { return nil, nil, fmt.Errorf("wait for SSH channel: %w", err) } - return exec.CommandContext(ctx, "ssh", args...), release, nil + command := exec.CommandContext(ctx, "ssh", args...) + command.WaitDelay = 5 * time.Second + return command, release, nil } func runSSHCombinedOutput(ctx context.Context, endpoint domain.TransferEndpoint, args []string, input io.Reader) ([]byte, error) { @@ -401,6 +484,67 @@ func runSSHCombinedOutput(ctx context.Context, endpoint domain.TransferEndpoint, return command.CombinedOutput() } +// runSSHStdout keeps transport diagnostics out of machine-readable responses. +func runSSHStdout(ctx context.Context, endpoint domain.TransferEndpoint, args []string, input io.Reader) ([]byte, error) { + return runSSHStdoutMeasured(ctx, endpoint, args, input, nil, nil) +} + +func runSSHStdoutMeasured(ctx context.Context, endpoint domain.TransferEndpoint, args []string, input io.Reader, wait, execution *float64) ([]byte, error) { + queuedAt := time.Now() + command, release, err := queuedSSHCommand(ctx, endpoint, args) + if wait != nil { + *wait += time.Since(queuedAt).Seconds() + } + if err != nil { + return nil, err + } + defer release() + var stdout, stderr bytes.Buffer + command.Stdin = input + command.Stdout = &stdout + command.Stderr = &stderr + startedAt := time.Now() + err = command.Run() + if execution != nil { + *execution += time.Since(startedAt).Seconds() + } + if err != nil { + return stdout.Bytes(), fmt.Errorf("ssh command: %w: %s", err, strings.TrimSpace(stderr.String())) + } + return stdout.Bytes(), nil +} + +// runSSHProtocol exposes only bounded failure categories, never remote stderr. +// Successful machine-readable stdout remains separate from transport diagnostics. +func runSSHProtocol(ctx context.Context, endpoint domain.TransferEndpoint, args []string) ([]byte, error) { + return runSSHProtocolMeasured(ctx, endpoint, args, nil, nil) +} + +func runSSHProtocolMeasured(ctx context.Context, endpoint domain.TransferEndpoint, args []string, wait, execution *float64) ([]byte, error) { + output, err := runSSHStdoutMeasured(ctx, endpoint, args, nil, wait, execution) + if err == nil { + return output, nil + } + category := peerProbeFailureCategory(err) + message := strings.ToLower(err.Error()) + for _, diagnostic := range []struct{ text, category string }{ + {"connection closed", "connection_closed"}, + {"connection reset", "connection_reset"}, + {"broken pipe", "broken_pipe"}, + {"no space left", "disk_full"}, + {"sealed snapshot", "manifest_validation_failed"}, + } { + if strings.Contains(message, diagnostic.text) { + category = diagnostic.category + break + } + } + if ctx.Err() != nil { + return nil, fmt.Errorf("SSH protocol failure [%s]: %w", category, ctx.Err()) + } + return nil, fmt.Errorf("SSH protocol failure [%s]", category) +} + func sshControlPath(args []string) string { for _, argument := range args { if strings.HasPrefix(argument, "ControlPath=") { @@ -416,4 +560,4 @@ type readCloser struct { } func (r readCloser) Close() error { return r.close() } -func shell(v string) string { return "'" + strings.ReplaceAll(v, "'", "'\\\"'\\\"'") + "'" } +func shell(v string) string { return "'" + strings.ReplaceAll(v, "'", "'\"'\"'") + "'" } diff --git a/internal/infrastructure/transfer/rsync_ssh_test.go b/internal/infrastructure/transfer/rsync_ssh_test.go index 8857d6d8..42852702 100644 --- a/internal/infrastructure/transfer/rsync_ssh_test.go +++ b/internal/infrastructure/transfer/rsync_ssh_test.go @@ -3,16 +3,140 @@ package transfer import ( "bytes" "context" + "fmt" "io" "os" + "os/exec" "path/filepath" "reflect" "strings" + "sync" "testing" + "time" "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/provider" + "golang.org/x/crypto/ssh" ) +func TestConcurrentDirectCredentialRevocationPreservesUnrelatedKeys(t *testing.T) { + if _, err := exec.LookPath("flock"); err != nil { + t.Skip("worker credential integration requires Linux flock; run this test in the worker-compatible container") + } + directory := t.TempDir() + if strings.ContainsAny(directory, " '\"\n\t") { + t.Fatal("fixture path must be shell-safe") + } + publicKey, _, err := ephemeralSSHKey() + if err != nil { + t.Fatal(err) + } + lines := []string{"unrelated-user-key", "unrelated-transfer-token-suffix"} + var revoked []string + for i := 0; i < 8; i++ { + line, err := scopedDirectAuthorizedKey(publicKey, fmt.Sprintf("transfer-%d", i), "cat > /tmp/private-archive", time.Now().Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + lines = append(lines, line) + revoked = append(revoked, line) + } + keys := filepath.Join(directory, "authorized_keys") + if err := os.WriteFile(keys, []byte(strings.Join(lines, "\n")+"\n"), 0600); err != nil { + t.Fatal(err) + } + var wait sync.WaitGroup + for _, line := range revoked { + wait.Add(1) + go func(line string) { + defer wait.Done() + command := strings.ReplaceAll(directCredentialRemovalCommand(line), "~/.ssh", directory) + if output, err := exec.Command("sh", "-c", command).CombinedOutput(); err != nil { + t.Errorf("revoke: %v %s", err, output) + } + }(line) + } + wait.Wait() + content, err := os.ReadFile(keys) + if err != nil { + t.Fatal(err) + } + if string(content) != strings.Join(lines[:2], "\n")+"\n" { + t.Fatalf("unrelated keys lost or credentials retained: %s", content) + } + info, err := os.Stat(keys) + if err != nil || info.Mode().Perm() != 0600 { + t.Fatal("unsafe final authorized_keys permissions") + } +} + +func TestShellQuotingRoundTripsThroughNestedPOSIXCommands(t *testing.T) { + for _, value := range []string{"", "plain", "worker's file", "'\"$HOME; $(false)\nsecond line", "back\\slash"} { + for _, nested := range []bool{false, true} { + command := "printf '%s' " + shell(value) + if nested { + command = "sh -c " + shell(command) + } + output, err := exec.Command("sh", "-c", command).CombinedOutput() + if err != nil || string(output) != value { + t.Fatalf("nested=%v roundtrip failed: %v %q", nested, err, output) + } + } + } +} + +func TestEphemeralTransferKeyIsReadableByOpenSSH(t *testing.T) { + keygen, err := exec.LookPath("ssh-keygen") + if err != nil { + t.Fatal("OpenSSH is required for transfer credential integration") + } + publicKey, privateKey, err := ephemeralSSHKey() + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "identity") + if err := os.WriteFile(path, privateKey, 0600); err != nil { + t.Fatal(err) + } + output, err := exec.Command(keygen, "-y", "-f", path).CombinedOutput() + if err != nil { + t.Fatalf("OpenSSH cannot load ephemeral identity: %v %s", err, output) + } + expected, _, _, _, expectedErr := ssh.ParseAuthorizedKey(publicKey) + actual, _, _, _, actualErr := ssh.ParseAuthorizedKey(output) + if expectedErr != nil || actualErr != nil || !bytes.Equal(expected.Marshal(), actual.Marshal()) { + t.Fatal("OpenSSH identity does not match installed authorization") + } +} + +func TestDirectCredentialIsForcedAndExpiring(t *testing.T) { + publicKey, _, err := ephemeralSSHKey() + if err != nil { + t.Fatal(err) + } + command := "cat > '/tmp/preparation/archive'" + line, err := scopedDirectAuthorizedKey(publicKey, "transfer-token", command, time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatal(err) + } + _, comment, options, rest, err := ssh.ParseAuthorizedKey([]byte(line)) + if err != nil || len(rest) != 0 || comment != "transfer-token" { + t.Fatalf("invalid authorized key syntax: %v", err) + } + if !reflect.DeepEqual(options, []string{"restrict", `command="cat > '/tmp/preparation/archive'"`, `expiry-time="20261008090000Z"`}) { + t.Fatalf("authorization must not allow arbitrary commands: %#v", options) + } + for _, invalid := range []string{"", "cat > file\nother-command", "cat\x00"} { + if _, err := scopedDirectAuthorizedKey(publicKey, "token", invalid, time.Now()); err == nil { + t.Fatal("unsafe forced command accepted") + } + } + remove := directCredentialRemovalCommand(line) + if !strings.Contains(remove, "flock") || !strings.Contains(remove, "grep -F -x -v") || !strings.Contains(remove, "chmod 600") { + t.Fatal("revocation must serialize exact-line removal and preserve permissions") + } +} + func installFakeSSH(t *testing.T) { t.Helper() directory := t.TempDir() @@ -20,6 +144,9 @@ func installFakeSSH(t *testing.T) { command="" for argument in "$@"; do command="$argument"; done case "$command" in + *"sha256sum --"*checksum-error*) echo 'checksum unavailable' >&2; exit 2 ;; + *"sha256sum --"*) printf '%064d artifact\n' 0; exit 0 ;; + *"cat >"*upload-rejected*) echo 'remote filesystem quota exceeded' >&2; exit 23 ;; *missing*) exit 1 ;; *transport-error*) echo 'remote unavailable' >&2; exit 2 ;; *"cat --"*|*"tail -c"*) printf payload; exit 0 ;; @@ -69,11 +196,33 @@ func TestSSHTargetAndArguments(t *testing.T) { t.Fatalf("args=%q missing %q", joined, expected) } } - if shell("a'b") != `'a'\"'\"'b'` || !reflect.DeepEqual(sshArgs(domain.TransferEndpoint{}), []string{}) { + if shell("a'b") != `'a'"'"'b'` || !reflect.DeepEqual(sshArgs(domain.TransferEndpoint{}), []string{"-o", "BatchMode=yes", "-o", "ConnectionAttempts=1", "-o", "ConnectTimeout=10"}) { t.Fatalf("SSH helper output mismatch: shell=%q args=%#v", shell("a'b"), sshArgs(domain.TransferEndpoint{})) } } +func TestTransferSharesCanonicalConnectionSession(t *testing.T) { + t.Setenv("AKOFLOW_SSH_CONTROL_DIRECTORY", t.TempDir()) + endpoint := domain.TransferEndpoint{ + ConnectionID: "cloud-connection-instance-a", + URI: "ssh://worker@example.test/workspace?port=22&identityFile=/keys/id&knownHostsFile=/known&hostKeyAlias=instance-a", + } + _, expected, err := provider.SSHMultiplexArguments(provider.SSHSessionKey{ + ConnectionID: endpoint.ConnectionID, Username: "worker", Host: "example.test", Port: 22, + IdentityFile: "/keys/id", KnownHostsFile: "/known", HostKeyAlias: "instance-a", + }) + if err != nil { + t.Fatal(err) + } + if actual := sshControlPath(sshArgs(endpoint)); actual != expected { + t.Fatalf("transfer session %q differs from canonical connection session %q", actual, expected) + } + endpoint.ConnectionID = "cloud-connection-instance-b" + if actual := sshControlPath(sshArgs(endpoint)); actual == expected { + t.Fatal("distinct connections must retain distinct channel limits") + } +} + func TestRsyncSSHCommandLifecycle(t *testing.T) { installFakeSSH(t) connector := RsyncSSH{BufferSize: func(context.Context) int { return 1024 }} @@ -112,6 +261,51 @@ func TestRsyncSSHCommandLifecycle(t *testing.T) { } } +func TestSSHUploadPreservesEarlyExitDiagnostic(t *testing.T) { + installFakeSSH(t) + err := (RsyncSSH{}).Put(context.Background(), sshTestEndpoint(), "upload-rejected", bytes.NewReader(bytes.Repeat([]byte("x"), 1024*1024)), 0) + if err == nil || !strings.Contains(err.Error(), "remote filesystem quota exceeded") || !strings.Contains(err.Error(), "exit status 23") { + t.Fatalf("remote upload diagnostic lost: %v", err) + } +} + +func TestSSHRemoteChecksumMatchAndMismatch(t *testing.T) { + installFakeSSH(t) + connector := RsyncSSH{} + for _, tc := range []struct { + digest string + matches bool + }{ + {"sha256:" + strings.Repeat("0", 64), true}, + {"sha256:" + strings.Repeat("1", 64), false}, + } { + matches, err := connector.VerifyDigest(context.Background(), sshTestEndpoint(), "present", tc.digest) + if err != nil || matches != tc.matches { + t.Fatalf("matches=%v expected=%v err=%v", matches, tc.matches, err) + } + } + if _, err := connector.VerifyDigest(context.Background(), sshTestEndpoint(), "checksum-error", strings.Repeat("0", 64)); err == nil || !strings.Contains(err.Error(), "checksum unavailable") { + t.Fatalf("checksum failure lost: %v", err) + } +} + +func TestTransferSSHConnectionIsBounded(t *testing.T) { + args := strings.Join(sshArgs(sshTestEndpoint()), " ") + for _, option := range []string{"BatchMode=yes", "ConnectionAttempts=1", "ConnectTimeout=10"} { + if !strings.Contains(args, option) { + t.Fatalf("missing %s in %s", option, args) + } + } + command, release, err := queuedSSHCommand(context.Background(), domain.TransferEndpoint{}, []string{"host", "true"}) + if err != nil { + t.Fatal(err) + } + defer release() + if command.WaitDelay <= 0 { + t.Fatal("pipe cleanup is unbounded") + } +} + func TestRsyncSSHRejectsUnsafePathsBeforeExecution(t *testing.T) { connector := RsyncSSH{} endpoint := sshTestEndpoint() diff --git a/internal/infrastructure/transfer/ssh_read_retry.go b/internal/infrastructure/transfer/ssh_read_retry.go new file mode 100644 index 00000000..18b85cd6 --- /dev/null +++ b/internal/infrastructure/transfer/ssh_read_retry.go @@ -0,0 +1,27 @@ +package transfer + +import ( + "context" + "strings" + "time" +) + +// Only read-only probes may use this helper; retrying activity commands could +// execute user work twice after an ambiguous disconnect. +func retrySSHRead(ctx context.Context, delay time.Duration, read func() ([]byte, error)) ([]byte, error) { + for attempt := 0; ; attempt++ { + output, err := read() + message := strings.ToLower(string(output)) + transient := strings.Contains(message, "connection closed") || strings.Contains(message, "connection reset") || strings.Contains(message, "broken pipe") + if err == nil || !transient || attempt == 2 || ctx.Err() != nil { + return output, err + } + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return output, ctx.Err() + case <-timer.C: + } + } +} diff --git a/internal/infrastructure/transfer/ssh_read_retry_test.go b/internal/infrastructure/transfer/ssh_read_retry_test.go new file mode 100644 index 00000000..42a8d70f --- /dev/null +++ b/internal/infrastructure/transfer/ssh_read_retry_test.go @@ -0,0 +1,38 @@ +package transfer + +import ( + "context" + "errors" + "testing" +) + +func TestSSHReadRetryIsBoundedAndTransportOnly(t *testing.T) { + for _, test := range []struct { + message string + calls int + }{ + {"Connection closed by UNKNOWN port 65535", 3}, + {"Permission denied (publickey)", 1}, + {"", 1}, + } { + calls := 0 + _, err := retrySSHRead(context.Background(), 0, func() ([]byte, error) { calls++; return []byte(test.message), errors.New("failed") }) + if err == nil || calls != test.calls { + t.Fatalf("%q calls=%d err=%v", test.message, calls, err) + } + } +} + +func TestSSHReadRetryRecoversTransientDisconnect(t *testing.T) { + calls := 0 + _, err := retrySSHRead(context.Background(), 0, func() ([]byte, error) { + calls++ + if calls == 1 { + return []byte("Connection reset"), errors.New("failed") + } + return nil, nil + }) + if err != nil || calls != 2 { + t.Fatalf("calls=%d err=%v", calls, err) + } +} diff --git a/internal/infrastructure/transfer/ssh_stdout_test.go b/internal/infrastructure/transfer/ssh_stdout_test.go new file mode 100644 index 00000000..8381d05c --- /dev/null +++ b/internal/infrastructure/transfer/ssh_stdout_test.go @@ -0,0 +1,45 @@ +package transfer + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestSSHStdoutSeparatesDiagnostics(t *testing.T) { + bin := t.TempDir() + script := "#!/bin/sh\nprintf '/tmp/akoflow-peer-ABC123\\n'\nprintf 'transport diagnostic\\n' >&2\nexit \"${SSH_TEST_EXIT:-0}\"\n" + if err := os.WriteFile(filepath.Join(bin, "ssh"), []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + output, err := runSSHStdout(context.Background(), domain.TransferEndpoint{}, nil, nil) + if err != nil { + t.Fatal(err) + } + if string(output) != "/tmp/akoflow-peer-ABC123\n" { + t.Fatalf("contaminated stdout: %q", output) + } + t.Setenv("SSH_TEST_EXIT", "1") + _, err = runSSHStdout(context.Background(), domain.TransferEndpoint{}, nil, nil) + if err == nil || !strings.Contains(err.Error(), "transport diagnostic") { + t.Fatalf("missing failure diagnostic: %v", err) + } +} + +func TestSSHProtocolFailureIsCategorizedWithoutSensitiveDiagnostics(t *testing.T) { + bin := t.TempDir() + script := "#!/bin/sh\nprintf 'Connection closed by secret-host; private-key secret-value\\n' >&2\nexit 255\n" + if err := os.WriteFile(filepath.Join(bin, "ssh"), []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + _, err := runSSHProtocol(context.Background(), domain.TransferEndpoint{}, nil) + if err == nil || !strings.Contains(err.Error(), "connection_closed") || strings.Contains(err.Error(), "secret") { + t.Fatalf("unsafe or missing diagnostic category: %v", err) + } +} diff --git a/internal/infrastructure/transfer/workspace_archive.go b/internal/infrastructure/transfer/workspace_archive.go new file mode 100644 index 00000000..a8acc156 --- /dev/null +++ b/internal/infrastructure/transfer/workspace_archive.go @@ -0,0 +1,397 @@ +package transfer + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "context" + "crypto/sha256" + "fmt" + "io" + "io/fs" + "net/url" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func measureWorkspacePhase(run *domain.DataTransferRun, name string, operation func() error) error { + start := time.Now() + err := operation() + end := time.Now() + run.Route.Phases = append(run.Route.Phases, domain.TransferPhase{ + Name: name, StartedAt: float64(start.UnixNano()) / 1e9, + FinishedAt: float64(end.UnixNano()) / 1e9, + DurationSeconds: end.Sub(start).Seconds(), Failed: err != nil, + }) + return err +} + +// Record admission separately from destination command service. A zero-valued +// pointer proves the queue was measured; absence keeps legacy wall-time data +// distinguishable, so calibration cannot silently double-count contention. +func measureWorkspaceSSHPhase(ctx context.Context, run *domain.DataTransferRun, name string, endpoint domain.TransferEndpoint, args []string) ([]byte, error) { + var output []byte + var queue, execution float64 + var executed bool + err := measureWorkspacePhase(run, name, func() error { + queuedAt := time.Now() + command, release, err := queuedSSHCommand(ctx, endpoint, args) + queue = time.Since(queuedAt).Seconds() + if err != nil { + return err + } + defer release() + startedAt := time.Now() + output, err = command.CombinedOutput() + execution, executed = time.Since(startedAt).Seconds(), true + return err + }) + phase := &run.Route.Phases[len(run.Route.Phases)-1] + phase.SSHChannelWaitSeconds = &queue + if executed { + phase.ExecutionSeconds = &execution + } + return output, err +} + +func parallelWorkspacePreparation(ctx context.Context, count, limit int, prepare func(context.Context, int) error) error { + if limit <= 0 { + limit = 8 + } + if limit > 8 { + limit = 8 + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + slots := make(chan struct{}, limit) + var workers sync.WaitGroup + var once sync.Once + var firstError error + for i := 0; i < count; i++ { + workers.Add(1) + go func(i int) { + defer workers.Done() + select { + case slots <- struct{}{}: + case <-ctx.Done(): + return + } + defer func() { <-slots }() + if ctx.Err() != nil { + return + } + if err := prepare(ctx, i); err != nil { + once.Do(func() { firstError = err; cancel() }) + } + }(i) + } + workers.Wait() + if firstError != nil { + return firstError + } + return ctx.Err() +} + +// Archives contain snapshots, never a directly published consumer workspace. +// Validate every member locally and verify the archive digest on the receiver +// before extraction. The existing aggregate content/conflict barrier still runs. +func relayWorkspaceArchive(ctx context.Context, source, destination domain.TransferEndpoint, run *domain.DataTransferRun) error { + root, err := os.MkdirTemp("", "akoflow-archive-") + if err != nil { + return err + } + defer os.RemoveAll(root) + archivePath := filepath.Join(root, "workspace.tar.gz") + if err := receiveRelayWorkspaceArchive(ctx, source, archivePath, run); err != nil { + return err + } + var digest string + err = measureWorkspacePhase(run, "archive-validation", func() error { + var validationErr error + digest, validationErr = validateWorkspaceArchive(archivePath) + return validationErr + }) + if err != nil { + return err + } + info, err := os.Stat(archivePath) + if err != nil { + return err + } + if strings.HasPrefix(source.URI, "ssh://") { + run.NetworkBytes += info.Size() + } + run.Route.ArchiveBytes = info.Size() + remoteURL, err := url.Parse(destination.URI) + if err != nil { + return err + } + remoteURL.Path += ".archive" + spool := destination + spool.URI = remoteURL.String() + if err := makeWorkspaceDirectory(ctx, spool); err != nil { + return err + } + local := domain.TransferEndpoint{URI: (&url.URL{Scheme: "file", Path: root}).String()} + var stats rsyncStats + err = measureWorkspacePhase(run, "archive-upload", func() error { + var uploadErr error + stats, uploadErr = syncDirectory(ctx, local, spool, false) + return uploadErr + }) + if stats.executionSeconds > 0 { + phase := &run.Route.Phases[len(run.Route.Phases)-1] + phase.SSHChannelWaitSeconds = &stats.channelWaitSeconds + phase.ExecutionSeconds = &stats.executionSeconds + } + recordWorkspaceRsyncStats(run, stats, true) + if err != nil { + return err + } + if err := makeWorkspaceDirectory(ctx, destination); err != nil { + return err + } + host, target, err := sshTarget(destination, "") + if err != nil { + return err + } + _, spoolPath, err := sshTarget(spool, "") + if err != nil { + return err + } + remoteArchive := filepath.Join(spoolPath, "workspace.tar.gz") + command := "set -eu; actual=$(sha256sum -- " + shell(remoteArchive) + "); test \"${actual%% *}\" = " + shell(digest) + + "; tar --extract --file=" + shell(remoteArchive) + " --directory=" + shell(target) + " --no-same-owner --no-same-permissions" + var output []byte + output, err = measureWorkspaceSSHPhase(ctx, run, "receiver-verify-and-extract", destination, append(sshArgs(destination), host, command)) + if err != nil { + return fmt.Errorf("verify/extract workspace archive: %w: %s", err, strings.TrimSpace(string(output))) + } + return rejectWorkspaceSymlinks(ctx, destination) +} + +func packWorkspaceArchive(ctx context.Context, source domain.TransferEndpoint, archivePath string) error { + return packWorkspaceArchiveWithTiming(ctx, source, archivePath, &workspaceArchiveTiming{}) +} + +type workspaceArchiveTiming struct { + PreflightSeconds float64 + PreflightChannelWaitSeconds float64 + ChannelWaitSeconds float64 + ExecutionSeconds float64 +} + +func packWorkspaceArchiveWithTiming(ctx context.Context, source domain.TransferEndpoint, archivePath string, timing *workspaceArchiveTiming) error { + preflightStarted := time.Now() + preflightErr := rejectWorkspaceSymlinksWithWait(ctx, source, &timing.PreflightChannelWaitSeconds) + timing.PreflightSeconds = time.Since(preflightStarted).Seconds() + if preflightErr != nil { + return preflightErr + } + u, err := url.Parse(source.URI) + if err != nil { + return err + } + var command *exec.Cmd + release := func() {} + switch u.Scheme { + case "file": + started := time.Now() + defer func() { timing.ExecutionSeconds = time.Since(started).Seconds() }() + return packLocalWorkspaceArchive(ctx, u.Path, archivePath) + case "ssh": + host, origin, err := sshTarget(source, "") + if err != nil { + return err + } + queuedAt := time.Now() + command, release, err = queuedSSHCommand(ctx, source, append(sshArgs(source), host, + "set -eu; bad=$(find "+shell(origin)+" ! -type d ! -type f -print -quit); test -z \"$bad\"; bash -o pipefail -c "+shell("tar --hard-dereference -C \"$1\" -cf - . | gzip -1")+" archive "+shell(origin))) + timing.ChannelWaitSeconds = time.Since(queuedAt).Seconds() + if err != nil { + return err + } + default: + return fmt.Errorf("unsupported archive source scheme %q", u.Scheme) + } + defer release() + file, err := os.OpenFile(archivePath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return err + } + defer file.Close() + var stderr strings.Builder + command.Stdout, command.Stderr = file, &stderr + executionStarted := time.Now() + defer func() { timing.ExecutionSeconds = time.Since(executionStarted).Seconds() }() + if err := command.Run(); err != nil { + return fmt.Errorf("pack workspace archive: %w: %s", err, strings.TrimSpace(stderr.String())) + } + return file.Close() +} + +func packLocalWorkspaceArchive(ctx context.Context, root, filename string) error { + file, err := os.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return err + } + defer file.Close() + compressed, err := gzip.NewWriterLevel(file, gzip.BestSpeed) + if err != nil { + return err + } + defer compressed.Close() + writer := tar.NewWriter(compressed) + defer writer.Close() + err = filepath.WalkDir(root, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := ctx.Err(); err != nil { + return err + } + info, err := entry.Info() + if err != nil { + return err + } + if !info.IsDir() && !info.Mode().IsRegular() { + return fmt.Errorf("unsupported workspace entry %q", name) + } + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + rel, err := filepath.Rel(root, name) + if err != nil { + return err + } + header.Name = filepath.ToSlash(rel) + if err := writer.WriteHeader(header); err != nil { + return err + } + if info.IsDir() { + return nil + } + input, err := os.Open(name) + if err != nil { + return err + } + _, copyErr := io.Copy(writer, input) + closeErr := input.Close() + if copyErr != nil { + return copyErr + } + return closeErr + }) + if err != nil { + return err + } + if err := writer.Close(); err != nil { + return err + } + if err := compressed.Close(); err != nil { + return err + } + return file.Close() +} + +func validateWorkspaceArchive(filename string) (string, error) { + file, err := os.Open(filename) + if err != nil { + return "", err + } + defer file.Close() + hash := sha256.New() + transport := bufio.NewReader(io.TeeReader(file, hash)) + var reader io.Reader = transport + magic, _ := transport.Peek(2) + if len(magic) == 2 && magic[0] == 0x1f && magic[1] == 0x8b { + compressed, err := gzip.NewReader(transport) + if err != nil { + return "", fmt.Errorf("invalid compressed workspace archive: %w", err) + } + defer compressed.Close() + // BSD tar pads a gzip transport with zero blocks. Do not interpret this + // padding as a second gzip stream or accept unvalidated extra members. + compressed.Multistream(false) + reader = compressed + } + archive := tar.NewReader(reader) + seen := make(map[string]byte) + for { + header, err := archive.Next() + if err == io.EOF { + break + } + if err != nil { + return "", fmt.Errorf("invalid workspace archive: %w", err) + } + name := path.Clean(header.Name) + if path.IsAbs(header.Name) || name == ".." || strings.HasPrefix(name, "../") || strings.Contains("/"+header.Name+"/", "/../") { + return "", fmt.Errorf("unsafe archive path %q", header.Name) + } + if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeDir { + return "", fmt.Errorf("unsupported archive entry %q (links and special files forbidden)", header.Name) + } + if name == "." && header.Typeflag != tar.TypeDir { + return "", fmt.Errorf("invalid archive root") + } + if _, exists := seen[name]; exists { + return "", fmt.Errorf("duplicate archive path %q", name) + } + for parent := path.Dir(name); parent != "."; parent = path.Dir(parent) { + if kind, exists := seen[parent]; exists && kind != tar.TypeDir { + return "", fmt.Errorf("archive parent is a file: %q", parent) + } + } + seen[name] = header.Typeflag + } + // Drain the decompressor to validate its checksum/trailer, then include any + // remaining transport bytes in the digest verified before remote extraction. + if _, err := io.Copy(io.Discard, reader); err != nil { + return "", fmt.Errorf("invalid archive trailer: %w", err) + } + for { + b, err := transport.ReadByte() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + if b != 0 { + return "", fmt.Errorf("unexpected data after workspace archive") + } + } + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +func receiveRelayWorkspaceArchive(ctx context.Context, source domain.TransferEndpoint, archivePath string, run *domain.DataTransferRun) error { + if err := measureWorkspacePhase(run, "pack-and-receive", func() error { + var timing workspaceArchiveTiming + defer func() { + run.Route.SourcePackPreflightSeconds = timing.PreflightSeconds + run.Route.SourcePackPreflightSSHChannelWaitSeconds = timing.PreflightChannelWaitSeconds + run.Route.SourcePackPreflightServiceSeconds = max(0, timing.PreflightSeconds-timing.PreflightChannelWaitSeconds) + run.Route.SourcePackSSHChannelWaitSeconds = timing.ChannelWaitSeconds + run.Route.SourcePackExecutionSeconds = timing.ExecutionSeconds + }() + return packWorkspaceArchiveWithTiming(ctx, source, archivePath, &timing) + }); err != nil { + if strings.HasPrefix(source.URI, "ssh://") { + if partial, statErr := os.Stat(archivePath); statErr == nil { + run.NetworkBytes += partial.Size() + } + run.Route.NetworkBytesIncomplete = true + } + return err + } + return nil +} diff --git a/internal/infrastructure/transfer/workspace_archive_test.go b/internal/infrastructure/transfer/workspace_archive_test.go new file mode 100644 index 00000000..41468856 --- /dev/null +++ b/internal/infrastructure/transfer/workspace_archive_test.go @@ -0,0 +1,212 @@ +package transfer + +import ( + "archive/tar" + "context" + "crypto/sha256" + "fmt" + "net/url" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestSourcePackTimingSeparatesLocalWorkFromSSHQueue(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "data"), []byte("verified input"), 0600); err != nil { + t.Fatal(err) + } + archive := filepath.Join(t.TempDir(), "snapshot.tar.gz") + endpoint := domain.TransferEndpoint{URI: (&url.URL{Scheme: "file", Path: root}).String()} + var timing workspaceArchiveTiming + if err := packWorkspaceArchiveWithTiming(context.Background(), endpoint, archive, &timing); err != nil { + t.Fatal(err) + } + if _, err := validateWorkspaceArchive(archive); err != nil { + t.Fatal(err) + } + if timing.PreflightSeconds <= 0 || timing.ExecutionSeconds <= 0 || timing.ChannelWaitSeconds != 0 { + t.Fatalf("invalid local phase attribution: %+v", timing) + } + if err := os.Symlink("data", filepath.Join(root, "unsafe-link")); err != nil { + t.Fatal(err) + } + timing = workspaceArchiveTiming{} + err := packWorkspaceArchiveWithTiming(context.Background(), endpoint, filepath.Join(t.TempDir(), "blocked.tar.gz"), &timing) + if err == nil || timing.PreflightSeconds <= 0 || timing.ExecutionSeconds != 0 { + t.Fatalf("failed preflight attribution: %+v %v", timing, err) + } +} + +func TestArchivePreparationRunsTwoWorkersAndWaitsForBarrier(t *testing.T) { + entered := make(chan int, 4) + release := make(chan struct{}) + done := make(chan error, 1) + var active, peak, completed atomic.Int32 + go func() { + done <- parallelWorkspacePreparation(context.Background(), 4, 2, func(ctx context.Context, i int) error { + n := active.Add(1) + for old := peak.Load(); n > old; old = peak.Load() { + if peak.CompareAndSwap(old, n) { + break + } + } + defer active.Add(-1) + entered <- i + <-release + completed.Add(1) + return nil + }) + }() + for i := 0; i < 2; i++ { + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("preparations did not overlap") + } + } + select { + case <-entered: + t.Fatal("parallelism exceeded two") + case <-done: + t.Fatal("barrier released early") + case <-time.After(30 * time.Millisecond): + } + close(release) + if err := <-done; err != nil { + t.Fatal(err) + } + if peak.Load() != 2 || completed.Load() != 4 { + t.Fatalf("peak=%d completed=%d", peak.Load(), completed.Load()) + } +} + +func TestArchivePreparationDefaultAndMaximumEight(t *testing.T) { + for _, limit := range []int{0, 8, 100} { + t.Run(fmt.Sprint(limit), func(t *testing.T) { + entered := make(chan struct{}, 9) + release := make(chan struct{}) + done := make(chan error, 1) + go func() { + done <- parallelWorkspacePreparation(context.Background(), 9, limit, func(context.Context, int) error { + entered <- struct{}{} + <-release + return nil + }) + }() + for i := 0; i < 8; i++ { + select { + case <-entered: + case <-time.After(3 * time.Second): + close(release) + t.Fatal("eight workers did not overlap") + } + } + select { + case <-entered: + close(release) + t.Fatal("exceeded eight workers") + case <-done: + close(release) + t.Fatal("barrier released early") + case <-time.After(30 * time.Millisecond): + } + close(release) + if err := <-done; err != nil { + t.Fatal(err) + } + }) + } +} + +func TestArchivePreparationCancelsAndJoinsWorkersOnFailure(t *testing.T) { + started := make(chan struct{}) + var stopped atomic.Bool + err := parallelWorkspacePreparation(context.Background(), 2, 2, func(ctx context.Context, i int) error { + if i == 0 { + <-started + return fmt.Errorf("injected failure") + } + close(started) + <-ctx.Done() + stopped.Store(true) + return ctx.Err() + }) + if err == nil || !stopped.Load() { + t.Fatalf("failed barrier returned before worker stopped: %v", err) + } +} + +func TestWorkspaceArchiveRejectsUnsafeMembers(t *testing.T) { + for _, test := range []struct { + name string + kind byte + link string + }{ + {"../escape", tar.TypeReg, ""}, {"/absolute", tar.TypeReg, ""}, + {"link", tar.TypeSymlink, "/outside"}, {"hardlink", tar.TypeLink, "other"}, + {"fifo", tar.TypeFifo, ""}, + } { + t.Run(test.name, func(t *testing.T) { + filename := filepath.Join(t.TempDir(), "bad.tar") + file, err := os.Create(filename) + if err != nil { + t.Fatal(err) + } + writer := tar.NewWriter(file) + if err := writer.WriteHeader(&tar.Header{Name: test.name, Typeflag: test.kind, Linkname: test.link, Mode: 0600}); err != nil { + t.Fatal(err) + } + writer.Close() + file.Close() + if _, err := validateWorkspaceArchive(filename); err == nil { + t.Fatal("unsafe archive accepted") + } + }) + } +} + +func TestCompressedWorkspaceArchiveRoundtripDigest(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "nested"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "nested", "input.fits"), []byte("unchanged content"), 0600); err != nil { + t.Fatal(err) + } + filename := filepath.Join(t.TempDir(), "workspace.tar") + if err := packLocalWorkspaceArchive(context.Background(), root, filename); err != nil { + t.Fatal(err) + } + digest, err := validateWorkspaceArchive(filename) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filename) + if err != nil { + t.Fatal(err) + } + if digest != fmt.Sprintf("%x", sha256.Sum256(data)) { + t.Fatal("archive digest does not cover full transmitted file") + } + if err := os.WriteFile(filename, data[:len(data)-4], 0600); err != nil { + t.Fatal(err) + } + if _, err := validateWorkspaceArchive(filename); err == nil { + t.Fatal("truncated gzip trailer accepted") + } + if len(data) < 2 || data[0] != 0x1f || data[1] != 0x8b { + t.Fatal("not a gzip archive") + } + data[len(data)-1] ^= 1 + if err := os.WriteFile(filename, data, 0600); err != nil { + t.Fatal(err) + } + if _, err := validateWorkspaceArchive(filename); err == nil { + t.Fatal("corrupted gzip trailer accepted") + } +} diff --git a/internal/infrastructure/transfer/workspace_cleanup_retry_test.go b/internal/infrastructure/transfer/workspace_cleanup_retry_test.go new file mode 100644 index 00000000..fe47e576 --- /dev/null +++ b/internal/infrastructure/transfer/workspace_cleanup_retry_test.go @@ -0,0 +1,31 @@ +package transfer + +import ( + "os" + "path/filepath" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestOwnershipMarkerRemovalIsIdempotent(t *testing.T) { + marker := filepath.Join(t.TempDir(), "workspace.akoflow-owner") + if err := os.WriteFile(marker, []byte("validated ownership"), 0600); err != nil { + t.Fatal(err) + } + if err := removeWorkspaceMarker(marker); err != nil { + t.Fatal(err) + } + if err := removeWorkspaceMarker(marker); err != nil { + t.Fatalf("concurrent releaser's prior removal must succeed: %v", err) + } +} + +func TestOnlyCloudDockerWorkspaceCleanupUsesPrivilege(t *testing.T) { + for _, driver := range []string{"cloud-ssh", "ssh", "local", "kubernetes-pvc"} { + command := workspaceRemovalCommand(domain.ActivityWorkspace{Driver: driver}) + if driver == "cloud-ssh" && command != "sudo -n rm" || driver != "cloud-ssh" && command != "rm" { + t.Fatalf("driver=%s command=%s", driver, command) + } + } +} diff --git a/internal/infrastructure/transfer/workspace_cloud_peer.go b/internal/infrastructure/transfer/workspace_cloud_peer.go new file mode 100644 index 00000000..ef01df1a --- /dev/null +++ b/internal/infrastructure/transfer/workspace_cloud_peer.go @@ -0,0 +1,49 @@ +package transfer + +import ( + "net/netip" + "net/url" + "strings" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +// cloudWorkspacePeerEligible is only an eligibility gate, not proof of +// connectivity. EndpointResolver supplies the concrete generation, private +// address and Terraform network identity. A subsequent authenticated probe +// must prove reachability before any peer payload transfer is selected. +func cloudWorkspacePeerEligible(source, destination domain.TransferEndpoint) bool { + if source.CloudInstanceID == "" || destination.CloudInstanceID == "" || source.CloudInstanceID == destination.CloudInstanceID { + return false + } + if source.EnvironmentID == "" || source.EnvironmentID != destination.EnvironmentID || + source.NetworkDomain == "" || source.NetworkDomain != destination.NetworkDomain { + return false + } + addresses := make([]netip.Addr, 0, 2) + for _, endpoint := range []domain.TransferEndpoint{source, destination} { + if endpoint.ConnectionID == "" || !strings.HasSuffix(endpoint.ConnectionID, "-"+endpoint.CloudInstanceID) { + return false + } + u, err := url.Parse(endpoint.URI) + if err != nil || u.Scheme != "ssh" || u.User == nil || u.User.Username() == "" { + return false + } + q := u.Query() + if q.Get("strictHostKeyChecking") != "true" || q.Get("hostKeyAlias") == "" || q.Get("knownHostsFile") == "" || + q.Get("acceptNewHostKey") == "true" || q.Get("forwardAgent") == "true" { + return false + } + // The current cloud connector resolves port 22. Do not silently ignore + // a custom port when constructing a private peer connection. + if (u.Port() != "" && u.Port() != "22") || (q.Get("port") != "" && q.Get("port") != "22") { + return false + } + address, err := netip.ParseAddr(strings.TrimSpace(endpoint.Configuration["directAddress"])) + if err != nil || !address.Is4() || !address.IsPrivate() { + return false + } + addresses = append(addresses, address) + } + return addresses[0] != addresses[1] +} diff --git a/internal/infrastructure/transfer/workspace_cloud_peer_test.go b/internal/infrastructure/transfer/workspace_cloud_peer_test.go new file mode 100644 index 00000000..7b0d8adc --- /dev/null +++ b/internal/infrastructure/transfer/workspace_cloud_peer_test.go @@ -0,0 +1,48 @@ +package transfer + +import ( + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestCloudWorkspacePeerEligibilityFailsClosed(t *testing.T) { + endpoint := func(id, address string) domain.TransferEndpoint { + return domain.TransferEndpoint{ + CloudInstanceID: id, ConnectionID: "connection-" + id, EnvironmentID: "environment", + NetworkDomain: "project:region:default:", + URI: "ssh://worker@example.test/workspace?strictHostKeyChecking=true&hostKeyAlias=generation&knownHostsFile=/trusted/hosts&port=22", + Configuration: map[string]string{"directAddress": address}, + } + } + for _, tc := range []struct { + name string + change func(*domain.TransferEndpoint) + }{ + {"missing generation", func(e *domain.TransferEndpoint) { e.CloudInstanceID = "" }}, + {"same generation", func(e *domain.TransferEndpoint) { e.CloudInstanceID = "source" }}, + {"wrong resolved connection", func(e *domain.TransferEndpoint) { e.ConnectionID = "unbound" }}, + {"other environment", func(e *domain.TransferEndpoint) { e.EnvironmentID = "other" }}, + {"missing network proof", func(e *domain.TransferEndpoint) { e.NetworkDomain = "" }}, + {"different network", func(e *domain.TransferEndpoint) { e.NetworkDomain = "project:region:other:" }}, + {"public address", func(e *domain.TransferEndpoint) { e.Configuration["directAddress"] = "192.0.2.1" }}, + {"missing private address", func(e *domain.TransferEndpoint) { e.Configuration["directAddress"] = "" }}, + {"same private address", func(e *domain.TransferEndpoint) { e.Configuration["directAddress"] = "10.0.0.1" }}, + {"untrusted host", func(e *domain.TransferEndpoint) { e.URI = "ssh://worker@example.test/workspace" }}, + {"agent forwarding", func(e *domain.TransferEndpoint) { e.URI += "&forwardAgent=true" }}, + {"custom port", func(e *domain.TransferEndpoint) { + e.URI = "ssh://worker@example.test:2222/workspace?strictHostKeyChecking=true&hostKeyAlias=generation&knownHostsFile=/trusted/hosts" + }}, + } { + t.Run(tc.name, func(t *testing.T) { + a, b := endpoint("source", "10.0.0.1"), endpoint("destination", "10.0.0.2") + if !cloudWorkspacePeerEligible(a, b) { + t.Fatal("expected resolved private peer eligibility before mutation") + } + tc.change(&b) + if cloudWorkspacePeerEligible(a, b) || cloudWorkspacePeerEligible(b, a) { + t.Fatal("unsafe or unsupported peer must retain fallback") + } + }) + } +} diff --git a/internal/infrastructure/transfer/workspace_lifecycle.go b/internal/infrastructure/transfer/workspace_lifecycle.go index a8969d90..0e740afe 100644 --- a/internal/infrastructure/transfer/workspace_lifecycle.go +++ b/internal/infrastructure/transfer/workspace_lifecycle.go @@ -13,6 +13,7 @@ import ( "path/filepath" "strconv" "strings" + "time" "github.com/UFFeScience/akoflow/internal/domain" ) @@ -88,7 +89,9 @@ func (m ActivityWorkspaceManager) Inspect(ctx context.Context, workspace domain. } command := "test -d " + shell(path) + " || exit 0; find " + shell(path) + " -type f ! -name .akoflow-owner -exec wc -c {} \\; | awk '{bytes += $1; files += 1} END {printf \"%d %d\\n\", files, bytes}'" - output, err := runSSHCombinedOutput(ctx, endpoint, append(sshArgs(endpoint), host, command), nil) + output, err := retrySSHRead(ctx, time.Second, func() ([]byte, error) { + return runSSHStdout(ctx, endpoint, append(sshArgs(endpoint), host, command), nil) + }) if err != nil { return domain.WorkspaceUsage{}, fmt.Errorf("inspect remote workspace: %w: %s", err, strings.TrimSpace(string(output))) } @@ -136,7 +139,7 @@ func (m ActivityWorkspaceManager) PruneInputs(ctx context.Context, workspace dom if err != nil { return domain.WorkspaceReleaseResult{}, err } - output, err := runSSHCombinedOutput(ctx, endpoint, append(sshArgs(endpoint), host, command), strings.NewReader(payload)) + output, err := runSSHStdout(ctx, endpoint, append(sshArgs(endpoint), host, command), strings.NewReader(payload)) if err != nil { return domain.WorkspaceReleaseResult{}, fmt.Errorf("prune remote workspace inputs: %w: %s", err, strings.TrimSpace(string(output))) } @@ -171,11 +174,12 @@ func (m ActivityWorkspaceManager) Release(ctx context.Context, workspace domain. } marker := workspaceMarkerFor(workspace, path) expected := strings.TrimSpace(string(markerContents(workspace))) + removeCommand := workspaceRemovalCommand(workspace) command := "set -eu; if test ! -e " + shell(path) + "; then rm -f -- " + shell(marker) + "; exit 0; fi; " + "test ! -L " + shell(path) + "; canonical=$(realpath -- " + shell(path) + "); " + "test \"$canonical\" = " + shell(filepath.Clean(path)) + "; test -f " + shell(marker) + "; " + "test \"$(cat -- " + shell(marker) + ")\" = " + shell(expected) + "; " + - "rm -rf --one-file-system -- \"$canonical\"; rm -f -- " + shell(marker) + removeCommand + " -rf --one-file-system -- \"$canonical\"; rm -f -- " + shell(marker) if isKubernetesEndpoint(endpoint) { if m.Kubernetes == nil { return domain.WorkspaceReleaseResult{}, fmt.Errorf("Kubernetes workspace lifecycle runner is unavailable") @@ -199,13 +203,27 @@ func (m ActivityWorkspaceManager) Release(ctx context.Context, workspace domain. return domain.WorkspaceReleaseResult{ReclaimedBytes: usage.SizeBytes}, nil } +func workspaceRemovalCommand(workspace domain.ActivityWorkspace) string { + // Cloud Docker runs may produce root-owned bind-mounted files. Privilege is + // limited to the canonical, ownership-validated activity directory above. + if workspace.Driver == "cloud-ssh" { + return "sudo -n rm" + } + return "rm" +} + func (m ActivityWorkspaceManager) endpoint(ctx context.Context, workspace domain.ActivityWorkspace) (domain.TransferEndpoint, string, error) { if err := validateWorkspaceIdentity(workspace, workspace.ExecutionPath); err != nil { return domain.TransferEndpoint{}, "", err } u := &url.URL{Scheme: "file", Path: workspace.ExecutionPath} - if parsed, parseErr := url.Parse(workspace.URI); parseErr == nil && parsed.Scheme == "kubernetes" { - u = parsed + parsedWorkspaceURI, parseErr := url.Parse(workspace.URI) + if parseErr == nil && parsedWorkspaceURI.Scheme == "kubernetes" { + u = parsedWorkspaceURI + } + cloudInstanceID := "" + if parseErr == nil { + cloudInstanceID = parsedWorkspaceURI.Query().Get("cloudInstanceId") } if workspace.ConnectionID != "" { query := u.Query() @@ -215,6 +233,7 @@ func (m ActivityWorkspaceManager) endpoint(ctx context.Context, workspace domain location := domain.TransferLocation{ URI: u.String(), ResourceID: workspace.ResourceID, EnvironmentID: workspace.EnvironmentID, RuntimeID: workspace.RuntimeID, ConnectionID: workspace.ConnectionID, + CloudInstanceID: cloudInstanceID, } if workspace.ConnectionID == "" { return domain.TransferEndpoint{URI: location.URI}, workspace.ExecutionPath, nil @@ -308,7 +327,7 @@ func releaseLocalWorkspace(workspace domain.ActivityWorkspace, path string) erro } info, err := os.Lstat(path) if os.IsNotExist(err) { - return os.Remove(marker) + return removeWorkspaceMarker(marker) } if err != nil { return err @@ -319,7 +338,17 @@ func releaseLocalWorkspace(workspace domain.ActivityWorkspace, path string) erro if err := os.RemoveAll(path); err != nil { return err } - return os.Remove(marker) + return removeWorkspaceMarker(marker) +} + +// Ownership has already been checked. Another releaser may remove the marker +// after that check; an absent marker is the intended terminal cleanup state. +func removeWorkspaceMarker(marker string) error { + err := os.Remove(marker) + if os.IsNotExist(err) { + return nil + } + return err } func pruneLocalWorkspaceInputs(workspace domain.ActivityWorkspace, path string) (domain.WorkspaceReleaseResult, error) { @@ -417,13 +446,14 @@ func workspacePrunePayload(entries []domain.WorkspaceEntry) string { func workspacePruneCommand(path string, workspace domain.ActivityWorkspace) string { marker := workspaceMarkerFor(workspace, path) expected := strings.TrimSpace(string(markerContents(workspace))) - return "set -eu; test -f " + shell(marker) + "; test \"$(cat -- " + shell(marker) + ")\" = " + shell(expected) + `; ` + + return "set -eu; test ! -L " + shell(path) + "; test \"$(realpath -- " + shell(path) + ")\" = " + shell(filepath.Clean(path)) + "; test -f " + shell(marker) + "; test \"$(cat -- " + shell(marker) + ")\" = " + shell(expected) + `; ` + `removed=0; preserved=0; reclaimed=0; while IFS='|' read -r encoded size digest; do ` + `relative=$(printf '%s' "$encoded" | base64 -d); case "$relative" in ''|/*|..|../*|*/../*) echo "unsafe workspace input: $relative" >&2; exit 65;; esac; ` + "target=" + shell(path) + `/"$relative"; [ -e "$target" ] || continue; ` + + `test "$(realpath -- "$target")" = "$target" || { echo 'non-canonical workspace input' >&2; exit 65; }; ` + `if [ -L "$target" ] || [ ! -f "$target" ] || [ "$(wc -c < "$target")" -ne "$size" ]; then preserved=$((preserved+1)); continue; fi; ` + `actual=$(sha256sum "$target" | awk '{print $1}'); if [ "$actual" != "$digest" ]; then preserved=$((preserved+1)); continue; fi; ` + - `rm -f -- "$target"; removed=$((removed+1)); reclaimed=$((reclaimed+size)); done; ` + + workspaceRemovalCommand(workspace) + ` -f -- "$target"; removed=$((removed+1)); reclaimed=$((reclaimed+size)); done; ` + `printf '{"reclaimedBytes":%s,"removedFiles":%s,"preservedFiles":%s}\n' "$reclaimed" "$removed" "$preserved"` } diff --git a/internal/infrastructure/transfer/workspace_lifecycle_test.go b/internal/infrastructure/transfer/workspace_lifecycle_test.go index de5b8583..9f87fac7 100644 --- a/internal/infrastructure/transfer/workspace_lifecycle_test.go +++ b/internal/infrastructure/transfer/workspace_lifecycle_test.go @@ -14,6 +14,49 @@ import ( "github.com/UFFeScience/akoflow/internal/domain" ) +type workspaceEndpointResolverStub struct { + location domain.TransferLocation +} + +func TestPruneCommandScopesCloudPrivilegeToCanonicalVerifiedInputs(t *testing.T) { + workspace := domain.ActivityWorkspace{ID: "workspace-run-1-activity-1", RunID: "run-1", ActivityID: "activity-1", Driver: "cloud-ssh"} + command := workspacePruneCommand("/akoflow/workspace/runs/run-1/activity-1", workspace) + for _, required := range []string{"realpath --", "non-canonical workspace input", "sha256sum", "sudo -n rm -f --"} { + if !strings.Contains(command, required) { + t.Fatalf("missing pruning guard %q", required) + } + } + if strings.Index(command, "sha256sum") > strings.Index(command, "sudo -n rm") { + t.Fatal("privileged removal precedes content verification") + } + workspace.Driver = "ssh" + if strings.Contains(workspacePruneCommand("/home/user/run-1/activity-1", workspace), "sudo") { + t.Fatal("non-cloud pruning escalated privileges") + } +} + +func (stub *workspaceEndpointResolverStub) ResolveTransferEndpoint(_ context.Context, location domain.TransferLocation) (domain.TransferEndpoint, error) { + stub.location = location + return domain.TransferEndpoint{URI: "ssh://worker/" + strings.TrimPrefix(location.URI, "file://")}, nil +} + +func TestWorkspaceEndpointCarriesPersistedCloudInstanceAllocation(t *testing.T) { + resolver := &workspaceEndpointResolverStub{} + manager := ActivityWorkspaceManager{Resolver: resolver} + workspace := domain.ActivityWorkspace{ + ID: "workspace-run-1-activity-1", RunID: "run-1", ActivityID: "activity-1", + ResourceID: "capacity-1", RuntimeID: "cloud-runtime", ConnectionID: "cloud-connection", + URI: "ssh://cloud-connection/akoflow/workspace/runs/run-1/activity-1?cloudInstanceId=instance-1", + ExecutionPath: "/akoflow/workspace/runs/run-1/activity-1", + } + if _, _, err := manager.endpoint(context.Background(), workspace); err != nil { + t.Fatal(err) + } + if resolver.location.CloudInstanceID != "instance-1" { + t.Fatalf("cloud instance allocation was lost: %+v", resolver.location) + } +} + func TestLocalWorkspaceLifecycleRequiresOwnershipMarker(t *testing.T) { root := t.TempDir() path := filepath.Join(root, "run-1", "activity-1") @@ -127,6 +170,10 @@ func digestForTest(value []byte) string { func TestWorkspacePruneShellCommandRemovesOnlyVerifiedInputs(t *testing.T) { root := t.TempDir() + root, err := filepath.EvalSymlinks(root) + if err != nil { + t.Fatal(err) + } path := filepath.Join(root, "run-1", "activity-1") workspace := domain.ActivityWorkspace{ID: "workspace-run-1-activity-1", RunID: "run-1", ActivityID: "activity-1", ExecutionPath: path} @@ -163,3 +210,16 @@ func TestWorkspacePruneShellCommandRemovesOnlyVerifiedInputs(t *testing.T) { t.Fatalf("output changed: %q %v", content, err) } } +func TestInspectRemoteWorkspaceSeparatesSSHWarningsFromUsage(t *testing.T) { + installWorkspaceSSHCheck(t, "2 106346880\n", "mux_client_request_session: session request failed: Session open refused by peer\nControlSocket already exists, disabling multiplexing\n", 0) + m := ActivityWorkspaceManager{Resolver: &workspaceEndpointResolverStub{}} + w := domain.ActivityWorkspace{ID: "workspace-run-1-activity-1", RunID: "run-1", ActivityID: "activity-1", ConnectionID: "ssh", URI: "ssh://worker/home/user/run-1/activity-1", ExecutionPath: "/home/user/run-1/activity-1"} + u, err := m.Inspect(context.Background(), w) + if err != nil || u.FileCount != 2 || u.SizeBytes != 106346880 { + t.Fatalf("usage=%+v err=%v", u, err) + } + installWorkspaceSSHCheck(t, "2 106346880\n", "remote failure\n", 1) + if _, err = m.Inspect(context.Background(), w); err == nil { + t.Fatal("nonzero SSH exit was accepted") + } +} diff --git a/internal/infrastructure/transfer/workspace_merge_script.go b/internal/infrastructure/transfer/workspace_merge_script.go new file mode 100644 index 00000000..6e224281 --- /dev/null +++ b/internal/infrastructure/transfer/workspace_merge_script.go @@ -0,0 +1,93 @@ +package transfer + +import ( + "math" + "strconv" + "strings" +) + +func remoteMergeSeconds(output, stage string) float64 { + prefix := "__AKOFLOW_MERGE_" + stage + "_SECONDS__=" + for _, line := range strings.Split(output, "\n") { + if value, ok := strings.CutPrefix(line, prefix); ok { + n, err := strconv.ParseFloat(value, 64) + if err == nil && n >= 0 && !math.IsNaN(n) && !math.IsInf(n, 0) { + return n + } + } + } + return 0 +} + +// AKO-131: retain the whole-workspace conflict/publication contract without +// forking sha256sum/cat and creating three metadata files for every path. +func remoteWorkspaceScript(sources []string, destination string) string { + args := []string{"python3", "-", destination} + args = append(args, sources...) + for i := range args { + args[i] = shell(args[i]) + } + return strings.Join(args, " ") + " <<'AKOFLOW_MERGE_PY'\n" + remoteWorkspaceMergePython + "\nAKOFLOW_MERGE_PY\n" +} + +const remoteWorkspaceMergePython = `import hashlib, os, stat, subprocess, sys, time +destination, sources = sys.argv[1], sys.argv[2:] +expected = {} +directories = set() + +def digest(path): + h = hashlib.sha256() + with open(path, 'rb') as f: + for chunk in iter(lambda: f.read(1024 * 1024), b''): + h.update(chunk) + return h.digest() + +def tree(root): + if not os.path.isdir(root) or os.path.islink(root): + raise RuntimeError('source workspace disappeared or is a symlink: ' + root) + for parent, dirs, files in os.walk(root, followlinks=False): + for name in dirs: + if os.path.islink(os.path.join(parent, name)): + raise RuntimeError('workspace contains directory symlink: ' + name) + directories.add(os.path.relpath(os.path.join(parent, name), root)) + for name in files: + path = os.path.join(parent, name) + if not stat.S_ISREG(os.lstat(path).st_mode): + raise RuntimeError('workspace contains symlink or special file: ' + name) + yield os.path.relpath(path, root), path + +try: + os.makedirs(destination, exist_ok=True) + started = time.monotonic() + for root in [destination] + sources: + for rel, path in tree(root): + value = digest(path) + previous = expected.get(rel) + if previous is not None and previous[0] != value: + raise RuntimeError('workspace content conflict at ' + rel + ' between ' + previous[1] + ' and ' + root) + expected.setdefault(rel, (value, root)) + if directories.intersection(expected): + raise RuntimeError('workspace file/directory conflict at ' + sorted(directories.intersection(expected))[0]) + print('__AKOFLOW_MERGE_CHECK_SECONDS__=' + str(time.monotonic() - started), flush=True) + started = time.monotonic() + for index, source in enumerate(sources): + print('__AKOFLOW_WORKSPACE_BEGIN_' + str(index) + '__', flush=True) + subprocess.run(['rsync', '--ignore-existing', '-r', '--checksum', '--stats', '--out-format=%i %n', + '--no-perms', '--no-owner', '--no-group', '--no-times', '--', + os.path.join(source, ''), os.path.join(destination, '')], check=True) + print('__AKOFLOW_WORKSPACE_END_' + str(index) + '__', flush=True) + print('__AKOFLOW_MERGE_COPY_SECONDS__=' + str(time.monotonic() - started), flush=True) + started = time.monotonic() + actual_paths = set() + for rel, path in tree(destination): + actual_paths.add(rel) + if rel not in expected or digest(path) != expected[rel][0]: + raise RuntimeError('workspace verification failed at ' + rel) + if actual_paths != set(expected): + raise RuntimeError('workspace verification failed: missing destination paths') + print('__AKOFLOW_MERGE_VERIFIED_FILES__=' + str(len(actual_paths)), flush=True) + print('__AKOFLOW_MERGE_VERIFY_SECONDS__=' + str(time.monotonic() - started), flush=True) +except Exception as error: + print(str(error), file=sys.stderr) + sys.exit(45) +` diff --git a/internal/infrastructure/transfer/workspace_merge_script_test.go b/internal/infrastructure/transfer/workspace_merge_script_test.go new file mode 100644 index 00000000..03b716f9 --- /dev/null +++ b/internal/infrastructure/transfer/workspace_merge_script_test.go @@ -0,0 +1,89 @@ +package transfer + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestSingleProcessMergePreservesWholeWorkspaceAndRejectsConflicts(t *testing.T) { + for _, mode := range []string{"mixed", "conflict", "symlink", "directory-conflict"} { + t.Run(mode, func(t *testing.T) { + root := t.TempDir() + a, b, dest := filepath.Join(root, "a"), filepath.Join(root, "b"), filepath.Join(root, "destination") + for _, path := range []string{a, b, dest} { + if err := os.MkdirAll(path, 0700); err != nil { + t.Fatal(err) + } + } + name := "space ' quote\nnewline.fits" + for _, path := range []string{a, b} { + if err := os.WriteFile(filepath.Join(path, name), []byte("same"), 0600); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(a, "only-a"), []byte("a"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(b, "only-b"), []byte("b"), 0600); err != nil { + t.Fatal(err) + } + if mode == "conflict" { + if err := os.WriteFile(filepath.Join(b, name), []byte("different"), 0600); err != nil { + t.Fatal(err) + } + } + if mode == "symlink" { + if err := os.Symlink(filepath.Join(a, name), filepath.Join(b, "link")); err != nil { + t.Fatal(err) + } + } + if mode == "directory-conflict" { + if err := os.MkdirAll(filepath.Join(b, "only-a"), 0700); err != nil { + t.Fatal(err) + } + } + output, err := exec.Command("bash", "-c", remoteWorkspaceScript([]string{a, b}, dest)).CombinedOutput() + if mode != "mixed" { + if err == nil { + t.Fatalf("unsafe source accepted: %s", output) + } + entries, readErr := os.ReadDir(dest) + if readErr != nil || len(entries) != 0 { + t.Fatal("failure published files before validation") + } + return + } + if err != nil { + t.Fatalf("merge: %v %s", err, output) + } + if !strings.Contains(string(output), "__AKOFLOW_MERGE_VERIFIED_FILES__=3") { + t.Fatalf("whole workspace not verified: %s", output) + } + // Recovery must safely reuse the already verified whole workspace. + if output, err := exec.Command("bash", "-c", remoteWorkspaceScript([]string{a, b}, dest)).CombinedOutput(); err != nil { + t.Fatalf("idempotent recovery: %v %s", err, output) + } + if err := os.WriteFile(filepath.Join(dest, name), []byte("consumer changed"), 0600); err != nil { + t.Fatal(err) + } + content, err := os.ReadFile(filepath.Join(a, name)) + if err != nil || string(content) != "same" { + t.Fatal("consumer copy modified producer") + } + }) + } +} + +func TestRemoteMergeSecondsRejectsInvalidTelemetry(t *testing.T) { + for _, value := range []string{"-1", "NaN", "+Inf", "garbage"} { + if got := remoteMergeSeconds("__AKOFLOW_MERGE_CHECK_SECONDS__="+value, "CHECK"); got != 0 { + t.Fatalf("accepted %s: %v", value, got) + } + } + if got := remoteMergeSeconds("__AKOFLOW_MERGE_CHECK_SECONDS__=1.25\n", "CHECK"); got != 1.25 { + t.Fatalf("got %v", got) + } +} diff --git a/internal/infrastructure/transfer/workspace_peer_archive.go b/internal/infrastructure/transfer/workspace_peer_archive.go new file mode 100644 index 00000000..d3d19727 --- /dev/null +++ b/internal/infrastructure/transfer/workspace_peer_archive.go @@ -0,0 +1,250 @@ +package transfer + +import ( + "context" + "errors" + "fmt" + "net/url" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +var peerArchiveDigest = regexp.MustCompile(`^[a-f0-9]{64}$`) + +// preparePeerWorkspaceArchive keeps payload on the two workers. The probe +// writes only an empty file in the private preparation directory. Failure of +// that probe leaves the gateway fallback available; payload/validation errors +// after a successful probe are surfaced, never silently retried as another route. +func preparePeerWorkspaceArchive(ctx context.Context, source, destination domain.TransferEndpoint, run *domain.DataTransferRun) (bool, error) { + if !cloudWorkspacePeerEligible(source, destination) { + return false, nil + } + sourceHost, _, err := sshTarget(source, "") + if err != nil { + return true, err + } + targetHost, target, err := sshTarget(destination, "") + if err != nil { + return true, err + } + output, err := runSSHStdout(ctx, source, append(sshArgs(source), sourceHost, "mktemp -d /tmp/akoflow-peer-XXXXXX"), nil) + if err != nil { + return false, nil + } + root := strings.TrimSpace(string(output)) + if !filepath.IsAbs(root) || filepath.Dir(root) != "/tmp" || !strings.HasPrefix(filepath.Base(root), "akoflow-peer-") { + return true, fmt.Errorf("invalid peer preparation directory") + } + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + _, _ = runSSHCombinedOutput(cleanupCtx, source, append(sshArgs(source), sourceHost, "rm -rf -- "+shell(root)), nil) + }() + archiveSource := source + u, _ := url.Parse(source.URI) + u.Path = root + archiveSource.URI = u.String() + if err := makeWorkspaceDirectory(ctx, destination); err != nil { + return true, err + } + if _, err := runSSHCombinedOutput(ctx, source, append(sshArgs(source), sourceHost, "umask 077; touch -- "+shell(filepath.Join(root, "probe"))), nil); err != nil { + return true, err + } + if ready, err := probePeerWorkspaceConnectivity(ctx, archiveSource, destination, targetHost, target, run); !ready || err != nil { + return ready, err + } + if err := rejectWorkspaceSymlinks(ctx, source); err != nil { + return true, err + } + snapshot, release, err := acquirePeerWorkspaceSnapshot(ctx, source, run) + if err != nil { + return true, err + } + defer release() + if run.Route.SnapshotReused { + if err := measureWorkspacePhase(run, "peer-reused-source-manifest-check", func() error { return verifyPeerSnapshotSource(ctx, source) }); err != nil { + return true, err + } + } + u.Path = snapshot.root + archiveSource.URI = u.String() + run.Route.ArchiveBytes = snapshot.size + err = measureWorkspacePhase(run, "peer-private-upload", func() error { + _, err := (RsyncSSH{}).TransferRoute(ctx, domain.TransferDirectRuntime, archiveSource, destination, "workspace.tar.gz", "peer-workspace.tar.gz", 0) + return err + }) + if err != nil { + run.Route.NetworkBytesIncomplete = true + return true, err + } + run.NetworkBytes = snapshot.size + return true, receivePeerWorkspaceArchive(ctx, destination, targetHost, target, snapshot, run) +} + +// Persist only bounded categories: SSH stderr may contain addresses, paths or +// remote diagnostic text that must not be promoted into scientific metadata. +func peerProbeFailureCategory(err error) string { + if errors.Is(err, context.DeadlineExceeded) { + return "probe_timeout" + } + message := strings.ToLower(err.Error()) + for _, stage := range []struct{ fragment, category string }{ + {"install temporary destination credential", "destination_credential_installation_failed"}, + {"install temporary source credential", "source_credential_installation_failed"}, + {"install verified direct destination host key", "source_host_key_installation_failed"}, + {"read verified direct destination host key", "verified_host_key_unavailable"}, + {"host key verification failed", "host_key_rejected"}, + {"invalid format", "identity_format_rejected"}, + {"error in libcrypto", "identity_format_rejected"}, + {"permission denied", "authentication_rejected"}, + {"connection refused", "connection_refused"}, + {"timed out", "probe_timeout"}, + } { + if strings.Contains(message, stage.fragment) { + return stage.category + } + } + return "ssh_probe_failed" +} + +// Validate every member before publishing even a single extracted file. Only +// regular files and directories are accepted; links, duplicates and path +// traversal cannot escape the private, mktemp-based merge preparation root. +const peerArchiveVerifyScript = `import gzip, hashlib, os, posixpath, shutil, sys, tarfile +archive, root, digest = sys.argv[1:] +hasher = hashlib.sha256() +with open(archive, 'rb') as f: + for block in iter(lambda: f.read(65536), b''): hasher.update(block) +if hasher.hexdigest() != digest: raise ValueError('peer archive digest mismatch') +with gzip.open(archive, 'rb') as stream: + with tarfile.open(fileobj=stream, mode='r:') as tar: + members = tar.getmembers() + seen = {} + for member in members: + name = posixpath.normpath(member.name) + if member.name.startswith('/') or '..' in member.name.split('/') or name == '..' or name.startswith('../'): + raise ValueError('unsafe peer archive path') + if member.type not in (tarfile.REGTYPE, tarfile.AREGTYPE, tarfile.DIRTYPE) or name in seen or (name == '.' and not member.isdir()): + raise ValueError('unsupported or duplicate peer archive member') + seen[name] = member.isdir() + for name in seen: + parent = posixpath.dirname(name) + while parent not in ('', '.'): + if parent in seen and not seen[parent]: raise ValueError('archive parent is a file') + parent = posixpath.dirname(parent) + for block in iter(lambda: stream.read(65536), b''): + if any(block): raise ValueError('unexpected peer archive trailer') + # Use a separate extraction tree so the probe/archive themselves never + # become scientific workspace inputs. + output = os.path.join(root, 'contents') + os.mkdir(output, 0o700) + # Reopen sequentially after complete validation. Random extractfile + # access on gzip otherwise replays decompression after the scan. +with tarfile.open(archive, mode='r|gz') as tar: + for member in tar: + name = posixpath.normpath(member.name) + if name == '.': continue + destination = os.path.join(output, name) + if member.isdir(): + os.makedirs(destination, mode=0o700, exist_ok=True) + else: + os.makedirs(os.path.dirname(destination), mode=0o700, exist_ok=True) + fd = os.open(destination, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, member.mode & 0o777) + with os.fdopen(fd, 'wb') as f, tar.extractfile(member) as source: shutil.copyfileobj(source, f, 65536) +os.unlink(archive) +os.unlink(os.path.join(root, 'peer-probe')) +` + +func acquirePeerWorkspaceSnapshot(ctx context.Context, source domain.TransferEndpoint, run *domain.DataTransferRun) (*peerSnapshot, func(), error) { + var snapshot *peerSnapshot + var release func() + err := measureWorkspacePhase(run, "peer-snapshot-acquire", func() error { + var reused bool + var acquireErr error + acquireStarted := time.Now() + snapshot, reused, release, acquireErr = sharedPeerSnapshots.acquire(ctx, peerSnapshotKey(source), func(buildCtx context.Context) (*peerSnapshot, error) { + return buildPeerSnapshot(buildCtx, source) + }) + run.Route.SnapshotReused = reused + if acquireErr == nil && snapshot != nil { + run.Route.SnapshotBuildWallSeconds = snapshot.buildWallSeconds + waitSeconds, commandSeconds := snapshot.buildSSHWaitSeconds, snapshot.buildCommandSeconds + run.Route.SnapshotBuildSSHWaitSeconds = &waitSeconds + run.Route.SnapshotBuildCommandSeconds = &commandSeconds + if snapshot.serviceTiming != nil { + serviceSeconds, cpuSeconds := snapshot.serviceTiming.WallSeconds, snapshot.serviceTiming.CPUSeconds + run.Route.SnapshotBuildServiceSeconds = &serviceSeconds + run.Route.SnapshotBuildCPUSeconds = &cpuSeconds + } + wait := time.Since(acquireStarted).Seconds() + if !reused { + wait -= snapshot.buildWallSeconds + } + if wait < 0 { + wait = 0 + } + run.Route.SnapshotAcquireWaitSeconds = wait + } + return acquireErr + }) + if err != nil { + return nil, nil, err + } + return snapshot, release, nil +} + +func receivePeerWorkspaceArchive(ctx context.Context, destination domain.TransferEndpoint, targetHost, target string, snapshot *peerSnapshot, run *domain.DataTransferRun) error { + var output []byte + command := "python3 -c " + shell(peerArchiveVerifyScript) + " " + shell(filepath.Join(target, "peer-workspace.tar.gz")) + " " + shell(target) + " " + shell(snapshot.digest) + command = "python3 -c " + shell(peerSnapshotTimingScript) + " " + shell(command) + var receiverWait, receiverCommand float64 + err := measureWorkspacePhase(run, "peer-receiver-verify-and-extract", func() error { + var commandErr error + output, commandErr = runSSHProtocolMeasured(ctx, destination, append(sshArgs(destination), targetHost, command), &receiverWait, &receiverCommand) + return commandErr + }) + phase := &run.Route.Phases[len(run.Route.Phases)-1] + phase.SSHChannelWaitSeconds = &receiverWait + phase.ExecutionSeconds = &receiverCommand + if err != nil { + return fmt.Errorf("verify private peer archive: %w: %s", err, strings.TrimSpace(string(output))) + } + if len(strings.TrimSpace(string(output))) > 0 { + timing, timingErr := parsePeerSnapshotServiceTiming(strings.TrimSpace(string(output))) + if timingErr != nil { + return timingErr + } + run.Route.PeerReceiverServiceSeconds = &timing.WallSeconds + run.Route.PeerReceiverCPUSeconds = &timing.CPUSeconds + } + return rejectWorkspaceSymlinks(ctx, destination) +} + +func probePeerWorkspaceConnectivity(ctx context.Context, archiveSource, destination domain.TransferEndpoint, targetHost, target string, run *domain.DataTransferRun) (bool, error) { + probeCtx, cancel := context.WithTimeout(ctx, 45*time.Second) + probeErr := measureWorkspacePhase(run, "peer-authenticated-connectivity-probe", func() error { + _, err := (RsyncSSH{}).TransferRoute(probeCtx, domain.TransferDirectRuntime, archiveSource, destination, "probe", "peer-probe", 0) + return err + }) + cancel() + if probeErr != nil { + run.Route.PeerProbeFailure = peerProbeFailureCategory(probeErr) + cleanupCtx, release := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + _, cleanupErr := runSSHCombinedOutput(cleanupCtx, destination, append(sshArgs(destination), targetHost, "rm -f -- "+shell(filepath.Join(target, "peer-probe"))), nil) + release() + if cleanupErr != nil { + return true, fmt.Errorf("clean failed peer probe: %w", cleanupErr) + } + if ctx.Err() != nil { + return true, ctx.Err() + } + run.Route.Fallback = domain.TransferGateway + run.Route.Reason = "private peer authentication/connectivity not proven; gateway fallback" + return false, nil + } + return true, nil +} diff --git a/internal/infrastructure/transfer/workspace_peer_archive_test.go b/internal/infrastructure/transfer/workspace_peer_archive_test.go new file mode 100644 index 00000000..78f8eed1 --- /dev/null +++ b/internal/infrastructure/transfer/workspace_peer_archive_test.go @@ -0,0 +1,177 @@ +package transfer + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestPeerProbeDiagnosticsDoNotPublishRawSSHText(t *testing.T) { + for _, tc := range []struct { + err error + want string + }{ + {context.DeadlineExceeded, "probe_timeout"}, + {fmt.Errorf("stream directly between cloud instances: worker@private.example: Permission denied (publickey)."), "authentication_rejected"}, + {fmt.Errorf("install temporary destination credential: Permission denied secret-path"), "destination_credential_installation_failed"}, + {fmt.Errorf("Host key verification failed for private.example"), "host_key_rejected"}, + {fmt.Errorf("Load key secret-path: invalid format"), "identity_format_rejected"}, + {fmt.Errorf("unexpected SSH text containing credential-data"), "ssh_probe_failed"}, + } { + if got := peerProbeFailureCategory(tc.err); got != tc.want { + t.Fatalf("diagnostic category %q want %q", got, tc.want) + } + } +} + +func TestPeerArchiveRejectsDamagedGzipBeforePublishing(t *testing.T) { + python, err := exec.LookPath("python3") + if err != nil { + t.Fatal(err) + } + var buffer bytes.Buffer + gz := gzip.NewWriter(&buffer) + tw := tar.NewWriter(gz) + if err := tw.WriteHeader(&tar.Header{Name: "file", Mode: 0600, Size: 1, Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte("x")); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + for _, kind := range []string{"truncated", "bad-crc"} { + t.Run(kind, func(t *testing.T) { + data := append([]byte(nil), buffer.Bytes()...) + if kind == "truncated" { + data = data[:len(data)-4] + } else { + data[len(data)-8] ^= 1 + } + root := t.TempDir() + archive := filepath.Join(root, "peer-workspace.tar.gz") + if err := os.WriteFile(archive, data, 0600); err != nil { + t.Fatal(err) + } + digest := fmt.Sprintf("%x", sha256.Sum256(data)) + if output, err := exec.Command(python, "-c", peerArchiveVerifyScript, archive, root, digest).CombinedOutput(); err == nil { + t.Fatalf("damaged gzip accepted: %s", output) + } + if _, err := os.Stat(filepath.Join(root, "contents")); !os.IsNotExist(err) { + t.Fatal("published damaged archive") + } + }) + } +} + +func TestPeerArchiveVerifiesBeforeExtraction(t *testing.T) { + python, err := exec.LookPath("python3") + if err != nil { + t.Fatal("Python receiver runtime is required for this integration test") + } + for _, tc := range []struct { + name string + headers []*tar.Header + wrongHash bool + wantError bool + }{ + {"regular", []*tar.Header{{Name: "file", Mode: 0600, Size: 1, Typeflag: tar.TypeReg}}, false, false}, + {"traversal", []*tar.Header{{Name: "../escape", Size: 1, Typeflag: tar.TypeReg}}, false, true}, + {"absolute", []*tar.Header{{Name: "/escape", Size: 1, Typeflag: tar.TypeReg}}, false, true}, + {"symlink", []*tar.Header{{Name: "link", Linkname: "../escape", Typeflag: tar.TypeSymlink}}, false, true}, + {"hardlink", []*tar.Header{{Name: "link", Linkname: "file", Typeflag: tar.TypeLink}}, false, true}, + {"duplicate", []*tar.Header{{Name: "file", Size: 1, Typeflag: tar.TypeReg}, {Name: "file", Size: 1, Typeflag: tar.TypeReg}}, false, true}, + {"parent file", []*tar.Header{{Name: "parent/child", Size: 1, Typeflag: tar.TypeReg}, {Name: "parent", Size: 1, Typeflag: tar.TypeReg}}, false, true}, + {"digest mismatch", []*tar.Header{{Name: "file", Size: 1, Typeflag: tar.TypeReg}}, true, true}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + archive, digest := writePeerArchiveFixture(t, root, tc.headers) + if tc.wrongHash { + digest = fmt.Sprintf("%064d", 0) + } + if err := os.WriteFile(filepath.Join(root, "peer-probe"), nil, 0600); err != nil { + t.Fatal(err) + } + // Reject random-access extraction: a regression would repeatedly seek + // through gzip after the complete validation pass. + streamGuard := `import tarfile +original_extractfile = tarfile.TarFile.extractfile +def streaming_extractfile(self, member): + if not isinstance(self.fileobj, tarfile._Stream): + raise AssertionError('extraction must use sequential streaming') + return original_extractfile(self, member) +tarfile.TarFile.extractfile = streaming_extractfile +` + output, err := exec.Command(python, "-c", streamGuard+peerArchiveVerifyScript, archive, root, digest).CombinedOutput() + if tc.wantError { + if err == nil { + t.Fatal("unsafe archive accepted") + } + if _, err := os.Stat(filepath.Join(root, "contents")); !os.IsNotExist(err) { + t.Fatal("published extraction before full validation") + } + return + } + if err != nil { + t.Fatalf("receiver failed: %v: %s", err, output) + } + content, err := os.ReadFile(filepath.Join(root, "contents", "file")) + if err != nil || string(content) != "x" { + t.Fatalf("wrong extracted content: %q %v", content, err) + } + for _, name := range []string{archive, filepath.Join(root, "peer-probe")} { + if _, err := os.Stat(name); !os.IsNotExist(err) { + t.Fatal("transport files must not become workspace inputs") + } + } + }) + } +} + +func writePeerArchiveFixture(t *testing.T, root string, headers []*tar.Header) (string, string) { + t.Helper() + archive := filepath.Join(root, "peer-workspace.tar.gz") + f, err := os.Create(archive) + if err != nil { + t.Fatal(err) + } + gz := gzip.NewWriter(f) + writer := tar.NewWriter(gz) + for _, header := range headers { + if err := writer.WriteHeader(header); err != nil { + t.Fatal(err) + } + if header.Size > 0 { + if _, err := writer.Write([]byte("x")); err != nil { + t.Fatal(err) + } + } + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(archive) + if err != nil { + t.Fatal(err) + } + digest := fmt.Sprintf("%x", sha256.Sum256(data)) + return archive, digest +} diff --git a/internal/infrastructure/transfer/workspace_routes.go b/internal/infrastructure/transfer/workspace_routes.go new file mode 100644 index 00000000..8b3e94f3 --- /dev/null +++ b/internal/infrastructure/transfer/workspace_routes.go @@ -0,0 +1,289 @@ +package transfer + +import ( + "context" + "fmt" + "net/url" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +// A same-instance route requires independently resolved generation trust, not +// merely equal public addresses or a shared capacity target. +func sameVerifiedCloudWorkspaceEndpoint(source, destination domain.TransferEndpoint) bool { + if source.CloudInstanceID == "" || source.CloudInstanceID != destination.CloudInstanceID || !sameSSHWorkspaceEndpoint(source, destination) { + return false + } + a, errA := url.Parse(source.URI) + b, errB := url.Parse(destination.URI) + if errA != nil || errB != nil { + return false + } + x, y := a.Query(), b.Query() + return x.Get("strictHostKeyChecking") == "true" && y.Get("strictHostKeyChecking") == "true" && + x.Get("hostKeyAlias") != "" && x.Get("hostKeyAlias") == y.Get("hostKeyAlias") && + x.Get("knownHostsFile") != "" && x.Get("knownHostsFile") == y.Get("knownHostsFile") +} + +func sameCloudWorkspaceProof(ctx context.Context, source, destination domain.TransferEndpoint) (string, bool) { + if !sameVerifiedCloudWorkspaceEndpoint(source, destination) { + return "", false + } + host, target, err := sshTarget(destination, "") + if err != nil { + return "", false + } + _, origin, err := sshTarget(source, "") + if err != nil { + return "", false + } + output, err := runSSHCombinedOutput(ctx, destination, append(sshArgs(destination), host, sharedWorkspaceProofCommand(origin, target)), nil) + if err != nil { + return "", false + } + lines := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(lines) != 2 || lines[0] != lines[1] { + return "", false + } + fields := strings.Fields(lines[0]) + if len(fields) != 5 || fields[1] == "" || fields[3] == "" || !strings.HasPrefix(fields[4], "/") { + return "", false + } + return fields[0] + " " + strings.Join(fields[1:], "|"), true +} + +// sharedWorkspaceProof verifies access and filesystem identity at execution +// time. A matching SSH connection or a zero filesystem ID alone is not proof +// of shared storage: the mounted source and mount path must also match. +func sharedWorkspaceProof(ctx context.Context, source, destination domain.TransferEndpoint) (string, bool) { + proof, ok, _ := sharedWorkspaceProofDiagnostic(ctx, source, destination) + return proof, ok +} + +// Never persist command output, addresses, or paths in proof failure metadata. +func sharedWorkspaceProofDiagnostic(ctx context.Context, source, destination domain.TransferEndpoint) (string, bool, string) { + if !sameSSHWorkspaceEndpoint(source, destination) { + return "", false, "different_ssh_endpoints" + } + host, target, err := sshTarget(destination, "") + if err != nil { + return "", false, "invalid_target_endpoint" + } + _, origin, err := sshTarget(source, "") + if err != nil { + return "", false, "invalid_source_endpoint" + } + command := sharedWorkspaceProofCommand(origin, target) + output, err := runSSHCombinedOutput(ctx, destination, append(sshArgs(destination), host, command), nil) + if err != nil { + return "", false, "proof_command_failed" + } + proof, ok := parseSharedWorkspaceProof(string(output)) + if !ok { + return "", false, "filesystem_identity_or_format_rejected" + } + return proof, true, "" +} + +func sharedWorkspaceProofCommand(origin, target string) string { + return "set -eu; test -d " + shell(origin) + "; test -r " + shell(origin) + + "; test -x " + shell(origin) + "; test -d " + shell(target) + "; test -w " + shell(target) + + "; for workspace in " + shell(origin) + " " + shell(target) + "; do " + + "stat -f -c '%T %i' -- \"$workspace\" | tr '\\n' ' '; " + + "findmnt -rn -T \"$workspace\" -o FSTYPE,SOURCE,TARGET; done" +} + +func parseSharedWorkspaceProof(output string) (string, bool) { + lines := strings.Split(strings.TrimSpace(output), "\n") + if len(lines) != 2 || lines[0] != lines[1] { + return "", false + } + fields := strings.Fields(lines[0]) + if len(fields) != 5 || fields[1] == "" || fields[3] == "" || !strings.HasPrefix(fields[4], "/") { + return "", false + } + compatible := false + switch fields[0] { + case "lustre": + compatible = fields[2] == "lustre" + case "nfs", "nfs4": + compatible = fields[2] == "nfs" || fields[2] == "nfs4" + case "beegfs", "fhgfs": + compatible = fields[2] == "beegfs" || fields[2] == "fhgfs" + } + if !compatible { + return "", false + } + return fields[0] + " " + strings.Join(fields[1:], "|"), true +} + +func (s WorkspaceRsync) syncDependencyRoutes(ctx context.Context, plans []domain.DataTransferPlan, sources []domain.TransferEndpoint, destination domain.TransferEndpoint, runs []domain.DataTransferRun) ([]domain.DataTransferRun, error) { + if err := makeWorkspaceDirectory(ctx, destination); err != nil { + return runs, err + } + host, target, err := sshTarget(destination, "") + if err != nil { + return runs, err + } + prepared := append([]domain.TransferEndpoint(nil), sources...) + proofs := make([]string, len(plans)) + localCloud := make([]bool, len(plans)) + privatePeers := make([]bool, len(plans)) + network := make([]rsyncStats, len(plans)) + var remoteRoot string + defer func() { + if remoteRoot != "" { + // Only the exact mktemp-generated preparation directory is removed. + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + _, _ = runSSHCombinedOutput(cleanupCtx, destination, append(sshArgs(destination), host, "rm -rf -- "+shell(remoteRoot)), nil) + } + }() + for i, source := range sources { + runs[i].Route.SourceAddress = workspaceRouteAddress(source.URI) + runs[i].Route.TargetAddress = workspaceRouteAddress(destination.URI) + proof, ok := sameCloudWorkspaceProof(ctx, source, destination) + localCloud[i] = ok + if !ok { + proof, ok, runs[i].Route.SharedStorageProofFailure = sharedWorkspaceProofDiagnostic(ctx, source, destination) + } + if ok { + proofs[i] = proof + identity := strings.Fields(proof) + runs[i].Route.SharedFilesystem = identity[0] + runs[i].Route.SharedFilesystemID = identity[1] + runs[i].Strategy, runs[i].Route.Strategy = domain.TransferDirectRuntime, domain.TransferDirectRuntime + runs[i].Route.Reason = "verified shared filesystem: " + proof + if localCloud[i] { + runs[i].Strategy, runs[i].Route.Strategy = domain.TransferRuntimeLocal, domain.TransferRuntimeLocal + runs[i].Route.Reason = "verified same concrete cloud instance and accessible workspace filesystem: " + proof + } + continue + } + runs[i].Route.Reason = "per-dependency staging fallback; shared storage access not proven" + if strings.HasPrefix(source.URI, "file://") { + runs[i].Strategy, runs[i].Route.Strategy = domain.TransferDirectRuntime, domain.TransferDirectRuntime + runs[i].Route.Reason = "network rsync from local producer to remote merge preparation" + } + if remoteRoot == "" { + remoteRoot, err = createRemoteMergeDirectory(ctx, destination, host, target) + if err != nil { + return runs, err + } + } + remoteURL, _ := url.Parse(destination.URI) + remoteURL.Path = filepath.Join(remoteRoot, strconv.Itoa(i)) + prepared[i] = destination + prepared[i].URI = remoteURL.String() + } + // Separate staging directories allow concurrent preparation without concurrent + // writes into the consumer workspace. Cancellation waits for every worker. + barrierStart := time.Now() + if err := parallelWorkspacePreparation(ctx, len(sources), s.Parallelism, func(ctx context.Context, i int) error { + if proofs[i] != "" { + return nil + } + peer, err := prepareDependencyArchive(ctx, sources[i], &prepared[i], &runs[i]) + privatePeers[i] = peer + network[i].bytes = runs[i].NetworkBytes + return err + }); err != nil { + return runs, err + } + barrierEnd := time.Now() + recordSharedPreparationBarrier(runs, barrierStart, barrierEnd) + // Validate all paths/content together before any successor publication. + result, err := syncRemoteWorkspaceDirect(ctx, plans, prepared, destination, runs) + annotateDependencyRoutes(sources, proofs, localCloud, privatePeers, network, result) + return result, err +} + +func createRemoteMergeDirectory(ctx context.Context, destination domain.TransferEndpoint, host, target string) (string, error) { + output, err := runSSHCombinedOutput(ctx, destination, append(sshArgs(destination), host, + "mktemp -d -- "+shell(filepath.Join(filepath.Dir(target), ".akoflow-merge-XXXXXX"))), nil) + if err != nil { + return "", fmt.Errorf("prepare remote merge: %w", err) + } + root := strings.TrimSpace(string(output)) + if !filepath.IsAbs(root) || filepath.Dir(root) != filepath.Dir(target) || !strings.HasPrefix(filepath.Base(root), ".akoflow-merge-") { + return "", fmt.Errorf("invalid remote preparation directory") + } + return root, nil +} + +func recordSharedPreparationBarrier(runs []domain.DataTransferRun, start, end time.Time) { + for i := range runs { + runs[i].Route.Phases = append(runs[i].Route.Phases, domain.TransferPhase{ + Name: "shared-preparation-barrier", StartedAt: float64(start.UnixNano()) / 1e9, + FinishedAt: float64(end.UnixNano()) / 1e9, DurationSeconds: end.Sub(start).Seconds(), + }) + } +} + +// Route evidence must never serialize SSH credentials or URI query options. +func workspaceRouteAddress(raw string) string { + address, err := url.Parse(raw) + if err != nil { + return "" + } + address.User = nil + address.RawQuery = "" + address.Fragment = "" + return address.String() +} + +func prepareDependencyArchive(ctx context.Context, source domain.TransferEndpoint, prepared *domain.TransferEndpoint, run *domain.DataTransferRun) (bool, error) { + peer := false + start := float64(time.Now().UnixNano()) / 1e9 + (*run).StartedAt = start + (*run).Route.PreparationStartedAt = start + (*run).Route.Packaging = "tar-gzip" + usedPeer, err := preparePeerWorkspaceArchive(ctx, source, (*prepared), &(*run)) + if usedPeer && err == nil { + peer = true + preparedURL, _ := url.Parse((*prepared).URI) + preparedURL.Path = filepath.Join(preparedURL.Path, "contents") + (*prepared).URI = preparedURL.String() + } else if !usedPeer && err == nil { + err = relayWorkspaceArchive(ctx, source, (*prepared), &(*run)) + } + end := float64(time.Now().UnixNano()) / 1e9 + (*run).Route.PreparationFinishedAt = end + (*run).Route.PreparationSeconds = end - start + return peer, err +} + +func annotateDependencyRoutes(sources []domain.TransferEndpoint, proofs []string, localCloud, privatePeers []bool, network []rsyncStats, result []domain.DataTransferRun) { + for i := range result { + result[i].NetworkBytes = network[i].bytes + result[i].Route.FilesTransferred = result[i].FilesTransferred + serverBytes := network[i].bytes + if proofs[i] != "" || privatePeers[i] { + serverBytes = 0 + } + result[i].Route.ServerNetworkBytes = &serverBytes + if proofs[i] != "" { + result[i].Route.Reason = "verified shared filesystem: " + proofs[i] + if localCloud[i] { + result[i].Strategy, result[i].Route.Strategy = domain.TransferRuntimeLocal, domain.TransferRuntimeLocal + result[i].Route.Reason = "verified same concrete cloud instance and accessible workspace filesystem: " + proofs[i] + } + } else { + result[i].Strategy, result[i].Route.Strategy = domain.TransferGateway, domain.TransferGateway + result[i].Route.Reason = "per-dependency staging fallback; shared storage access not proven" + if privatePeers[i] { + result[i].Strategy, result[i].Route.Strategy = domain.TransferDirectRuntime, domain.TransferDirectRuntime + result[i].Route.NetworkDomain = sources[i].NetworkDomain + result[i].Route.Reason = "verified private cloud peer generation and authenticated whole-workspace archive transfer" + } + if strings.HasPrefix(sources[i].URI, "file://") { + result[i].Strategy, result[i].Route.Strategy = domain.TransferDirectRuntime, domain.TransferDirectRuntime + result[i].Route.Reason = "network rsync from local producer to remote merge preparation" + } + } + } +} diff --git a/internal/infrastructure/transfer/workspace_routes_test.go b/internal/infrastructure/transfer/workspace_routes_test.go new file mode 100644 index 00000000..79b6d48a --- /dev/null +++ b/internal/infrastructure/transfer/workspace_routes_test.go @@ -0,0 +1,359 @@ +package transfer + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestWorkspacePhaseRecordsFailureWithoutChangingError(t *testing.T) { + var run domain.DataTransferRun + want := errors.New("transport interrupted") + if got := measureWorkspacePhase(&run, "archive-upload", func() error { return want }); got != want { + t.Fatalf("error changed: %v", got) + } + phase := run.Route.Phases[0] + if !phase.Failed || phase.Name != "archive-upload" || phase.FinishedAt < phase.StartedAt || phase.DurationSeconds < 0 { + t.Fatalf("invalid phase: %+v", phase) + } +} + +func TestSharedStorageProofFailureIsBoundedAndDoesNotRelaxTrust(t *testing.T) { + a := domain.TransferEndpoint{URI: "ssh://user@host-a/workspace", ConnectionID: "fixture"} + b := domain.TransferEndpoint{URI: "ssh://user@host-b/workspace", ConnectionID: "fixture"} + proof, ok, category := sharedWorkspaceProofDiagnostic(context.Background(), a, b) + if proof != "" || ok || category != "different_ssh_endpoints" { + t.Fatalf("unsafe endpoint proof: %q %v %q", proof, ok, category) + } + bin := t.TempDir() + if err := os.WriteFile(filepath.Join(bin, "ssh"), []byte("#!/bin/sh\nprintf 'private diagnostic' >&2\nexit 7\n"), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + proof, ok, category = sharedWorkspaceProofDiagnostic(context.Background(), a, a) + if proof != "" || ok || category != "proof_command_failed" { + t.Fatalf("command failure leaked or accepted: %q %v %q", proof, ok, category) + } +} + +func TestReceiverPhaseSeparatesSSHAdmissionAndCommandService(t *testing.T) { + bin := t.TempDir() + if err := os.WriteFile(filepath.Join(bin, "ssh"), []byte("#!/bin/sh\nprintf 'receiver-fixture'\nexit \"$1\"\n"), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + for _, code := range []string{"0", "7"} { + var run domain.DataTransferRun + output, err := measureWorkspaceSSHPhase(context.Background(), &run, "receiver-verify-and-extract", domain.TransferEndpoint{}, []string{code}) + phase := run.Route.Phases[0] + if string(output) != "receiver-fixture" || (err != nil) != (code == "7") || phase.Failed != (code == "7") { + t.Fatalf("lost command output or failure: phase=%+v err=%v", phase, err) + } + if phase.ExecutionSeconds == nil || phase.SSHChannelWaitSeconds == nil || *phase.ExecutionSeconds <= 0 || *phase.SSHChannelWaitSeconds < 0 { + t.Fatalf("missing measured receiver service: %+v", phase) + } + if *phase.ExecutionSeconds+*phase.SSHChannelWaitSeconds > phase.DurationSeconds+0.001 { + t.Fatalf("service and queue exceed phase wall: %+v", phase) + } + } +} + +func TestRemoteWorkspaceVerificationReadsUniqueDestinationOnce(t *testing.T) { + if _, err := exec.LookPath("python3"); err != nil { + t.Fatal("python3 is required for the remote-script fixture") + } + for _, corrupt := range []bool{false, true} { + t.Run(fmt.Sprint("corrupt=", corrupt), func(t *testing.T) { + root := t.TempDir() + bin, dest := filepath.Join(root, "bin"), filepath.Join(root, "dest") + if err := os.Mkdir(bin, 0700); err != nil { + t.Fatal(err) + } + var sources []string + for i := 0; i < 3; i++ { + source := filepath.Join(root, fmt.Sprint("source", i)) + if err := os.Mkdir(source, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(source, "shared.fits"), []byte("same content"), 0600); err != nil { + t.Fatal(err) + } + sources = append(sources, source) + } + // Only copying is a fixture. Python hashes actual contents and rejects + // corruption injected after all predecessor checks. + copy := "#!/bin/bash\nargs=(\"$@\"); src=${args[${#args[@]}-2]}; dst=${args[${#args[@]}-1]}; mkdir -p \"$dst\"; if [[ ! -e $dst/shared.fits ]]; then cp \"$src/shared.fits\" \"$dst/shared.fits\"; fi\n" + if corrupt { + copy += "printf 'corrupt' >\"$dst/shared.fits\"\n" + } + if err := os.WriteFile(filepath.Join(bin, "rsync"), []byte(copy), 0700); err != nil { + t.Fatal(err) + } + cmd := exec.Command("bash", "-c", remoteWorkspaceScript(sources, dest)) + cmd.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH")) + output, runErr := cmd.CombinedOutput() + if corrupt { + if runErr == nil || !strings.Contains(string(output), "workspace verification failed") { + t.Fatalf("corruption accepted: %v %s", runErr, output) + } + } else if runErr != nil { + t.Fatalf("merge failed: %v %s", runErr, output) + } + if !corrupt && !strings.Contains(string(output), "__AKOFLOW_MERGE_VERIFIED_FILES__=1") { + t.Fatalf("destination not verified once per unique path: %s", output) + } + }) + } +} + +func TestSharedWorkspaceProofRequiresMatchingFilesystemAndMount(t *testing.T) { + for _, test := range []struct { + name, output string + want bool + }{ + {"shared Lustre", "lustre a12 lustre server:/lustre /home\nlustre a12 lustre server:/lustre /home\n", true}, + {"shared NFS zero ID", "nfs 0 nfs4 server:/home /home\nnfs 0 nfs4 server:/home /home\n", true}, + {"shared BeeGFS", "fhgfs 0 beegfs beegfs_nodev /beegfs\nfhgfs 0 beegfs beegfs_nodev /beegfs\n", true}, + {"different mount source", "nfs 0 nfs server:/a /home\nnfs 0 nfs server:/b /home\n", false}, + {"different mount path", "nfs 0 nfs server:/a /home\nnfs 0 nfs server:/a /scratch\n", false}, + {"different filesystem identity", "lustre a12 lustre server:/a /home\nlustre b34 lustre server:/a /home\n", false}, + {"local filesystem", "ext4 a12 ext4 /dev/sda /home\next4 a12 ext4 /dev/sda /home\n", false}, + {"filesystem type mismatch", "nfs 0 ext4 /dev/sda /home\nnfs 0 ext4 /dev/sda /home\n", false}, + {"not Lustre", "ext4 a12\next4 a12\n", false}, + {"missing destination", "lustre a12\n", false}, + {"missing identity", "lustre\nlustre\n", false}, + } { + t.Run(test.name, func(t *testing.T) { + _, got := parseSharedWorkspaceProof(test.output) + if got != test.want { + t.Fatalf("proof accepted=%v want=%v", got, test.want) + } + }) + } +} + +func TestSameCloudWorkspaceRequiresExactGenerationAndTrust(t *testing.T) { + u := "ssh://worker.test/workspace?strictHostKeyChecking=true&hostKeyAlias=generation-a&knownHostsFile=/trusted/hosts" + a := domain.TransferEndpoint{URI: u, ConnectionID: "cloud-instance-a", CloudInstanceID: "instance-a"} + b := a + if !sameVerifiedCloudWorkspaceEndpoint(a, b) { + t.Fatal("verified same generation rejected") + } + b.CloudInstanceID = "instance-b" + if sameVerifiedCloudWorkspaceEndpoint(a, b) { + t.Fatal("different instance accepted") + } + b = a + b.CloudInstanceID = "" + if sameVerifiedCloudWorkspaceEndpoint(a, b) { + t.Fatal("missing instance accepted") + } + b = a + b.URI = "ssh://worker.test/workspace" + if sameVerifiedCloudWorkspaceEndpoint(a, b) { + t.Fatal("untrusted endpoint accepted") + } + b = a + b.URI = strings.Replace(u, "generation-a", "generation-b", 1) + if sameVerifiedCloudWorkspaceEndpoint(a, b) { + t.Fatal("different trust generation accepted") + } +} + +func TestSameCloudWorkspaceLocalFilesystemProof(t *testing.T) { + bin := t.TempDir() + if err := os.WriteFile(filepath.Join(bin, "ssh"), []byte("#!/bin/sh\nprintf 'ext2/ext3 a12 ext4 /dev/sda /akoflow\\next2/ext3 a12 ext4 /dev/sda /akoflow\\n'\n"), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("AKOFLOW_SSH_CONTROL_DIRECTORY", t.TempDir()) + u := "ssh://worker.test/workspace?strictHostKeyChecking=true&hostKeyAlias=generation-a&knownHostsFile=/trusted/hosts" + a := domain.TransferEndpoint{URI: u, ConnectionID: "cloud-a", CloudInstanceID: "instance-a"} + if proof, ok := sameCloudWorkspaceProof(context.Background(), a, a); !ok || !strings.HasPrefix(proof, "ext2/ext3 ") { + t.Fatalf("local filesystem proof rejected: %q %v", proof, ok) + } +} + +// Exercise real rsync and the remote shell script, with transport redirected +// into temporary directories. This is not a live Lustre integration test. +func TestMixedWorkspaceRoutesConflictAndRecovery(t *testing.T) { + for _, sameCloud := range []bool{false, true} { + t.Run(fmt.Sprintf("sameCloud=%v", sameCloud), func(t *testing.T) { testMixedWorkspaceRoutesConflictAndRecovery(t, sameCloud) }) + } +} + +func testMixedWorkspaceRoutesConflictAndRecovery(t *testing.T, sameCloud bool) { + fixture := newMixedRouteFixture(t, sameCloud) + root, hpc, fog, target := fixture.root, fixture.hpc, fixture.fog, fixture.target + write, plans := fixture.write, fixture.plans + syncer := WorkspaceRsync{Resolver: directWorkspaceEndpointResolver{}} + runs, err := syncer.Sync(context.Background(), plans) + if err != nil { + t.Fatal(err) + } + assertMixedRouteEvidence(t, sameCloud, runs) + for _, name := range []string{"hpc.fits", "fog.fits", "fog2.fits"} { + if _, err := os.Stat(filepath.Join(target, name)); err != nil { + t.Fatal(err) + } + } + // Consumer mutation must not modify producer data (no hardlinks). + write(filepath.Join(target, "hpc.fits"), "consumer mutation") + content, _ := os.ReadFile(filepath.Join(hpc, "hpc.fits")) + if string(content) != "HPC" { + t.Fatal("producer mutated") + } + if _, err := syncer.Sync(context.Background(), plans); err == nil { + t.Fatal("conflict accepted") + } + write(filepath.Join(target, "hpc.fits"), "HPC") + runs, err = syncer.Sync(context.Background(), plans) + if err != nil { + t.Fatal(err) + } + if runs[0].FilesTransferred != 0 || runs[1].FilesTransferred != 0 { + t.Fatalf("valid destination recopied: %+v", runs) + } + // An interrupted network route must not release the merge barrier. A retry + // preserves producer data and reuses files already validated at destination. + write(filepath.Join(fog, "after-interruption"), "new") + t.Setenv("AKOFLOW_TEST_FAIL_ONCE", filepath.Join(root, "failure-injected")) + failed, err := syncer.Sync(context.Background(), plans) + if err == nil || failed[1].Status != domain.TransferFailed || failed[1].Error == "" { + t.Fatalf("interruption not recorded: runs=%+v err=%v", failed, err) + } + if _, err := os.Stat(filepath.Join(target, "after-interruption")); !os.IsNotExist(err) { + t.Fatal("published after failed network preparation") + } + runs, err = syncer.Sync(context.Background(), plans) + if err != nil || runs[0].FilesTransferred != 0 || runs[1].FilesTransferred != 1 { + t.Fatalf("recovery did not reuse valid publication: runs=%+v err=%v", runs, err) + } + if content, err := os.ReadFile(filepath.Join(fog, "after-interruption")); err != nil || string(content) != "new" { + t.Fatal("producer data lost during recovery") + } + // A conflict between mixed producers blocks publication of a new file. + write(filepath.Join(hpc, "collision"), "one") + write(filepath.Join(fog, "collision"), "two") + write(filepath.Join(fog, "not-published"), "new") + if _, err := syncer.Sync(context.Background(), plans); err == nil { + t.Fatal("mixed conflict accepted") + } + if _, err := os.Stat(filepath.Join(target, "not-published")); !os.IsNotExist(err) { + t.Fatal("published before aggregate validation") + } +} + +func TestWorkspaceRouteAddressRedactsCredentials(t *testing.T) { + got := workspaceRouteAddress("ssh://user:secret@hpc.test/workspace?identityFile=/secret/key#secret") + if got != "ssh://hpc.test/workspace" { + t.Fatalf("unsafe route address: %q", got) + } +} + +type mixedRouteFixture struct { + root, hpc, fog, target string + write func(string, string) + plans []domain.DataTransferPlan +} + +func newMixedRouteFixture(t *testing.T, sameCloud bool) mixedRouteFixture { + t.Helper() + realRsync, err := exec.LookPath("rsync") + if err != nil { + t.Skip("rsync unavailable") + } + bin := t.TempDir() + // Strip the GNU-only archive option in this local BSD-tar transport fixture. + ssh := ("#!/bin/sh\nwhile [ \"$#\" -gt 1 ]; do shift; done\ncase \"$1\" in *'stat -f'*) printf 'nfs 0 nfs4 server:/" + + "fixture /home\\nnfs 0 nfs4 server:/fixture /home\\n'; exit 0;; esac\ncommand=$(printf '%s' \"$1\" | sed '" + + "s/--hard-dereference //g'); exec sh -c \"$command\"\n") + if sameCloud { + ssh = strings.ReplaceAll(ssh, "nfs 0 nfs4 server:/fixture /home", "ext2/ext3 a12 ext4 /dev/sda /akoflow") + } + rsync := fmt.Sprintf( + ("#!/bin/bash\nif [[ -n ${AKOFLOW_TEST_FAIL_ONCE:-} && ! -e $AKOFLOW_TEST_FAIL_ONCE ]]; then touch \"$AK" + + "OFLOW_TEST_FAIL_ONCE\"; echo 'injected network interruption' >&2; exit 12; fi\nargs=(); skip=0\nfor arg" + + " in \"$@\"; do if ((skip)); then skip=0; continue; fi; case \"$arg\" in -e) skip=1;; --secluded-args) ;;" + + " hpc.test:*) args+=(\"${arg#hpc.test:}\");; *) args+=(\"$arg\");; esac; done\nexec %s \"${args[@]}\"\n"), + shell(realRsync), + ) + for name, body := range map[string]string{"ssh": ssh, "rsync": rsync} { + if err := os.WriteFile(filepath.Join(bin, name), []byte(body), 0700); err != nil { + t.Fatal(err) + } + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("AKOFLOW_SSH_CONTROL_DIRECTORY", t.TempDir()) + root := t.TempDir() + hpc, fog, fog2, target := filepath.Join(root, "hpc"), filepath.Join(root, "fog"), filepath.Join(root, "fog2"), filepath.Join(root, "target") + for _, path := range []string{hpc, fog, fog2, target} { + if err := os.Mkdir(path, 0700); err != nil { + t.Fatal(err) + } + } + write := func(path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + write(filepath.Join(hpc, "hpc.fits"), "HPC") + write(filepath.Join(fog, "fog.fits"), "FOG") + write(filepath.Join(fog2, "fog2.fits"), "FOG2") + remote := func(path string) string { + u := &url.URL{Scheme: "ssh", Host: "hpc.test", Path: path} + if sameCloud { + u.RawQuery = "strictHostKeyChecking=true&hostKeyAlias=verified-generation&knownHostsFile=/trusted/hosts" + } + return u.String() + } + plans := []domain.DataTransferPlan{ + {ID: "hpc", Source: domain.TransferLocation{URI: remote(hpc)}, Destination: domain.TransferLocation{URI: remote(target)}}, + {ID: "fog", Source: domain.TransferLocation{URI: workspaceURL(fog)}, Destination: domain.TransferLocation{URI: remote(target)}}, + {ID: "fog2", Source: domain.TransferLocation{URI: (&url.URL{Scheme: "ssh", Host: "cloud.test", Path: fog2}).String()}, Destination: domain.TransferLocation{URI: remote(target)}}, + } + if sameCloud { + for i := range plans { + plans[i].Destination.CloudInstanceID = "verified-instance" + } + plans[0].Source.CloudInstanceID = "verified-instance" + } + return mixedRouteFixture{root: root, hpc: hpc, fog: fog, target: target, write: write, plans: plans} +} + +func assertMixedRouteEvidence(t *testing.T, sameCloud bool, runs []domain.DataTransferRun) { + t.Helper() + expectedReason, expectedFS, expectedMount := "verified shared", "nfs", "server:/fixture" + if sameCloud { + expectedReason, expectedFS, expectedMount = "verified same concrete cloud", "ext2/ext3", "/dev/sda" + if runs[0].Strategy != domain.TransferRuntimeLocal { + t.Fatalf("same cloud did not use runtime-local: %+v", runs[0]) + } + } + if runs[0].NetworkBytes != 0 || !strings.Contains(runs[0].Route.Reason, expectedReason) || runs[1].NetworkBytes <= 0 { + t.Fatalf("mixed routes: %+v", runs) + } + for _, run := range runs[1:] { + if run.Route.ArchiveBytes <= 0 || len(run.Route.Phases) != 6 { + t.Fatalf("missing phase/archive evidence: %+v", run.Route) + } + if run.Route.Packaging != "tar-gzip" || run.Route.PreparationSeconds <= 0 || run.NetworkBytes <= 0 { + t.Fatalf("missing archive preparation evidence: %+v", run) + } + } + if len(runs[0].Route.Phases) != 2 || runs[0].Route.ArchiveBytes != 0 { + t.Fatalf("shared origin incorrectly attributed archive work: %+v", runs[0].Route) + } + if runs[0].Route.SharedFilesystem != expectedFS || !strings.Contains(runs[0].Route.SharedFilesystemID, expectedMount) || runs[1].Route.SharedFilesystem != "" || runs[0].Route.FilesTransferred != runs[0].FilesTransferred { + t.Fatalf("missing or incorrect route evidence: %+v", runs) + } +} diff --git a/internal/infrastructure/transfer/workspace_rsync.go b/internal/infrastructure/transfer/workspace_rsync.go index 84e14a95..3d2d0e07 100644 --- a/internal/infrastructure/transfer/workspace_rsync.go +++ b/internal/infrastructure/transfer/workspace_rsync.go @@ -22,7 +22,9 @@ import ( // In particular, no predecessor is allowed to overwrite a different file from // another predecessor or an already populated successor workspace. type WorkspaceRsync struct { - Resolver interface { + // Parallelism bounds independent preparations; merge remains a single barrier. + Parallelism int + Resolver interface { ResolveTransferEndpoint(context.Context, domain.TransferLocation) (domain.TransferEndpoint, error) } } @@ -62,7 +64,11 @@ func (s WorkspaceRsync) Sync(ctx context.Context, plans []domain.DataTransferPla return fail(err) } if direct { - return syncRemoteWorkspaceDirect(ctx, plans, sources, destination, runs) + result, syncErr := s.syncDependencyRoutes(ctx, plans, sources, destination, runs) + if syncErr != nil { + return fail(syncErr) + } + return result, nil } root, err := os.MkdirTemp("", "akoflow-workspace-sync-") if err != nil { @@ -100,10 +106,9 @@ func (s WorkspaceRsync) directRemoteEndpoints(ctx context.Context, plans []domai return nil, destination, false, fmt.Errorf("resolve producer %q: %w", plan.ProducerActivityID, resolveErr) } sources[index] = source - if !sameSSHWorkspaceEndpoint(source, destination) { - return sources, destination, false, nil - } } + // Remote merge can combine independently prepared network snapshots with + // proven shared-filesystem sources; a mixed origin never downgrades all routes. return sources, destination, true, nil } @@ -133,7 +138,33 @@ func syncRemoteWorkspaceDirect(ctx context.Context, plans []domain.DataTransferP sourcePaths[index] = sourcePath } script := remoteWorkspaceScript(sourcePaths, destinationPath) - output, err := runSSHCombinedOutput(ctx, destination, append(sshArgs(destination), host, "bash -s"), strings.NewReader(script)) + mergeStart := time.Now() + command, release, err := queuedSSHCommand(ctx, destination, append(sshArgs(destination), host, "bash -s")) + admittedAt := time.Now() + var output []byte + if err == nil { + command.Stdin = strings.NewReader(script) + output, err = command.CombinedOutput() + release() + } + mergeEnd := time.Now() + for i := range runs { + runs[i].Route.MergeSSHChannelWaitSeconds = admittedAt.Sub(mergeStart).Seconds() + if command != nil { + runs[i].Route.MergeExecutionSeconds = mergeEnd.Sub(admittedAt).Seconds() + } + runs[i].Route.MergeCheckSeconds = remoteMergeSeconds(string(output), "CHECK") + runs[i].Route.MergeCopySeconds = remoteMergeSeconds(string(output), "COPY") + runs[i].Route.MergeVerifySeconds = remoteMergeSeconds(string(output), "VERIFY") + // Residual command wall time includes transport/remote startup and + // untimed script work; it is not network throughput or service queue. + runs[i].Route.MergeCommandOverheadSeconds = max(0, runs[i].Route.MergeExecutionSeconds- + runs[i].Route.MergeCheckSeconds-runs[i].Route.MergeCopySeconds-runs[i].Route.MergeVerifySeconds) + runs[i].Route.Phases = append(runs[i].Route.Phases, domain.TransferPhase{ + Name: "shared-conflict-check-copy-and-verify", StartedAt: float64(mergeStart.UnixNano()) / 1e9, + FinishedAt: float64(mergeEnd.UnixNano()) / 1e9, DurationSeconds: mergeEnd.Sub(mergeStart).Seconds(), Failed: err != nil, + }) + } if err != nil { return runs, fmt.Errorf("direct remote workspace sync: %w: %s", err, strings.TrimSpace(string(output))) } @@ -154,41 +185,6 @@ func syncRemoteWorkspaceDirect(ctx context.Context, plans []domain.DataTransferP return runs, nil } -func remoteWorkspaceScript(sources []string, destination string) string { - var script strings.Builder - script.WriteString("set -euo pipefail\n") - script.WriteString("workspace_manifest=$(mktemp -d); trap 'rm -rf -- \"$workspace_manifest\"' EXIT\n") - script.WriteString( - "check_tree() { local root=$1 label=$2 file rel digest key previous origin; " + - "while IFS= read -r -d '' file; do rel=${file#\"$root\"/}; " + - "digest=$(sha256sum -- \"$file\"); digest=${digest%% *}; " + - "key=$(printf '%s' \"$rel\" | sha256sum); key=${key%% *}; " + - "if [[ -f $workspace_manifest/$key.digest ]]; then " + - "previous=$(cat \"$workspace_manifest/$key.digest\"); " + - "if [[ $previous != \"$digest\" ]]; then " + - "origin=$(cat \"$workspace_manifest/$key.origin\"); " + - "echo \"workspace content conflict at $rel between $origin and $label\" >&2; exit 44; fi; " + - "else printf '%s' \"$digest\" >\"$workspace_manifest/$key.digest\"; " + - "printf '%s' \"$label\" >\"$workspace_manifest/$key.origin\"; fi; " + - "done < <(find \"$root\" -type f -print0); }\n", - ) - script.WriteString("destination=" + shell(destination) + "\n") - script.WriteString("mkdir -p -- \"$destination\"\n") - script.WriteString("link=$(find \"$destination\" -type l -print -quit); test -z \"$link\" || { echo \"destination workspace contains symlink: $link\" >&2; exit 41; }\n") - script.WriteString("check_tree \"$destination\" destination\n") - for _, source := range sources { - script.WriteString("test -d " + shell(source) + " || { echo " + shell("source workspace disappeared: "+source) + " >&2; exit 42; }\n") - script.WriteString("link=$(find " + shell(source) + " -type l -print -quit); test -z \"$link\" || { echo \"source workspace contains symlink: $link\" >&2; exit 43; }\n") - script.WriteString("check_tree " + shell(source) + " " + shell(source) + "\n") - } - for index, source := range sources { - script.WriteString(fmt.Sprintf("echo __AKOFLOW_WORKSPACE_BEGIN_%d__\n", index)) - script.WriteString("rsync --ignore-existing -r --checksum --stats --out-format='%i %n' --no-perms --no-owner --no-group --no-times -- " + shell(filepath.Clean(source)+"/") + " " + shell(filepath.Clean(destination)+"/") + "\n") - script.WriteString(fmt.Sprintf("echo __AKOFLOW_WORKSPACE_END_%d__\n", index)) - } - return script.String() -} - func remoteWorkspaceStats(output string, index int) string { begin := fmt.Sprintf("__AKOFLOW_WORKSPACE_BEGIN_%d__", index) end := fmt.Sprintf("__AKOFLOW_WORKSPACE_END_%d__", index) @@ -219,12 +215,10 @@ func (s WorkspaceRsync) stageProducers(ctx context.Context, plans []domain.DataT } stages[i] = stage stats, err := syncDirectory(ctx, source, domain.TransferEndpoint{URI: (&url.URL{Scheme: "file", Path: stage}).String()}, false) + recordWorkspaceRsyncStats(&runs[i], stats, strings.HasPrefix(source.URI, "ssh://")) if err != nil { return nil, nil, fmt.Errorf("stage producer %q: %w", plan.ProducerActivityID, err) } - if strings.HasPrefix(source.URI, "ssh://") { - runs[i].NetworkBytes += stats.bytes - } ingress[i] = stats if err := inspectWorkspace(stage, seen); err != nil { return nil, nil, fmt.Errorf("producer %q: %w", plan.ProducerActivityID, err) @@ -240,6 +234,7 @@ func publishWorkspace(ctx context.Context, plans []domain.DataTransferPlan, stag return fmt.Errorf("producer %q: %w", plans[i].ProducerActivityID, err) } stats, copyErr := syncDirectory(ctx, source, destination, true) + recordWorkspaceRsyncStats(&runs[i], stats, strings.HasPrefix(destination.URI, "ssh://")) if copyErr != nil { return fmt.Errorf("publish producer %q: %w", plans[i].ProducerActivityID, copyErr) } @@ -253,9 +248,6 @@ func publishWorkspace(ctx context.Context, plans []domain.DataTransferPlan, stag runs[i].FilesTransferred = ingress[i].files } runs[i].Route.FilesTransferred = runs[i].FilesTransferred - if strings.HasPrefix(destination.URI, "ssh://") { - runs[i].NetworkBytes += stats.bytes - } runs[i].FinishedAt = float64(time.Now().UnixNano()) / 1e9 runs[i].DurationSeconds = max(0, runs[i].FinishedAt-runs[i].StartedAt) runs[i].Status = domain.TransferCompleted @@ -298,7 +290,14 @@ func inspectWorkspace(root string, seen map[string][32]byte) error { }) } -type rsyncStats struct{ bytes, fileBytes, files int64 } +type rsyncStats struct { + bytes, fileBytes, files int64 + retries int + lastRetryError string + networkBytesIncomplete bool + channelWaitSeconds float64 + executionSeconds float64 +} func syncDirectory(ctx context.Context, source, destination domain.TransferEndpoint, ignoreExisting bool) (rsyncStats, error) { args, err := rsyncArgs(source, destination) @@ -308,16 +307,23 @@ func syncDirectory(ctx context.Context, source, destination domain.TransferEndpo if ignoreExisting { args = append([]string{"--ignore-existing"}, args...) } + waitStarted := time.Now() release, err := acquireRsyncSSHChannel(ctx, source, destination) + waitSeconds := time.Since(waitStarted).Seconds() if err != nil { - return rsyncStats{}, err + return rsyncStats{channelWaitSeconds: waitSeconds}, err } defer release() - output, err := exec.CommandContext(ctx, "rsync", args...).CombinedOutput() - if err != nil { - return rsyncStats{}, fmt.Errorf("rsync: %w: %s", err, strings.TrimSpace(string(output))) - } - return parseRsyncStats(string(output), strings.HasPrefix(destination.URI, "ssh://")), nil + var executionSeconds float64 + stats, err := retryWorkspaceRsync(ctx, time.Second, strings.HasPrefix(destination.URI, "ssh://"), func() ([]byte, error) { + started := time.Now() + output, copyErr := exec.CommandContext(ctx, "rsync", args...).CombinedOutput() + executionSeconds += time.Since(started).Seconds() + return output, copyErr + }) + stats.channelWaitSeconds = waitSeconds + stats.executionSeconds = executionSeconds + return stats, err } func rsyncArgs(source, destination domain.TransferEndpoint) ([]string, error) { @@ -342,7 +348,7 @@ func rsyncArgs(source, destination domain.TransferEndpoint) ([]string, error) { // shell. Newer rsync versions otherwise treat shell quotes as literal // characters in the path. args = append(args, "--secluded-args") - ssh := []string{"ssh", "-o", "BatchMode=yes"} + ssh := []string{"ssh"} ssh = append(ssh, sshArgs(remote)...) for i := range ssh { ssh[i] = rsyncRemoteShellArgument(ssh[i]) @@ -409,6 +415,10 @@ func makeWorkspaceDirectory(ctx context.Context, endpoint domain.TransferEndpoin } func rejectWorkspaceSymlinks(ctx context.Context, endpoint domain.TransferEndpoint) error { + return rejectWorkspaceSymlinksWithWait(ctx, endpoint, nil) +} + +func rejectWorkspaceSymlinksWithWait(ctx context.Context, endpoint domain.TransferEndpoint, channelWait *float64) error { u, err := url.Parse(endpoint.URI) if err != nil { return err @@ -431,7 +441,11 @@ func rejectWorkspaceSymlinks(ctx context.Context, endpoint domain.TransferEndpoi if err != nil { return err } + waitStarted := time.Now() command, release, err := queuedSSHCommand(ctx, endpoint, append(sshArgs(endpoint), host, "find "+shell(path)+" -type l -print -quit")) + if channelWait != nil { + *channelWait = time.Since(waitStarted).Seconds() + } if err != nil { return err } diff --git a/internal/infrastructure/transfer/workspace_rsync_retry.go b/internal/infrastructure/transfer/workspace_rsync_retry.go new file mode 100644 index 00000000..3abae575 --- /dev/null +++ b/internal/infrastructure/transfer/workspace_rsync_retry.go @@ -0,0 +1,89 @@ +package transfer + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +// Retries operate on the same staging directory. Rsync's checksum comparison +// reuses completed files; no activity command or publication gate is retried. +func retryWorkspaceRsync(ctx context.Context, delay time.Duration, sending bool, copy func() ([]byte, error)) (rsyncStats, error) { + var total rsyncStats + for attempt := 0; attempt < 3; attempt++ { + if err := ctx.Err(); err != nil { + return total, err + } + output, err := copy() + stats := parseRsyncStats(string(output), sending) + total.bytes += stats.bytes + if err == nil { + total.fileBytes, total.files = stats.fileBytes, stats.files + return total, nil + } + // A failed rsync may omit its final statistics. Keep reported received + // bytes as a lower bound and explicitly flag incomplete accounting. + if stats.bytes == 0 { + total.bytes += interruptedRsyncBytes(string(output)) + total.networkBytesIncomplete = true + } + failure := fmt.Errorf("rsync: %w: %s", err, strings.TrimSpace(string(output))) + if ctx.Err() != nil || !transientWorkspaceRsync(string(output)) || attempt == 2 { + return total, failure + } + total.retries++ + lines := strings.Split(strings.TrimSpace(string(output)), "\n") + total.lastRetryError = lines[len(lines)-1] + timer := time.NewTimer(delay * time.Duration(attempt+1)) + select { + case <-ctx.Done(): + timer.Stop() + return total, ctx.Err() + case <-timer.C: + } + } + return total, fmt.Errorf("workspace rsync retries exhausted") +} + +func transientWorkspaceRsync(output string) bool { + message := strings.ToLower(output) + for _, permanent := range []string{"permission denied", "host key verification failed", "no such file", "content conflict", "no space left"} { + if strings.Contains(message, permanent) { + return false + } + } + return strings.Contains(message, "connection unexpectedly closed") || strings.Contains(message, "connection reset") || strings.Contains(message, "connection closed") || strings.Contains(message, "broken pipe") +} + +func interruptedRsyncBytes(output string) int64 { + var received int64 + for _, line := range strings.Split(output, "\n") { + _, suffix, ok := strings.Cut(line, "connection unexpectedly closed (") + if !ok { + continue + } + value, _, ok := strings.Cut(suffix, " bytes received so far)") + if ok { + n, _ := strconv.ParseInt(value, 10, 64) + received = max(received, n) + } + } + return received +} + +func recordWorkspaceRsyncStats(run *domain.DataTransferRun, stats rsyncStats, network bool) { + run.Route.SSHChannelWaitSeconds += stats.channelWaitSeconds + run.Route.RsyncExecutionSeconds += stats.executionSeconds + if network { + run.NetworkBytes += stats.bytes + run.Route.NetworkBytesIncomplete = run.Route.NetworkBytesIncomplete || stats.networkBytesIncomplete + } + run.Route.RetryCount += stats.retries + if stats.lastRetryError != "" { + run.Route.LastRetryError = stats.lastRetryError + } +} diff --git a/internal/infrastructure/transfer/workspace_rsync_retry_test.go b/internal/infrastructure/transfer/workspace_rsync_retry_test.go new file mode 100644 index 00000000..461ca2e1 --- /dev/null +++ b/internal/infrastructure/transfer/workspace_rsync_retry_test.go @@ -0,0 +1,121 @@ +package transfer + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestRsyncStageTimingAccumulatesSeparatelyFromBytes(t *testing.T) { + run := domain.DataTransferRun{} + recordWorkspaceRsyncStats(&run, rsyncStats{channelWaitSeconds: 12, executionSeconds: 3, bytes: 100}, true) + recordWorkspaceRsyncStats(&run, rsyncStats{channelWaitSeconds: 2, executionSeconds: 5, bytes: 50}, true) + if run.Route.SSHChannelWaitSeconds != 14 || run.Route.RsyncExecutionSeconds != 8 || run.NetworkBytes != 150 { + t.Fatalf("distinct stage accounting: %+v", run) + } +} + +func TestSyncDirectoryMeasuresCommandExecutionSeparately(t *testing.T) { + bin := t.TempDir() + script := "#!/bin/sh\nsleep 0.05\nprintf 'Total bytes sent: 100\\n'\n" + if err := os.WriteFile(filepath.Join(bin, "rsync"), []byte(script), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + stats, err := syncDirectory(context.Background(), domain.TransferEndpoint{URI: "file:///source"}, domain.TransferEndpoint{URI: "file:///target"}, false) + if err != nil || stats.executionSeconds < 0.04 || stats.channelWaitSeconds >= stats.executionSeconds { + t.Fatalf("command timing must exclude channel acquisition: stats=%+v err=%v", stats, err) + } +} + +func TestWorkspaceRsyncRetriesTransientFailureAndAccountsReportedBytes(t *testing.T) { + calls := 0 + stats, err := retryWorkspaceRsync(context.Background(), 0, false, func() ([]byte, error) { + calls++ + if calls == 1 { + return []byte("rsync: connection unexpectedly closed (100 bytes received so far) [receiver]\nBroken pipe"), errors.New("exit 255") + } + return []byte("Total bytes received: 50\nNumber of regular files transferred: 1\nTotal transferred file size: 40"), nil + }) + if err != nil || calls != 2 || stats.bytes != 150 || stats.retries != 1 || !stats.networkBytesIncomplete || stats.lastRetryError == "" { + t.Fatalf("retry statistics: calls=%d stats=%+v err=%v", calls, stats, err) + } +} + +func TestWorkspaceRsyncRetryReusesRealChecksumValidFiles(t *testing.T) { + realRsync, err := exec.LookPath("rsync") + if err != nil { + t.Skip("rsync unavailable") + } + root := t.TempDir() + source, destination, bin := filepath.Join(root, "source"), filepath.Join(root, "destination"), filepath.Join(root, "bin") + for _, directory := range []string{source, destination, bin} { + if err := os.Mkdir(directory, 0700); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(source, "complete.fits"), []byte("checksum-valid producer data"), 0600); err != nil { + t.Fatal(err) + } + marker := filepath.Join(root, "interrupted") + script := fmt.Sprintf( + "#!/bin/sh\nif test ! -e %s; then %s \"$@\" || exit $?; touch %s; echo 'connection unexpectedly closed (100 bytes received so far)' >&2; exit 255; fi\nexec %s \"$@\"\n", + shell(marker), + shell(realRsync), + shell(marker), + shell(realRsync), + ) + if err := os.WriteFile(filepath.Join(bin, "rsync"), []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + stats, err := syncDirectory(context.Background(), domain.TransferEndpoint{URI: workspaceURL(source)}, domain.TransferEndpoint{URI: workspaceURL(destination)}, false) + if err != nil || stats.retries != 1 || stats.files != 0 { + t.Fatalf("validated files recopied on retry: stats=%+v err=%v", stats, err) + } + for _, directory := range []string{source, destination} { + content, err := os.ReadFile(filepath.Join(directory, "complete.fits")) + if err != nil || string(content) != "checksum-valid producer data" { + t.Fatalf("producer or destination corrupted: %q err=%v", content, err) + } + } +} + +func TestWorkspaceRsyncDoesNotRetryPermanentErrorsOrCancellation(t *testing.T) { + for _, message := range []string{"Permission denied\nBroken pipe", "workspace content conflict", "No space left on device", "No such file or directory"} { + calls := 0 + _, err := retryWorkspaceRsync(context.Background(), 0, false, func() ([]byte, error) { + calls++ + return []byte(message), errors.New("copy failed") + }) + if err == nil || calls != 1 { + t.Fatalf("permanent error retried: %q calls=%d", message, calls) + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := retryWorkspaceRsync(ctx, 0, false, func() ([]byte, error) { + t.Fatal("copy executed after cancellation") + return nil, nil + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation lost: %v", err) + } +} + +func TestWorkspaceRsyncTransientRetriesAreBounded(t *testing.T) { + calls := 0 + _, err := retryWorkspaceRsync(context.Background(), 0, false, func() ([]byte, error) { + calls++ + return []byte("connection reset by peer"), errors.New("exit 255") + }) + if err == nil || calls != 3 { + t.Fatalf("unbounded or missing retries: calls=%d err=%v", calls, err) + } +} diff --git a/internal/infrastructure/transfer/workspace_rsync_test.go b/internal/infrastructure/transfer/workspace_rsync_test.go index 1596bbb7..181f3446 100644 --- a/internal/infrastructure/transfer/workspace_rsync_test.go +++ b/internal/infrastructure/transfer/workspace_rsync_test.go @@ -22,7 +22,7 @@ func (workspaceEndpointResolver) ResolveTransferEndpoint(_ context.Context, loca type directWorkspaceEndpointResolver struct{} func (directWorkspaceEndpointResolver) ResolveTransferEndpoint(_ context.Context, location domain.TransferLocation) (domain.TransferEndpoint, error) { - return domain.TransferEndpoint{URI: location.URI, ConnectionID: "shared-hpc"}, nil + return domain.TransferEndpoint{URI: location.URI, ConnectionID: "shared-hpc", CloudInstanceID: location.CloudInstanceID}, nil } func workspaceURL(path string) string { return (&url.URL{Scheme: "file", Path: path}).String() } @@ -69,7 +69,7 @@ func TestWorkspaceRsyncSSHRemoteShellKeepsProxyCommandInOneArgument(t *testing.T t.Fatalf("rsync arguments lack remote shell: %q", args) } for _, expected := range []string{ - "ssh -o BatchMode=yes -i storage/credentials/ssh/ovvesley-personal", + "ssh -o BatchMode=yes -o ConnectionAttempts=1 -o ConnectTimeout=10 -i storage/credentials/ssh/ovvesley-personal", `-o "ProxyCommand=ssh `, ` -W plafrim:22" -A`, } { @@ -210,7 +210,11 @@ func TestWorkspaceRsyncCopiesDirectlyInsideSharedHPC(t *testing.T) { } bin := t.TempDir() fakeSSH := filepath.Join(bin, "ssh") - if err := os.WriteFile(fakeSSH, []byte("#!/bin/sh\nwhile [ \"$#\" -gt 1 ]; do shift; done\nexec sh -c \"$1\"\n"), 0o700); err != nil { + if err := os.WriteFile( + fakeSSH, + []byte("#!/bin/sh\nwhile [ \"$#\" -gt 1 ]; do shift; done\ncase \"$1\" in *'stat -f'*) printf 'lustre test-fsid lustre server:/fixture /home\\nlustre test-fsid lustre server:/fixture /home\\n'; exit 0;; esac\nexec sh -c \"$1\"\n"), + 0o700, + ); err != nil { t.Fatal(err) } t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) @@ -246,6 +250,17 @@ func TestWorkspaceRsyncCopiesDirectlyInsideSharedHPC(t *testing.T) { if run.Status != domain.TransferCompleted || run.Strategy != domain.TransferDirectRuntime || run.NetworkBytes != 0 || run.FilesTransferred != 1 { t.Fatalf("unexpected direct transfer: %+v", run) } + if run.Route.MergeExecutionSeconds <= 0 || run.Route.MergeSSHChannelWaitSeconds < 0 || run.Route.MergeCommandOverheadSeconds < 0 { + t.Fatalf("missing separated remote merge timings: %+v", run.Route) + } + for _, phase := range run.Route.Phases { + if phase.Name == "shared-conflict-check-copy-and-verify" { + difference := phase.DurationSeconds - run.Route.MergeSSHChannelWaitSeconds - run.Route.MergeExecutionSeconds + if difference < -1e-6 || difference > 1e-6 { + t.Fatalf("queue and command do not account for merge wall time: %g", difference) + } + } + } } runs, err = (WorkspaceRsync{Resolver: directWorkspaceEndpointResolver{}}).Sync(context.Background(), plans) if err != nil { diff --git a/internal/montagetools/activity_comparison.go b/internal/montagetools/activity_comparison.go new file mode 100644 index 00000000..d870660e --- /dev/null +++ b/internal/montagetools/activity_comparison.go @@ -0,0 +1,117 @@ +package montagetools + +import ( + "math" + "sort" + "time" +) + +func compareActivityTimes(request, real, simulation Object) Object { + plan := map[string]Object{} + for _, a := range rows(obj(request["Plan"])["assignments"]) { + plan[str(a["activityId"])] = a + } + sim := map[string]Object{} + for _, a := range rows(simulation["tasks"]) { + sim[str(a["activityId"])] = a + } + logicalReady := map[string]float64{} + for _, dependency := range rows(obj(request["Workflow"])["dependencies"]) { + producer := sim[str(dependency["dependsOnActivityId"])] + check(producer != nil, "missing simulated producer publication") + consumer := str(dependency["activityId"]) + logicalReady[consumer] = math.Max(logicalReady[consumer], num(producer["finishedAt"])) + } + completed := map[string]float64{} + for _, event := range rows(real["events"]) { + if event["eventType"] != "activity.completed" { + continue + } + at, err := time.Parse(time.RFC3339Nano, str(event["occurredAt"])) + check(err == nil, "invalid completion timestamp") + completed[str(event["aggregateId"])] = float64(at.UnixNano()) / 1e9 + } + handles := map[string]int{} + for _, h := range rows(real["handles"]) { + handles[str(h["activityId"])]++ + } + results, computeErrors, wallErrors := compareMatchedActivities(real, plan, sim, logicalReady, completed, handles) + sort.Slice(results, func(i, j int) bool { return str(obj(results[i])["activityId"]) < str(obj(results[j])["activityId"]) }) + within := func(values []any) int { + n := 0 + for _, v := range values { + if num(v) <= 20 { + n++ + } + } + return n + } + return Object{ + "referenceRunId": obj(real["run"])["id"], + "role": "per-activity-historical-comparison-not-independent-validation", + "activityCount": len(results), + "singleAttemptComparableCount": len(computeErrors), + "samePlacementWallComparableCount": len(wallErrors), + "computeWithin20PercentCount": within(computeErrors), + "readyToPublishedWithin20PercentCount": within(wallErrors), + "absoluteComputeErrorPercent": summary(computeErrors), + "absoluteReadyToPublishedErrorPercent": summary(wallErrors), + "activities": results, + "limitations": list( + "Queue overlaps preparation; these counters must not be added.", + "Transfer work sums parallel dependencies and is not consumer elapsed time.", + "Ready-to-published includes preparation, lane waiting, launch, compute and publication, but not time before readiness.", + "Recovered activities are excluded from single-attempt wall comparisons.", + "Publication and input-observation coefficients were fitted on this reference; their agreement is in-sample, not independent validation.", + ), + } +} + +func compareMatchedActivities(real Object, plan, sim map[string]Object, logicalReady, completed map[string]float64, handles map[string]int) ([]any, []any, []any) { + results := []any{} + var computeErrors, wallErrors []any + for _, activity := range rows(real["activities"]) { + id := str(activity["activityId"]) + a, s := plan[id], sim[id] + check(a != nil && s != nil && activity["status"] == "completed" && s["status"] == "completed", "missing completed matched activity") + samePlacement := activity["allocatedResourceId"] == s["allocatedResourceId"] + metadata := obj(activity["metadata"]) + finish, hasCompletion := completed[str(activity["id"])] + ready := num(activity["readyAt"]) + comparable := handles[id] == 1 && hasCompletion && ready > 0 && finish >= ready + realCompute, simCompute := num(activity["runtimeSeconds"]), num(s["runtimeSeconds"]) + computeError := errorPercent(simCompute, realCompute) + simMetadata := obj(s["metadata"]) + simReady := logicalReady[id] + record := Object{"activityId": id, "resourceClass": resourceClass(str(activity["allocatedResourceId"])), + "samePlacement": samePlacement, "observedResourceId": activity["allocatedResourceId"], "simulatedResourceId": s["allocatedResourceId"], + "plannedComputeSeconds": a["predictedRuntimeSeconds"], "simulatedComputeSeconds": simCompute, "observedComputeSeconds": realCompute, + "computeErrorPercent": computeError, "singleAttemptComparable": comparable, + "observedPreparationElapsedSeconds": metadata["transferElapsedSeconds"], "observedTransferWorkSecondsNotElapsed": activity["transferSeconds"], + "observedLaunchWaitSeconds": metadata["launchWaitSeconds"], "observedQueueSecondsOverlapsPreparation": activity["queueSeconds"], + "simulatedLogicalReadyAt": simReady, "simulatedReadyAtAfterTransfers": s["readyAt"], + "simulatedPreparationElapsedSeconds": num(simMetadata["workspacePreparationFinishedAt"]) - simReady, + "simulatedPostPreparationWaitSeconds": num(s["startedAt"]) - num(simMetadata["workspacePreparationFinishedAt"]), + "simulatedPublicationSeconds": simMetadata["workspacePublicationSeconds"], + "simulatedQueueSecondsNotDirectlyComparable": s["queueSeconds"], + "observedBeforePreparationWaitSeconds": num(metadata["preparationStartedAt"]) - ready, + "observedPostPreparationWaitSeconds": num(activity["startedAt"]) - num(metadata["preparationFinishedAt"]), + "observedRuntimeWallSeconds": num(activity["finishedAt"]) - num(activity["startedAt"]), + "simulatedReadyToPublishedSeconds": num(s["finishedAt"]) - simReady} + if comparable { + realWall := finish - ready + simWall := num(s["finishedAt"]) - simReady + record["observedPublicationAndPollSeconds"] = finish - num(activity["finishedAt"]) + record["observedReadyToPublishedSeconds"] = realWall + record["readyToPublishedErrorPercent"] = errorPercent(simWall, realWall) + computeErrors = append(computeErrors, math.Abs(num(computeError))) + if samePlacement { + wallErrors = append(wallErrors, math.Abs(num(errorPercent(simWall, realWall)))) + } + } else { + record["excludedWallReason"] = "recovered attempt or missing matched publication evidence" + } + results = append(results, record) + } + return results, computeErrors, wallErrors +} diff --git a/internal/montagetools/activity_comparison_test.go b/internal/montagetools/activity_comparison_test.go new file mode 100644 index 00000000..4165f8a9 --- /dev/null +++ b/internal/montagetools/activity_comparison_test.go @@ -0,0 +1,62 @@ +package montagetools + +import "testing" + +func comparisonFixture() (Object, Object, Object) { + id, resource := "montage-mprojectid0000001", "montage-gcp-fog-e2-micro-01" + request := Object{"Plan": Object{"assignments": []any{Object{"activityId": id, "resourceId": resource, "predictedRuntimeSeconds": 10.0}}}} + real := Object{ + "run": Object{"id": "reference"}, + "activities": []any{ + Object{ + "id": "task-1", + "activityId": id, + "status": "completed", + "allocatedResourceId": resource, + "runtimeSeconds": 10.0, + "readyAt": 100.0, + "transferSeconds": 1000.0, + "queueSeconds": 7.0, + "metadata": Object{"transferElapsedSeconds": 3.0}, + }, + }, + "handles": []any{Object{"activityId": id}}, + "events": []any{Object{"aggregateId": "task-1", "eventType": "activity.completed", "occurredAt": "1970-01-01T00:01:56Z"}}, + } + sim := Object{"tasks": []any{Object{"activityId": id, "allocatedResourceId": resource, "status": "completed", "readyAt": 0.0, "finishedAt": 14.0, "runtimeSeconds": 10.0}}} + return request, real, sim +} + +func TestActivityComparisonUsesPublishedWallNotParallelWorkSum(t *testing.T) { + request, real, sim := comparisonFixture() + report := compareActivityTimes(request, real, sim) + row := obj(arr(report["activities"])[0]) + if num(row["observedReadyToPublishedSeconds"]) != 16 || num(row["observedTransferWorkSecondsNotElapsed"]) != 1000 || report["computeWithin20PercentCount"] != 1 { + t.Fatal("wall/compute/work contracts mixed", report) + } +} + +func TestActivityComparisonExcludesRecoveredAndChangedPlacementWall(t *testing.T) { + request, real, sim := comparisonFixture() + obj(arr(sim["tasks"])[0])["allocatedResourceId"] = "montage-gcp-fog-e2-micro-02" + if num(compareActivityTimes(request, real, sim)["samePlacementWallComparableCount"]) != 0 { + t.Fatal("changed placement presented as pure model residual") + } + real["handles"] = append(arr(real["handles"]), Object{"activityId": "montage-mprojectid0000001"}) + if num(compareActivityTimes(request, real, sim)["singleAttemptComparableCount"]) != 0 { + t.Fatal("recovery presented as independent attempt") + } +} + +func TestActivityComparisonIncludesTransfersBeforeSimgridReadyTimestamp(t *testing.T) { + request, real, sim := comparisonFixture() + request["Workflow"] = Object{"dependencies": []any{Object{"activityId": "montage-mprojectid0000001", "dependsOnActivityId": "producer"}}} + sim["tasks"] = append(arr(sim["tasks"]), Object{"activityId": "producer", "finishedAt": 5.0}) + consumer := obj(arr(sim["tasks"])[0]) + consumer["readyAt"], consumer["finishedAt"] = 10.0, 20.0 + consumer["metadata"] = Object{"workspacePreparationFinishedAt": 12.0} + row := obj(arr(compareActivityTimes(request, real, sim)["activities"])[0]) + if num(row["simulatedReadyToPublishedSeconds"]) != 15 || num(row["simulatedPreparationElapsedSeconds"]) != 7 { + t.Fatal("transfer window silently excluded", row) + } +} diff --git a/internal/montagetools/audit.go b/internal/montagetools/audit.go new file mode 100644 index 00000000..b3e602b3 --- /dev/null +++ b/internal/montagetools/audit.go @@ -0,0 +1,411 @@ +package montagetools + +import ( + "fmt" + "sort" + "strings" + "time" +) + +func audit(request, response, simulation Object, fresh bool) Object { + attempts := indexMontageAuditAttempts(request, response) + run, assignments, multiple := attempts.run, attempts.assignments, attempts.multiple + + if fresh { + validateFreshMontageAudit(attempts, response) + } + evidence := validateMontageTransferEvidence(request, response, attempts) + reportAudit, transfers, deps, cross := evidence.report, evidence.transfers, evidence.deps, evidence.cross + metrics := func(m Object) Object { return pick(m, "makespanSeconds", "cost", "transferredBytes", "breakdown") } + network, relay, routeRows := auditTransferRows(transfers) + shared := auditSharedPreparationRows(deps) + activityRows := auditActivityRows(attempts) + history, attemptLimitation := "single_observed_attempt_per_activity", "No multiple attempts observed; this alone does not prove an uninterrupted benchmark." + if multiple { + history, attemptLimitation = "multiple_attempts", "Multiple attempts: performance can be contaminated by recovery." + } + wall := auditRunTimestampWall(run) + billing := obj(response["cloudBilling"]) + generations := auditCloudGenerations(billing) + costLimitation := "Cloud prices are incomplete; cost zero does not mean free cloud execution." + if billing["status"] == "estimated" { + costLimitation = "Cloud lifecycle cost is a list-rate estimate with explicit exclusions, not the provider invoice. The task/window API cost and plan retention-window cost have different bases." + } + return Object{ + "schema_version": 1, + "run_id": run["id"], + "attempt_history": history, + "audit": reportAudit, + "plan": metrics(obj(get(obj(request["plan"]), "predicted"))), + "plan_assignment_compute_work_seconds": sum(rows(fileValues(assignments)), "predictedRuntimeSeconds"), + "plan_assignment_transfer_work_seconds": sum(rows(fileValues(assignments)), "predictedTransferSeconds"), + "simulation": metrics(obj(get(simulation, "run"))), + "real_api_reported": metrics(run), + "real_cloud_lifecycle_estimate": pick(billing, "status", "currency", "estimatedCost", "knownCost", "costBasis", "excludedCosts", "asOf"), + "real_cloud_generations": generations, + "real_api_run_timestamp_wall_seconds": wall, + "real_network_bytes_from_routes": network, + "transfer_route_count": len(transfers), + "dependency_route_count": len(deps), + "cross_cloud_hpc_relay_count": cross, + "relay_server_network_bytes": relay, + "routes": routeRows, + "shared_preparations": shared, + "activities": activityRows, + "limitations": list( + attemptLimitation, + "Runtime finish semantics depend on the driver; observation delay is exported only for exact-attempt handles with validated terminal timestamps.", + "API aggregate counters can contain older attempts; audit uses latest distinct IDs.", + "Transfer work is summed overlapping route time, not wall-clock preparation.", + "Plan assignment work sums are not makespan; API timestamp wall and recovered makespan can differ.", + costLimitation, + "e2-medium interference extrapolated to HPC/micro; micro speedup 0.25 is approximate.", + "Whole-workspace real traffic differs from estimated plan edge volumes.", + "Zero reported interference does not prove absence of physical contention.", + ), + } +} + +type montageAuditAttempts struct { + run, expected, assignments, latest Object + tasks, handles []Object + multiple bool +} + +func indexMontageAuditAttempts(request, response Object) montageAuditAttempts { + run := obj(get(response, "run")) + check(run["status"] == "completed", "Run is not completed") + expected := Object{} + for _, a := range rows(get(obj(get(request, "workflow")), "activities")) { + id := str(get(a, "id")) + check(expected[id] == nil, "Duplicate expected activity IDs") + expected[id] = true + } + assignments := index(get(obj(get(request, "plan")), "assignments"), "activityId") + latest := Object{} + counts := map[string]int{} + multiple := false + tasks := rows(get(response, "activityAttempts")) + handles := rows(response["handles"]) + for _, t := range tasks { + id := str(get(t, "activityId")) + counts[id]++ + old := obj(latest[id]) + if old["attempt"] == nil || num(t["attempt"]) > num(old["attempt"]) || (num(t["attempt"]) == num(old["attempt"]) && num(t["startedAt"]) > num(old["startedAt"])) { + latest[id] = t + } + multiple = multiple || counts[id] > 1 || num(t["attempt"]) > 1 + } + return montageAuditAttempts{run: run, expected: expected, assignments: assignments, latest: latest, tasks: tasks, handles: handles, multiple: multiple} +} + +func matchingAuditHandle(run Object, handles []Object, id string, task Object) Object { + canonical := str(run["id"]) + ":" + id + if num(task["attempt"]) > 1 { + canonical += fmt.Sprintf(":attempt-%.0f", num(task["attempt"])) + } + var best Object + for _, h := range handles { + attempt := obj(h["metadata"])["executionAttempt"] + if h["activityId"] == id && (eq(attempt, task["attempt"]) || (attempt == nil && h["id"] == canonical && h["runId"] == run["id"])) { + if best == nil || num(h["startedAt"]) > num(best["startedAt"]) { + best = h + } + } + } + return best +} + +func validateFreshMontageAudit(attempts montageAuditAttempts, response Object) { + run, expected, tasks, handles := attempts.run, attempts.expected, attempts.tasks, attempts.handles + check(len(tasks) == 58 && len(handles) == 58, "Unexpected attempts or runtime handles in fresh experiment") + for _, t := range tasks { + check(num(t["attempt"]) == 1 && t["status"] == "completed", "Fresh experiment contains retries or failed attempts") + } + for id := range expected { + hs := []Object{} + for _, h := range handles { + a := obj(h["metadata"])["executionAttempt"] + if h["id"] == str(run["id"])+":"+id && h["runId"] == run["id"] && h["activityId"] == id && (a == nil || num(a) == 1) { + hs = append(hs, h) + } + } + check(len(hs) == 1 && hs[0]["status"] == "completed" && hs[0]["exitCode"] != nil && num(hs[0]["exitCode"]) == 0, "Missing successful exact-attempt handle: "+id) + } + b := obj(get(response, "cloudBilling")) + check(b["status"] == "estimated" && b["currency"] == "USD" && num(b["estimatedCost"]) > 0, "Cloud lifecycle prices are incomplete") +} + +func missingAuditDependencyRoutes(request Object, deps []Object) int { + missingRoutes := 0 + for _, e := range rows(obj(request["workflow"])["dependencies"]) { + found := false + for _, t := range deps { + r := obj(t["route"]) + found = found || (r["producerActivityId"] == e["dependsOnActivityId"] && r["consumerActivityId"] == e["activityId"]) + } + if !found { + missingRoutes++ + } + } + return missingRoutes +} + +func auditCrossCloudRelayRoutes(deps []Object, assignments, types Object) (int, int) { + cross, nonRelay := 0, 0 + for _, t := range deps { + r := obj(t["route"]) + s := types[str(obj(get(assignments, str(r["producerActivityId"])))["resourceId"])] + target := types[str(obj(get(assignments, str(r["consumerActivityId"])))["resourceId"])] + isHPC := func(v any) bool { return v == "hpc_machine" || v == "hpc_partition" } + if (s == "cloud_vm" && isHPC(target)) || (target == "cloud_vm" && isHPC(s)) { + cross++ + if t["strategy"] != "gateway" { + nonRelay++ + } + } + } + return cross, nonRelay +} + +func auditMicroLaneOverlaps(request, latest, assignments Object) int { + overlaps := 0 + for _, r := range rows(request["resources"]) { + id := str(r["id"]) + if !strings.Contains(id, "micro") || num(r["cpuCapacity"]) != 1 { + continue + } + intervals := []Object{} + for key, v := range latest { + if obj(assignments[key])["resourceId"] == id { + t := obj(v) + check(t["startedAt"] != nil && t["finishedAt"] != nil && num(t["finishedAt"]) >= num(t["startedAt"]), "Missing or invalid micro execution timestamps") + intervals = append(intervals, t) + } + } + sort.SliceStable(intervals, func(i, j int) bool { return num(intervals[i]["startedAt"]) < num(intervals[j]["startedAt"]) }) + for i := 1; i < len(intervals); i++ { + if num(intervals[i-1]["finishedAt"]) > num(intervals[i]["startedAt"]) { + overlaps++ + } + } + } + return overlaps +} + +type montageAuditTransferEvidence struct { + report Object + transfers, deps []Object + cross int +} + +func validateMontageTransferEvidence(request, response Object, attempts montageAuditAttempts) montageAuditTransferEvidence { + expected, latest, assignments := attempts.expected, attempts.latest, attempts.assignments + transfers := rows(get(response, "artifactTransferRuns")) + deps := []Object{} + for _, t := range transfers { + if obj(t["route"])["producerActivityId"] != nil { + deps = append(deps, t) + } + } + types := Object{} + for _, r := range rows(request["resources"]) { + types[str(r["id"])] = r["type"] + } + incomplete, barriers, leases := 0, 0, 0 + missingRoutes := missingAuditDependencyRoutes(request, deps) + cross, nonRelay := auditCrossCloudRelayRoutes(deps, assignments, types) + for _, t := range transfers { + if t["status"] != "completed" { + incomplete++ + } + task := obj(latest[str(t["activityId"])]) + if task["startedAt"] != nil && num(t["finishedAt"]) > num(task["startedAt"]) { + barriers++ + } + } + for _, l := range rows(get(response, "workspaceLeases")) { + if l["released"] != true { + leases++ + } + } + overlaps := auditMicroLaneOverlaps(request, latest, assignments) + missing, unexpected, notCompleted := []any{}, []any{}, []any{} + for _, id := range sortedKeys(expected) { + if latest[id] == nil { + missing = append(missing, id) + } + if obj(latest[id])["status"] != "completed" { + notCompleted = append(notCompleted, id) + } + } + for _, id := range sortedKeys(latest) { + if expected[id] == nil { + unexpected = append(unexpected, id) + } + } + reportAudit := Object{ + "expected_count": len(expected), + "missing": missing, + "unexpected": unexpected, + "not_completed": notCompleted, + "incomplete_transfer_count": incomplete, + "barrier_violation_count": barriers, + "unreleased_lease_count": leases, + "single_lane_micro_overlap_count": overlaps, + "missing_dependency_route_count": missingRoutes, + "cross_cloud_hpc_non_relay_count": nonRelay, + } + check(len(expected) == 58 && len(missing) == 0 && len(unexpected) == 0 && len(notCompleted) == 0 && incomplete+barriers+leases+overlaps+missingRoutes+nonRelay == 0, "Activity/transfer audit failed") + return montageAuditTransferEvidence{report: reportAudit, transfers: transfers, deps: deps, cross: cross} +} + +func auditTransferRows(transfers []Object) (float64, float64, []any) { + network, relay := 0.0, 0.0 + routeRows := []any{} + for _, t := range transfers { + r := obj(t["route"]) + network += num(t["networkBytes"]) + relay += num(r["serverNetworkBytes"]) + o := Object{ + "consumer_activity_id": t["activityId"], + "producer_activity_id": r["producerActivityId"], + "strategy": t["strategy"], + "duration_seconds": t["durationSeconds"], + "logical_bytes": t["logicalBytes"], + "network_bytes": t["networkBytes"], + "relay_server_network_bytes": r["serverNetworkBytes"], + "archive_bytes": r["archiveBytes"], + "files": r["filesTransferred"], + "snapshot_reused": r["snapshotReused"], + "shared_storage_proof_failure": r["sharedStorageProofFailure"], + "ssh_channel_wait_seconds": r["sshChannelWaitSeconds"], + "rsync_execution_seconds": r["rsyncExecutionSeconds"], + "source_pack_preflight_seconds": r["sourcePackPreflightSeconds"], + "source_pack_preflight_ssh_channel_wait_seconds": r["sourcePackPreflightSshChannelWaitSeconds"], + "source_pack_preflight_service_seconds": r["sourcePackPreflightServiceSeconds"], + "source_pack_ssh_channel_wait_seconds": r["sourcePackSshChannelWaitSeconds"], + "source_pack_execution_seconds": r["sourcePackExecutionSeconds"], + } + phases := []any{} + for _, p := range rows(r["phases"]) { + phases = append(phases, pick(p, "name", "durationSeconds", "sshChannelWaitSeconds", "executionSeconds")) + } + o["phases"] = phases + routeRows = append(routeRows, o) + } + return network, relay, routeRows +} + +func auditSharedPreparationRows(deps []Object) []any { + shared := []any{} + seen := Object{} + for _, t := range deps { + r := obj(t["route"]) + var barrier, merge Object + for _, p := range rows(r["phases"]) { + if p["name"] == "shared-preparation-barrier" { + barrier = p + } + if p["name"] == "shared-conflict-check-copy-and-verify" { + merge = p + } + } + if barrier["startedAt"] == nil || barrier["finishedAt"] == nil { + continue + } + key := fmt.Sprint(r["consumerActivityId"], ":", barrier["startedAt"], ":", barrier["finishedAt"]) + if seen[key] != nil { + continue + } + seen[key] = true + shared = append(shared, Object{ + "consumer_activity_id": r["consumerActivityId"], + "barrier_seconds": barrier["durationSeconds"], + "merge_seconds": merge["durationSeconds"], + "merge_check_seconds": r["mergeCheckSeconds"], + "merge_copy_seconds": r["mergeCopySeconds"], + "merge_verify_seconds": r["mergeVerifySeconds"], + "merge_ssh_channel_wait_seconds": r["mergeSshChannelWaitSeconds"], + "merge_execution_seconds": r["mergeExecutionSeconds"], + "merge_command_overhead_seconds": r["mergeCommandOverheadSeconds"], + }) + } + return shared +} + +func auditActivityRows(attempts montageAuditAttempts) []any { + expected, latest, assignments := attempts.expected, attempts.latest, attempts.assignments + activityRows := []any{} + for _, id := range sortedKeys(expected) { + t, a := obj(latest[id]), obj(get(assignments, id)) + h := matchingAuditHandle(attempts.run, attempts.handles, id, t) + metadata := obj(t["metadata"]) + observed := obj(h["metadata"])["terminalObservedAt"] + var runtime, delay any + if observed != nil && h["finishedAt"] != nil && h["startedAt"] != nil && num(observed) >= num(h["finishedAt"]) && num(h["finishedAt"]) >= num(h["startedAt"]) { + runtime = num(h["finishedAt"]) - num(h["startedAt"]) + delay = num(observed) - num(h["finishedAt"]) + } + activityRows = append(activityRows, Object{ + "activity_id": id, + "resource_id": a["resourceId"], + "attempt": t["attempt"], + "planned_compute_seconds": a["predictedRuntimeSeconds"], + "planned_transfer_seconds": a["predictedTransferSeconds"], + "observed_runtime_seconds": t["runtimeSeconds"], + "validated_handle_runtime_seconds": runtime, + "terminal_observation_delay_seconds": delay, + "preparation_elapsed_seconds": metadata["transferElapsedSeconds"], + "overlapping_transfer_work_seconds": metadata["transferWorkSeconds"], + "launch_wait_seconds": metadata["launchWaitSeconds"], + "queue_seconds": t["queueSeconds"], + "network_bytes": t["transferBytes"], + }) + } + return activityRows +} + +func auditRunTimestampWall(run Object) any { + var wall any + if run["startedAt"] != nil && run["finishedAt"] != nil { + s, e := time.Parse(time.RFC3339Nano, str(run["startedAt"])) + check(e == nil, "invalid run start timestamp") + f, e := time.Parse(time.RFC3339Nano, str(run["finishedAt"])) + check(e == nil, "invalid run finish timestamp") + wall = f.Sub(s).Seconds() + } + return wall +} + +func auditCloudGenerations(billing Object) []any { + generations := []any{} + for _, i := range rows(billing["instances"]) { + generations = append( + generations, + Object{ + "instance_id": i["instanceId"], + "status": i["status"], + "billing": pick( + obj(i["billing"]), + "pricingStatus", + "priceCurrency", + "priceSource", + "priceObservedAt", + "priceRegion", + "priceMachineType", + "startedAt", + "finishedAt", + "computeStartedAt", + "accumulatedComputeSeconds", + "computePricePerSecond", + "diskPricePerSecond", + "asOf", + "currentCost", + "missingPrices", + ), + }, + ) + } + return generations +} diff --git a/internal/montagetools/calibration.go b/internal/montagetools/calibration.go new file mode 100644 index 00000000..d81d9ace --- /dev/null +++ b/internal/montagetools/calibration.go @@ -0,0 +1,253 @@ +package montagetools + +import "strings" + +func validateBaseline(data Object, d string, projection Object) { + if projection != nil { + check(eq(projection["baselineRunId"], data["run_id"]) && projection["auditedEvidenceSHA256"] == d, "Workspace projection baseline provenance mismatch") + } + audit := obj(get(data, "audit")) + check(num(get(audit, "expected_count")) == 58 && get(data, "attempt_history") == "single_observed_attempt_per_activity", "Baseline must contain 58 single-attempt completed activities") + for _, k := range []string{"missing", "unexpected", "not_completed"} { + check(len(arr(get(audit, k))) == 0, "Unsafe baseline "+k) + } + for _, k := range []string{"incomplete_transfer_count", "barrier_violation_count", "unreleased_lease_count", "single_lane_micro_overlap_count", "missing_dependency_route_count", "cross_cloud_hpc_non_relay_count"} { + check(num(get(audit, k)) == 0, "Unsafe baseline "+k) + } + a := rows(get(data, "activities")) + check(len(a) == 58 && len(index(data["activities"], "activity_id")) == 58, "Duplicate or missing baseline activity") +} +func calibration(data Object, d string, projection Object) Object { + validateBaseline(data, d, projection) + activities := rows(data["activities"]) + indexed := index(data["activities"], "activity_id") + work := obj(projection["activityComputeWork"]) + compute := calibrateComputeObservations(activities, work, projection) + phases := calibrationPhaseSamples(data, indexed, projection) + observations := calibratePhaseObservations(phases) + mergeRows := []Object{} + for _, p := range rows(get(data, "shared_preparations")) { + r := clone(p) + id := str(p["consumer_activity_id"]) + r["family"] = family(id) + r["class"] = resourceClass(str(obj(get(indexed, id))["resource_id"])) + mergeRows = append(mergeRows, r) + } + merges := []any{} + for _, g := range groups(mergeRows, "family", "class") { + merges = append(merges, Object{ + "activityFamily": g.key[0], + "resourceClass": g.key[1], + "mergeSeconds": summary(vals(g.rows, "merge_seconds")), + "checkSeconds": summary(vals(g.rows, "merge_check_seconds")), + "copySeconds": summary(vals(g.rows, "merge_copy_seconds")), + "verifySeconds": summary(vals(g.rows, "merge_verify_seconds")), + }) + } + var mergeRates any + if projection != nil { + rates := []any{} + for _, g := range groups(mergeRows, "class") { + o := Object{} + for key, pair := range map[string][2]string{ + "checkSecondsPerInputByte": {"merge_check_seconds", "inputTotalBytes"}, + "copySecondsPerUniqueByte": {"merge_copy_seconds", "inputUniqueBytes"}, + "verifySecondsPerUniqueByte": {"merge_verify_seconds", "inputUniqueBytes"}, + } { + values := []any{} + for _, r := range g.rows { + bytes := num(get(obj(get(obj(get(projection, "observedConsumerInputVolumes")), str(r["consumer_activity_id"]))), pair[1])) + if bytes > 0 { + values = append(values, num(get(r, pair[0]))/bytes) + } + } + o[key] = summary(values) + } + rates = append(rates, Object{ + "targetClass": g.key[0], + "source": "baseline:" + str(data["run_id"]), + "observations": o, + "commandOverheadSeconds": summary(vals(g.rows, "merge_command_overhead_seconds")), + "channelWaitSeconds": summary(vals(g.rows, "merge_ssh_channel_wait_seconds")), + }) + } + mergeRates = rates + } + basis := "unavailable-do-not-use-logical-output-bytes-as-workspace-volume" + if projection != nil { + basis = "producer-final-snapshot-manifest" + } + return Object{ + "schemaVersion": 1, + "baselineRunId": data["run_id"], + "baselineSHA256": d, + "computeWorkBasis": projection["computeWorkBasis"], + "synchronizationContract": "whole-workspace-v1", + "workspaceVolumeBasis": basis, + "computeObservations": compute, + "materializationPhaseObservations": observations, + "consumerMergeObservations": merges, + "mergeRateObservations": mergeRates, + "limitations": list( + "Single baseline; ranges are observations, not confidence intervals.", + "Do not replay per-activity durations or total makespan as a holdout prediction.", + "No destination cache hit or shared-storage route may be assumed without evidence.", + "Workspace manifests and topology-specific contention are required before applying these observations.", + "Compute samples are contention-conditioned; do not apply interference slowdown a second time.", + ), + } +} + +func calibrateComputeObservations(activities []Object, work, projection Object) []any { + computeRows := []Object{} + for _, a := range activities { + p := family(str(a["activity_id"])) + if len(work) > 0 { + p = str(get(obj(get(work, str(a["activity_id"]))), "activityFamily")) + } + r := clone(a) + r["family"] = p + r["class"] = resourceClass(str(a["resource_id"])) + computeRows = append(computeRows, r) + } + compute := []any{} + for _, g := range groups(computeRows, "family", "class") { + times, normalized := []any{}, []any{} + fallback := 0 + for _, a := range g.rows { + runtime := a["validated_handle_runtime_seconds"] + if runtime == nil { + runtime = get(a, "observed_runtime_seconds") + fallback++ + } + times = append(times, runtime) + if projection != nil { + reference := num(get(obj(get(projection, "activityReferenceDurations")), str(a["activity_id"]))) + if len(work) > 0 { + reference = num(get(obj(get(work, str(a["activity_id"]))), "computeWorkUnits")) + } + check(reference > 0, "Invalid reference work") + normalized = append(normalized, num(runtime)/reference) + } + } + compute = append(compute, Object{ + "activityFamily": g.key[0], + "resourceClass": g.key[1], + "runtimeSeconds": summary(times), + "taskRuntimeFallbackSamples": fallback, + "wallSecondsPerReferenceSecond": summary(normalized), + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand", + }) + } + return compute +} + +func calibrationPhaseSamples(data, indexed, projection Object) []Object { + phases := []Object{} + for _, r := range rows(get(data, "routes")) { + if r["producer_activity_id"] == nil { + continue + } + producer := str(r["producer_activity_id"]) + source := obj(get(indexed, producer)) + target := obj(get(indexed, str(r["consumer_activity_id"]))) + for _, p := range rows(get(r, "phases")) { + name := str(get(p, "name")) + if strings.HasPrefix(name, "shared-") { + continue + } + seconds := get(p, "durationSeconds") + basis := "combined-phase-wall-includes-unseparated-service-queue" + var phaseWait, sourceWait any + measured := name == "pack-and-receive" && r["source_pack_execution_seconds"] != nil + if measured { + seconds = r["source_pack_execution_seconds"] + sourceWait = num(r["source_pack_ssh_channel_wait_seconds"]) + basis = "source-command-execution-excludes-channel-queue" + } + if name == "archive-upload" && r["rsync_execution_seconds"] != nil { + seconds = r["rsync_execution_seconds"] + phaseWait = num(r["ssh_channel_wait_seconds"]) + basis = "phase-command-execution-excludes-channel-queue" + } + if p["executionSeconds"] != nil { + seconds = p["executionSeconds"] + phaseWait = num(p["sshChannelWaitSeconds"]) + basis = "phase-command-execution-excludes-channel-queue" + } + var workspace, preflightWait any + if projection != nil { + workspace = get(obj(get(projection, "observedProducerWorkspaceBytes")), producer) + } + if r["source_pack_preflight_service_seconds"] != nil { + preflightWait = num(r["source_pack_preflight_ssh_channel_wait_seconds"]) + } + phases = append(phases, Object{ + "strategy": get(r, "strategy"), + "sourceClass": resourceClass(str(source["resource_id"])), + "targetClass": resourceClass(str(target["resource_id"])), + "phase": name, + "seconds": seconds, + "secondsBasis": basis, + "phaseChannelWaitSeconds": phaseWait, + "sourceChannelWaitSeconds": sourceWait, + "sourcePreflightServiceSeconds": r["source_pack_preflight_service_seconds"], + "sourcePreflightChannelWaitSeconds": preflightWait, + "sourcePreflightSeconds": r["source_pack_preflight_seconds"], + "logicalBytes": r["logical_bytes"], + "archiveBytes": r["archive_bytes"], + "files": r["files"], + "workspaceBytes": workspace, + }) + } + } + return phases +} + +func calibratePhaseObservations(phases []Object) []any { + observations := []any{} + for _, g := range groups(phases, "strategy", "sourceClass", "targetClass", "phase") { + bases := Object{} + ratio, workspaceRate, archiveRate := []any{}, []any{}, []any{} + for _, p := range g.rows { + bases[str(p["secondsBasis"])] = true + if p["archiveBytes"] != nil && num(p["workspaceBytes"]) > 0 { + ratio = append(ratio, num(p["archiveBytes"])/num(p["workspaceBytes"])) + } + if num(p["seconds"]) > 0 { + if p["workspaceBytes"] != nil { + workspaceRate = append(workspaceRate, num(p["workspaceBytes"])/num(p["seconds"])) + } + if p["archiveBytes"] != nil { + archiveRate = append(archiveRate, num(p["archiveBytes"])/num(p["seconds"])) + } + } + } + row := Object{ + "strategy": g.key[0], + "sourceClass": g.key[1], + "targetClass": g.key[2], + "phase": g.key[3], + "seconds": summary(vals(g.rows, "seconds")), + "secondsBases": list(sortedKeys(bases)...), + "pooledArchiveServiceRate": pooledRate(g.rows, "archiveBytes"), + "pooledWorkspaceServiceRate": pooledRate(g.rows, "workspaceBytes"), + "phaseChannelWaitSeconds": summary(vals(g.rows, "phaseChannelWaitSeconds")), + "archiveToWorkspaceRatio": summary(ratio), + "effectiveWorkspaceBytesPerSecond": summary(workspaceRate), + "effectiveArchiveBytesPerSecond": summary(archiveRate), + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth", + } + for _, k := range []string{"sourceChannelWaitSeconds", "sourcePreflightSeconds", "sourcePreflightServiceSeconds", "sourcePreflightChannelWaitSeconds"} { + row[k] = nil + if g.key[3] == "pack-and-receive" { + row[k] = summary(vals(g.rows, k)) + } + } + for _, k := range []string{"logicalBytes", "archiveBytes", "workspaceBytes", "files"} { + row[k] = summary(vals(g.rows, k)) + } + observations = append(observations, row) + } + return observations +} diff --git a/internal/montagetools/class_holdout.go b/internal/montagetools/class_holdout.go new file mode 100644 index 00000000..5a76cd15 --- /dev/null +++ b/internal/montagetools/class_holdout.go @@ -0,0 +1,132 @@ +package montagetools + +import ( + "encoding/json" + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain/workflow" + "sort" +) + +// Prepare a private simulation-only case with one VM and one execution lane. +// No API call, provisioning, real execution, or runtime catalog mutation. +func prepareClassHoldout(template, calibration Object, target, runID, planID string) Object { + check(runPattern.MatchString(runID) && runPattern.MatchString(planID), "invalid holdout identity") + var typed ports.ExecutionRequest + check(json.Unmarshal(encode(template), &typed) == nil, "invalid template") + out := Object{} + check(json.Unmarshal(encode(typed), &out) == nil, "cannot normalize template") + var resource Object + for _, r := range rows(out["Resources"]) { + if r["id"] == target { + resource = r + } + } + check(resource != nil && resource["type"] == "cloud_vm" && resource["schedulable"] == true, "eligible existing cloud target required") + metadata := obj(resource["metadata"]) + metadata["maximumConcurrentActivities"] = 1 + resource["metadata"] = metadata + class := resourceClass(target) + model := obj(obj(obj(out["NetworkTopology"])["metadata"])["workspaceMaterializationModel"]) + model["schemaVersion"] = 2 + calModel := obj(obj(obj(calibration["NetworkTopology"])["metadata"])["workspaceMaterializationModel"]) + for _, key := range []string{"mergeProfiles", "inputObservationProfiles", "publicationProfiles"} { + profiles := []any{} + found := false + for _, p := range rows(model[key]) { + if p["targetClass"] != class { + profiles = append(profiles, p) + } + } + for _, p := range rows(calModel[key]) { + if p["targetClass"] == class { + check(num(p["samples"]) > 0 && p["source"] != nil, "profile evidence required") + profiles = append(profiles, p) + found = true + } + } + check(found, "missing calibrated stage for class "+class) + model[key] = profiles + } + provenance := obj(obj(obj(calibration["NetworkTopology"])["metadata"])["runtimeStageCalibration"]) + check(provenance["targetClass"] == class, "calibration class mismatch") + fixed := num(provenance["fixedLaunchAndRuntimeOverheadSeconds"]) + resource["containerOverheadSeconds"] = fixed + profiles, err := workflow.CompleteSpeedupProfiles(typed.Workflow.Activities, typed.Resources, typed.ActivityProfiles) + check(err == nil, "cannot complete compute estimates") + out["ActivityProfiles"] = profiles + ordered := orderedHoldoutActivities(typed.Workflow) + assignments := []any{} + for i, id := range ordered { + assignments = append(assignments, Object{"id": planID + "-" + id, "planId": planID, "activityId": id, "resourceId": target, "coreId": target + "-core-1", "orderOnResource": i, "metadata": Object{"containerOverheadSeconds": fixed}}) + } + plan := obj(out["Plan"]) + plan["id"] = planID + plan["assignments"] = assignments + plan["lifecycleActions"] = []any{} + plan["source"] = "manual" + plan["algorithm"] = "fixed-class-holdout" + plan["metadata"] = Object{ + "reviewOnly": true, + "realExecutionAuthorized": false, + "experimentRole": "independent-class-holdout-preparation-only", + "targetClass": class, + "oneVmOneExecutionLane": true, + "trainingEvidence": provenance, + "priceStatus": "historical-estimate-revalidate-before-real", + } + plan["metadata"].(map[string]any)["remainingRealPreflight"] = list( + "User approval required.", + "Register/freeze lifecycle actions and include provisioning/disk costs before real.", + "Revalidate quota, SSH, runtime and physical disk availability.", + ) + out["Run"] = Object{"id": runID, "schedulePlanId": planID, "mode": "simulation", "seed": 1} + out["RuntimeAllocations"] = Object{} + return out +} + +func orderedHoldoutActivities(version workflow.WorkflowVersion) []string { + nodes := map[string]bool{} + indegree := map[string]int{} + successors := map[string][]string{} + edges := map[string]bool{} + for _, a := range version.Activities { + nodes[a.ID] = true + indegree[a.ID] = 0 + } + add := func(parent, child string) { + check(nodes[parent] && nodes[child], "unknown dependency") + key := parent + "\x00" + child + if !edges[key] { + edges[key] = true + indegree[child]++ + successors[parent] = append(successors[parent], child) + } + } + for _, d := range version.Dependencies { + add(d.DependsOnActivityID, d.ActivityID) + } + for _, d := range version.DataDependencies { + add(d.ProducerActivityID, d.ConsumerActivityID) + } + ordered := []string{} + ready := []string{} + for id, n := range indegree { + if n == 0 { + ready = append(ready, id) + } + } + for len(ready) > 0 { + sort.Strings(ready) + id := ready[0] + ready = ready[1:] + ordered = append(ordered, id) + for _, child := range successors[id] { + indegree[child]-- + if indegree[child] == 0 { + ready = append(ready, child) + } + } + } + check(len(ordered) == len(nodes), "cyclic workflow cannot be serialized") + return ordered +} diff --git a/internal/montagetools/class_holdout_test.go b/internal/montagetools/class_holdout_test.go new file mode 100644 index 00000000..61fa7371 --- /dev/null +++ b/internal/montagetools/class_holdout_test.go @@ -0,0 +1,48 @@ +package montagetools + +import ( + "encoding/json" + "github.com/stretchr/testify/require" + "os" + "path/filepath" + "testing" +) + +func TestPreparedHoldoutIsSimulationOnlyAndTopologicallySerialized(t *testing.T) { + r, _ := stageFixture() + rid := "montage-gcp-fog-e2-micro-01" + for _, a := range rows(obj(r["Workflow"])["activities"]) { + a["activityTypeId"] = "fixture" + a["simulation"] = Object{"durationSeconds": 1} + } + rows(r["Resources"])[0]["computeSpeedup"] = 1 + r["ActivityProfiles"] = []any{Object{"id": "measured", "activityTypeId": "fixture", "resourceId": rid, "runtimeSeconds": 1, "modelVersion": "reference-work-v1", "sampleSize": 1}} + p := Object{"targetClass": "e2-micro", "source": "fixture", "samples": 1} + cal := Object{ + "NetworkTopology": Object{ + "metadata": Object{ + "runtimeStageCalibration": Object{"targetClass": "e2-micro", "fixedLaunchAndRuntimeOverheadSeconds": 2}, + "workspaceMaterializationModel": Object{"mergeProfiles": []any{p}, "inputObservationProfiles": []any{p}, "publicationProfiles": []any{p}}, + }, + }, + } + out := prepareClassHoldout(r, cal, rid, "montage-holdout-sim", "montage-holdout-plan") + require.Equal(t, "simulation", obj(out["Run"])["mode"]) + require.Equal(t, false, obj(obj(out["Plan"])["metadata"])["realExecutionAuthorized"]) + assignments := rows(obj(out["Plan"])["assignments"]) + require.Equal(t, "new-rootid0000001", assignments[0]["activityId"]) + require.Equal(t, "new-childid0000002", assignments[1]["activityId"]) + require.Equal(t, assignments[0]["coreId"], assignments[1]["coreId"]) + require.NotContains(t, obj(rows(r["Resources"])[0]["metadata"]), "maximumConcurrentActivities") +} + +func TestNativeVerificationRejectsRealBeforeDockerOrOutputCreation(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "request.json") + b, _ := json.Marshal(Object{"Run": Object{"mode": "real"}}) + require.NoError(t, os.WriteFile(path, b, 0600)) + out := filepath.Join(dir, "output") + require.Panics(t, func() { verifyNativeCase(path, "unused", "/case/unused", out) }) + _, err := os.Stat(out) + require.True(t, os.IsNotExist(err)) +} diff --git a/internal/montagetools/cli.go b/internal/montagetools/cli.go new file mode 100644 index 00000000..bdbc9cc1 --- /dev/null +++ b/internal/montagetools/cli.go @@ -0,0 +1,265 @@ +package montagetools + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// Run returns bounded errors rather than raw API inputs. This CLI never reads +// database.db, submits runs, provisions machines or restarts the daemon. +func Run(args []string) (err error) { + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("%v", p) + } + }() + check(len(args) > 0, "Montage tool command required") + command := strings.ReplaceAll(args[0], "-", "_") + args = args[1:] + for _, dispatch := range []func(string, []string) bool{runObservationCommand, runPreparationCommand, runPreservationCommand} { + if dispatch(command, args) { + return nil + } + } + if command == "validate_montage_transfer_model" { + runTransferModelCommand(args) + return nil + } + panic("Unknown Montage tool command: " + command) +} + +func runObservationCommand(command string, args []string) bool { + load := func(i int) Object { m, _ := read(args[i]); return m } + arity := func(n int, usage string) { check(len(args) == n, usage) } + switch command { + case "verify_native_case": + arity(4, "verify_native_case REQUEST IMAGE BINARY_IN_CASE NEW_OUTPUT_DIR") + verifyNativeCase(args[0], args[1], args[2], args[3]) + case "runtime_reference_coverage": + check(len(args) >= 2, "runtime_reference_coverage OUTPUT REAL...") + references := []Object{} + for _, path := range args[1:] { + r, _ := read(path) + references = append(references, r) + } + write(args[0], runtimeReferenceCoverage(references)) + case "calibrate_runtime_stages": + arity(4, "calibrate_runtime_stages CLASS REQUEST TRAINING_REAL OUTPUT") + training, raw := read(args[2]) + write(args[3], calibrateRuntimeStages(load(1), training, digest(raw), args[0])) + case "calibrate_hpc_stages": + arity(3, "calibrate_hpc_stages REQUEST TRAINING_REAL OUTPUT") + training, raw := read(args[1]) + write(args[2], calibrateHPCStages(load(0), training, digest(raw))) + case "transfer_wall_sensitivity": + arity(3, "transfer_wall_sensitivity REQUEST REAL OUTPUT") + real, raw := read(args[1]) + write(args[2], transferWallSensitivity(load(0), real, digest(raw))) + case "compare_activity_times": + arity(4, "compare_activity_times REQUEST REAL SIM OUTPUT") + write(args[3], compareActivityTimes(load(0), load(1), load(2))) + case "compare_completed_dependency_reference": + arity(3, "compare_completed_dependency_reference REAL SIM OUTPUT") + write(args[2], dependencyReference(load(0), load(1))) + case "combined_observation_sensitivity": + arity(3, "combined_observation_sensitivity REQUEST REAL OUTPUT") + real, raw := read(args[1]) + write(args[2], combinedObservationSensitivity(load(0), real, digest(raw))) + case "publication_observation_reference": + arity(2, "publication_observation_reference REAL OUTPUT") + real, raw := read(args[0]) + write(args[1], publicationReference(real, digest(raw))) + case "input_observation_sensitivity": + arity(3, "input_observation_sensitivity REQUEST REAL OUTPUT") + real, raw := read(args[1]) + write(args[2], observationSensitivity(load(0), real, digest(raw))) + default: + return false + } + return true +} + +func runPreparationCommand(command string, args []string) bool { + load := func(i int) Object { m, _ := read(args[i]); return m } + arity := func(n int, usage string) { check(len(args) == n, usage) } + switch command { + case "prepare_class_holdout": + arity(6, "prepare_class_holdout TEMPLATE CALIBRATION TARGET RUN_ID PLAN_ID OUTPUT") + write(args[5], prepareClassHoldout(load(0), load(1), args[2], args[3], args[4])) + case "prepare_instrumented_round": + arity(5, "prepare_instrumented_round REQUEST SIM RUN_ID PLAN_ID OUTPUT_DIR") + request, rawRequest := read(args[0]) + simulation, rawSimulation := read(args[1]) + writeInstrumentedRound(request, simulation, rawRequest, rawSimulation, args[2], args[3], args[4]) + case "build_montage_execution_calibration": + check(len(args) == 2 || len(args) == 3, "build_montage_execution_calibration AUDIT OUTPUT [PROJECTION]") + data, p := read(args[0]) + var projection Object + if len(args) == 3 { + projection = load(2) + } + write(args[1], calibration(data, digest(p), projection)) + case "build_montage_workspace_projection": + arity(4, "build_montage_workspace_projection AUDIT API REQUEST OUTPUT") + a, p := read(args[0]) + write(args[3], projection(a, load(1), load(2), digest(p))) + case "freeze_montage_materialization": + arity(4, "freeze_montage_materialization CALIBRATION RESOURCES RELAY OUTPUT") + c, p := read(args[0]) + write(args[3], freezeMaterialization(c, digest(p), load(1), args[2])) + case "freeze_montage_compute_profiles": + arity(4, "freeze_montage_compute_profiles CALIBRATION RESOURCES NAMESPACE OUTPUT") + c, p := read(args[0]) + write(args[3], freezeCompute(c, digest(p), load(1), args[2])) + case "build_montage_work_features": + arity(4, "build_montage_work_features REQUEST INPUTS PROJECTION OUTPUT") + p := load(2) + p["activityComputeWork"] = workFeatures(load(0), load(1)) + p["computeWorkBasis"] = "immutable-dataset-independent-work-v1" + p["limitations"] = append(arr(get(p, "limitations")), "Non-project operation units apply only to this fixed dataset; arbitrary dataset scaling is not calibrated.") + write(args[3], p) + case "extract_montage_compute_work": + arity(4, "extract_montage_compute_work CONTAINER SIF PROJECTION OUTPUT") + write(args[3], extractWork(args[0], args[1], load(2))) + case "supplement_montage_root_compute": + arity(5, "supplement_montage_root_compute CALIBRATION PROJECTION REQUEST RESPONSE OUTPUT") + response, p := read(args[3]) + write(args[4], supplementRoots(load(0), load(1), load(2), response, digest(p))) + case "prepare_montage_measured_requests": + prepareRequests(args) + default: + return false + } + return true +} + +func runPreservationCommand(command string, args []string) bool { + load := func(i int) Object { m, _ := read(args[i]); return m } + arity := func(n int, usage string) { check(len(args) == n, usage) } + switch command { + case "audit_montage_results": + check(len(args) == 4 || len(args) == 5, "audit_montage_results REQUEST REAL SIM PRODUCT [OUTPUT]") + result := audit(load(0), load(1), load(2), os.Getenv("MONTAGE_REQUIRE_FRESH") == "1") + p, e := os.ReadFile(args[3]) + check(e == nil, "read product failed") + result["product"] = Object{"filename": filepath.Base(args[3]), "bytes": len(p), "sha256": digest(p)} + if len(args) == 5 { + write(args[4], result) + } else { + fmt.Print(string(encode(result))) + } + case "export_cloud_workspace": + arity(3, "export_cloud_workspace TARGET RUN OUTPUT.tar.gz") + exportCloud(args[0], args[1], args[2]) + case "export_hpc_workspace": + arity(3, "export_hpc_workspace TARGET RUN OUTPUT.tar.gz") + exportHPC(args[0], args[1], args[2]) + default: + return false + } + return true +} + +func runTransferModelCommand(args []string) { + load := func(i int) Object { m, _ := read(args[i]); return m } + arity := func(n int, usage string) { check(len(args) == n, usage) } + check(len(args) > 0, "validation mode required") + mode := args[0] + args = args[1:] + switch mode { + case "calibrate": + arity(5, "calibrate AUDIT RAW REQUEST RESOURCES OUTPUT_PREFIX") + a, p := read(args[0]) + v := measuredVolumes(a, load(1), load(2), digest(p)) + c := calibration(a, digest(p), v) + m := freezeMaterialization(c, digest(encode(c)), load(3), "montage-relay-controller") + outputs := Object{"volumes": v, "calibration": c, "model": m} + for name, value := range outputs { + write(args[4]+"-"+name+".json", value) + } + case "request": + arity(5, "request ORIGINAL MODEL AUDIT RUN_ID OUTPUT") + write(args[4], fixedRequest(load(0), load(1), load(2), args[3])) + case "supplement": + arity(4, "supplement CALIBRATION DIAGNOSTIC RESOURCES OUTPUT_PREFIX") + diagnostic, p := read(args[1]) + c := supplementSource(load(0), diagnostic, digest(p)) + resources := load(2) + for _, r := range resources { + resource := obj(r) + if get(resource, "domain") == "hpc" { + resource["bulkServiceGroupId"] = "plafrim-reference-ssh-bulk" + resource["bulkServiceSlots"] = 3 + resource["bulkServiceSource"] = "AKO-148 SSHChannelLimit4 reserve1; diagnostic source waits prove two waves of three" + } + } + m := freezeMaterialization(c, digest(encode(c)), resources, "montage-relay-controller") + write(args[3]+"-calibration.json", c) + write(args[3]+"-model.json", m) + case "compare": + arity(5, "compare AUDIT SIM_RESULT MODEL ROLE OUTPUT") + result := compare(load(0), load(1), args[3], load(2)) + write(args[4], result) + delete(result, "dependencies") + delete(result, "dominantTransfers") + fmt.Print(string(encode(result))) + case "shared-capacity": + arity(4, "shared-capacity REQUEST DIAGNOSTIC HPC_RESOURCE OUTPUT_PREFIX") + diagnostic, p := read(args[1]) + c := sourceCapacity(diagnostic, args[2], "e2-micro") + c["runId"] = get(obj(get(diagnostic, "run")), "id") + c["sha256"] = digest(p) + write(args[3]+"-request.json", applyCapacity(load(0), args[2], c)) + write(args[3]+"-capacity.json", c) + case "report": + arity(2, "report PRIVATE_EXPERIMENT_DIR OUTPUT") + write(args[1], validationReport(args[0])) + default: + panic("Use calibrate, supplement, request, compare, shared-capacity or report; no real runs are submitted") + } +} + +func validationReport(base string) Object { + cases := []any{} + for _, name := range []string{"ako151-original-comparison", "ako151-historical-comparison", "ako151-clean-v2-comparison", "ako151-shared-source-v2-comparison"} { + r, _ := read(filepath.Join(base, name+".json")) + delete(r, "dependencies") + cases = append(cases, r) + } + provenance := Object{} + for _, name := range []string{"ako151-clean-v2-calibration", "ako151-source-pool-v3-calibration", "ako151-source-pool-v3-model", "ako151-shared-source-v2-capacity"} { + r, p := read(filepath.Join(base, name+".json")) + provenance[name] = Object{"sha256": digest(p), "evidence": r} + } + previous := Object{} + for _, name := range []string{"montage-clean-simulation-20261011-v1-request.json", "selected-plan.json", "simulation-results.json"} { + _, p := read(filepath.Join(base, name)) + previous[name] = digest(p) + } + return Object{ + "schemaVersion": 1, + "issue": "AKO-151", + "realExecutionStarted": false, + "daemonRestarted": false, + "validationMode": "native SimGrid offline; exact production export; fixed frozen observed placement and compute", + "previousForecastFilesSHA256": previous, + "cases": cases, + "provenance": provenance, + "gate": Object{ + "accepted": false, + "newPlanningReleased": false, + "independentRealPrepared": false, + "reason": "Shared source-capacity fit meets makespan tolerance but dominant dependency residuals still exceed 100%; do not accept compensating errors.", + }, + "remainingLimitations": list( + "Receiver command service is not identifiable separately from queue in completed references. No destination pool is added to those mixed samples.", + "Nine completed source commands and measured HPC merges from an interrupted diagnostic are auxiliary calibration evidence, not a completed-run benchmark.", + "Aggregate source capacity varies with workload/concurrency; constant-rate fit cannot explain every dominant dependency.", + "New planning must retain only evidenced route classes. Unobserved placement/storage identity must not be silently inferred.", + "Provisioning and orchestration timing retained from the frozen original forecast; not yet independently validated.", + "Compute and lifecycle cost bases unchanged; e2-medium interference remains extrapolated and network/public-IP billing remains incomplete.", + ), + } +} diff --git a/internal/montagetools/dependency_reference.go b/internal/montagetools/dependency_reference.go new file mode 100644 index 00000000..a8091551 --- /dev/null +++ b/internal/montagetools/dependency_reference.go @@ -0,0 +1,71 @@ +package montagetools + +import ( + "sort" + "strings" +) + +func dependencyReference(real, sim Object) Object { + check(obj(get(real, "run"))["status"] == "completed", "completed real reference required") + latest := map[string]Object{} + for _, transfer := range rows(get(real, "artifactTransferRuns")) { + if transfer["status"] != "completed" { + continue + } + route := obj(transfer["route"]) + if route["producerActivityId"] == nil || route["consumerActivityId"] == nil { + continue + } + key := str(route["producerActivityId"]) + "\x00" + str(route["consumerActivityId"]) + if latest[key] == nil || num(transfer["startedAt"]) > num(latest[key]["startedAt"]) { + latest[key] = transfer + } + } + predicted := map[string]Object{} + for _, transfer := range rows(get(sim, "transfers")) { + predicted[str(get(transfer, "producerActivityId"))+"\x00"+str(get(transfer, "consumerActivityId"))] = transfer + } + records := []any{} + for key, transfer := range latest { + route := obj(transfer["route"]) + row := Object{"producer": route["producerActivityId"], "consumer": route["consumerActivityId"], "observedStrategy": transfer["strategy"], "observedNetworkBytes": transfer["networkBytes"]} + if transfer["strategy"] == "use-existing" { + row["comparable"] = false + row["reason"] = "recovery reuses validated copies; no fresh transfer duration" + } else { + var wall float64 + for _, phase := range rows(route["phases"]) { + if !strings.HasPrefix(str(get(phase, "name")), "shared-") { + wall += num(phase["durationSeconds"]) + } + } + if transfer["strategy"] == "runtime-local" { + wall = 0 + } + row["observedDependencyWallSeconds"] = wall + estimate, exists := predicted[key] + row["comparable"] = exists + if exists { + row["predictedStrategy"] = estimate["strategy"] + row["predictedNetworkBytes"] = estimate["networkBytes"] + row["predictedDependencySeconds"] = estimate["durationSeconds"] + row["durationErrorPercent"] = errorPercent(num(estimate["durationSeconds"]), wall) + } + } + records = append(records, row) + } + sort.Slice(records, func(i, j int) bool { + return num(obj(records[i])["observedDependencyWallSeconds"]) > num(obj(records[j])["observedDependencyWallSeconds"]) + }) + return Object{ + "referenceRunId": obj(real["run"])["id"], + "role": "recovered-reference-residuals-not-independent-validation", + "dependencies": records, + "limitations": list( + "Shared consumer barrier and merge phases are excluded from each dependency to avoid duplicate counting.", + "Observed phase wall time can contain admission wait; this is not queue-free service calibration.", + "Recovery copy reuse is explicitly excluded from fresh duration comparisons.", + "No parallel dependency durations are summed as makespan.", + ), + } +} diff --git a/internal/montagetools/dependency_reference_test.go b/internal/montagetools/dependency_reference_test.go new file mode 100644 index 00000000..c1143c98 --- /dev/null +++ b/internal/montagetools/dependency_reference_test.go @@ -0,0 +1,26 @@ +package montagetools + +import "testing" + +func TestDependencyReferenceExcludesSharedBarrierAndRecoveryReuse(t *testing.T) { + real := Object{"run": Object{"status": "completed", "id": "real"}, "artifactTransferRuns": []any{ + Object{ + "status": "completed", + "startedAt": 1.0, + "strategy": "direct-runtime", + "networkBytes": 100.0, + "route": Object{ + "producerActivityId": "a", + "consumerActivityId": "b", + "phases": []any{Object{"name": "upload", "durationSeconds": 4.0}, Object{"name": "shared-preparation-barrier", "durationSeconds": 10.0}, Object{"name": "shared-conflict-check-copy-and-verify", "durationSeconds": 3.0}}, + }, + }, + Object{"status": "completed", "startedAt": 2.0, "strategy": "use-existing", "route": Object{"producerActivityId": "c", "consumerActivityId": "b"}}, + }} + sim := Object{"transfers": []any{Object{"producerActivityId": "a", "consumerActivityId": "b", "strategy": "direct-runtime", "durationSeconds": 2.0, "networkBytes": 100.0}}} + report := dependencyReference(real, sim) + rows := rows(report["dependencies"]) + if rows[0]["observedDependencyWallSeconds"] != 4.0 || rows[0]["durationErrorPercent"] != -50.0 || rows[1]["comparable"] != false { + t.Fatal("shared merge or recovery reuse counted as dependency service") + } +} diff --git a/internal/montagetools/export.go b/internal/montagetools/export.go new file mode 100644 index 00000000..eaf43dbd --- /dev/null +++ b/internal/montagetools/export.go @@ -0,0 +1,84 @@ +package montagetools + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "regexp" + "strings" + "time" +) + +var targetPattern = regexp.MustCompile(`^montage-gcp-[a-z0-9-]+$`) +var runPattern = regexp.MustCompile(`^montage-[a-z0-9-]+$`) + +func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\"'\"'") + "'" } + +// Export is deliberately explicit and API-only. No command ever exposes the +// token or private console output in an error, and existing archives are safe. +func exportCloud(target, run, output string) { + check(targetPattern.MatchString(target) && runPattern.MatchString(run), "invalid target or run identity") + exportRemoteWorkspace(target, run, output, "/akoflow/workspace/runs/"+run) +} + +func exportHPC(target, run, output string) { + check((target == "plafrim-execution-ssh-node-diablo03" || target == "plafrim-execution-ssh-node-bora001") && runPattern.MatchString(run), "invalid HPC target or run identity") + exportRemoteWorkspace(target, run, output, "/home/wferreir/akoflow-workspaces/"+run) +} + +func exportRemoteWorkspace(target, run, output, sourcePath string) { + token, ok := os.LookupEnv("AKOFLOW_API_TOKEN") + check(ok && token != "", "AKOFLOW_API_TOKEN required") + client := http.Client{Timeout: 660 * time.Second} + console := func(command string) string { + payload, _ := json.Marshal(Object{"resourceId": target, "command": command, "timeoutSeconds": 600}) + request, e := http.NewRequestWithContext(context.Background(), http.MethodPost, "http://127.0.0.1:8080/akoflow-api/console-commands/", bytes.NewReader(payload)) + check(e == nil, "console request failed") + request.Header.Set("Authorization", "Bearer "+token) + request.Header.Set("Content-Type", "application/json") + response, e := client.Do(request) + check(e == nil, "console transport failed") + defer response.Body.Close() + check(response.StatusCode >= 200 && response.StatusCode < 300, fmt.Sprintf("console HTTP %d", response.StatusCode)) + var result Object + check(json.NewDecoder(io.LimitReader(response.Body, 8<<20)).Decode(&result) == nil, "invalid console response") + check(result["status"] == "completed" && result["exitCode"] != nil && num(result["exitCode"]) == 0, "remote archive command failed") + return str(get(result, "stdout")) + } + source := shellQuote(sourcePath) + archive := shellQuote("/tmp/akoflow-preserve-" + run + ".tar.gz") + metadata := strings.Fields(console("set -eu; test -d " + source + "; tar -czf " + archive + " -C " + source + " .; stat -c %s " + archive + "; sha256sum " + archive)) + check(len(metadata) >= 2, "invalid archive metadata") + size := int64(num(metadata[0])) + hash := metadata[1] + check(size > 0 && size <= 512*1024*1024 && imageDigestPattern.MatchString(hash), "invalid archive metadata") + _, e := os.Stat(output) + check(os.IsNotExist(e), "output already exists or inaccessible") + partial := output + ".partial" + file, e := os.OpenFile(partial, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + check(e == nil, "partial output already exists or cannot be created") + defer file.Close() + chunk := int64(4 * 1024 * 1024) + for offset := int64(0); offset < size; offset += chunk { + count := chunk + if count > size-offset { + count = size - offset + } + encoded := console(fmt.Sprintf("dd if=%s bs=1M skip=%d count=4 status=none | base64 -w0", archive, offset/(1024*1024))) + decoded, e := base64.StdEncoding.Strict().DecodeString(strings.TrimSpace(encoded)) + check(e == nil && int64(len(decoded)) == count, "archive chunk length mismatch") + _, e = file.Write(decoded) + check(e == nil, "archive write failed") + fmt.Printf("%s: %d/%d bytes preserved\n", target, offset+count, size) + } + check(file.Close() == nil, "archive close failed") + payload, e := os.ReadFile(partial) + check(e == nil && digest(payload) == hash, "archive checksum mismatch; partial retained") + check(os.Rename(partial, output) == nil, "archive rename failed") + fmt.Printf("VERIFIED %s: %d bytes SHA256 %s\n", target, size, hash) +} diff --git a/internal/montagetools/features.go b/internal/montagetools/features.go new file mode 100644 index 00000000..7f0a1c13 --- /dev/null +++ b/internal/montagetools/features.go @@ -0,0 +1,264 @@ +package montagetools + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "math" + "os/exec" + "regexp" + "strconv" + "strings" +) + +func fitsHeader(reader io.Reader) Object { + values := Object{} + card := make([]byte, 80) + for i := 0; i < 16384; i++ { + _, e := io.ReadFull(reader, card) + check(e == nil, "Truncated FITS header") + key := strings.TrimSpace(string(card[:8])) + if key == "END" { + return values + } + if key == "BITPIX" || key == "NAXIS" || key == "NAXIS1" || key == "NAXIS2" { + check(string(card[8:10]) == "= ", "Invalid FITS numeric card") + v, e := strconv.Atoi(strings.TrimSpace(strings.SplitN(string(card[10:]), "/", 2)[0])) + check(e == nil, "Invalid FITS numeric card") + values[key] = v + } + } + panic("FITS header exceeds bounded input limit") +} +func inputFeature(h Object) Object { + check(num(get(h, "NAXIS")) == 2, "Expected two-dimensional FITS input") + x, y := num(get(h, "NAXIS1")), num(get(h, "NAXIS2")) + check(x > 0 && y > 0, "Invalid image dimensions") + return Object{"unit": "input-megapixels", "computeWorkUnits": x * y / 1000000, "naxis1": x, "naxis2": y, "bitpix": h["BITPIX"]} +} + +// shellWords only parses command arguments; it never executes them. +func shellWords(text string) []string { + result := []string{} + var word strings.Builder + quote := rune(0) + escaped, started := false, false + for _, r := range text { + if escaped { + word.WriteRune(r) + escaped = false + started = true + continue + } + if r == '\\' && quote != '\'' { + escaped = true + started = true + continue + } + if quote != 0 { + if r == quote { + quote = 0 + } else { + word.WriteRune(r) + } + continue + } + if r == '\'' || r == '"' { + quote = r + started = true + continue + } + if r == ' ' || r == '\t' || r == '\n' { + if started { + result = append(result, word.String()) + word.Reset() + started = false + } + continue + } + word.WriteRune(r) + started = true + } + check(quote == 0 && !escaped, "Unmatched shell quoting") + if started { + result = append(result, word.String()) + } + return result +} +func workFeatures(request, inputs Object) Object { + paths := index(get(inputs, "inputs"), "path") + dependencies := Object{} + workflow := obj(get(request, "workflow")) + for _, d := range rows(get(workflow, "dependencies")) { + id := str(d["activityId"]) + dependencies[id] = append(arr(dependencies[id]), d) + } + result := Object{} + for _, a := range rows(get(workflow, "activities")) { + id := str(get(a, "id")) + f := family(id) + check(strings.Contains("|mproject|mdifffit|mconcatfit|mbgmodel|mbackground|mimgtbl|madd|mviewer|", "|"+f+"|"), "Unknown Montage family") + arguments := arr(get(obj(get(a, "command")), "arguments")) + check(len(arguments) > 0, "Missing activity command") + words := shellWords(str(arguments[len(arguments)-1])) + unit, work := "fixed-dataset-operation", 1.0 + if f == "mproject" { + source := []string{} + for _, w := range words { + if paths[w] != nil { + source = append(source, w) + } + } + check(len(source) == 1, "Ambiguous mProject input work") + feature := obj(paths[source[0]]) + f += "_" + str(get(feature, "band")) + unit = str(get(feature, "unit")) + work = num(get(feature, "computeWorkUnits")) + } else if f == "mconcatfit" { + check(dependencies[id] != nil, "Missing predecessor work") + unit, work = "predecessor-fit-record", float64(len(arr(dependencies[id]))) + } else if f == "mviewer" { + channels := 0 + for _, w := range words { + if w == "-red" || w == "-green" || w == "-blue" || w == "-gray" { + channels++ + } + } + check(channels == 1 || channels == 3, "Missing viewer channel work") + unit, work = "image-channel", float64(channels) + } + check(work > 0 && !math.IsNaN(work) && !math.IsInf(work, 0), "Invalid independent compute work") + result[id] = Object{"activityName": get(a, "name"), "activityFamily": f, "computeWorkUnits": work, "unit": unit, "sourceImageSHA256": get(inputs, "sourceImageSHA256")} + } + return result +} + +var imageDigestPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) +var inputPathPattern = regexp.MustCompile(`^poss2ukstu_(blue|red|ir)_\d{3}_\d{3}\.fits$`) + +func dockerRead(args ...string) []byte { + cmd := exec.Command("docker", args...) + var diagnostics bytes.Buffer + cmd.Stderr = &diagnostics + out, e := cmd.Output() + check(e == nil, "Scientific container read failed (diagnostic output withheld)") + return out +} +func extractWork(container, image string, p Object) Object { + listing := string(dockerRead("exec", container, "apptainer", "sif", "list", image)) + offset := "" + for _, line := range strings.Split(listing, "\n") { + if strings.Contains(line, "FS (Squashfs/") { + parts := strings.Split(line, "|") + check(len(parts) > 3, "Invalid SIF inventory") + value, e := strconv.Atoi(strings.TrimSpace(strings.SplitN(strings.TrimSpace(parts[3]), "-", 2)[0])) + check(e == nil, "Invalid SIF partition offset") + offset = strconv.Itoa(value) + break + } + } + check(offset != "", "Missing Squashfs partition") + hashFields := strings.Fields(string(dockerRead("exec", container, "sha256sum", image))) + check(len(hashFields) > 0 && imageDigestPattern.MatchString(hashFields[0]), "Invalid source image identity") + models := obj(get(p, "activityModels")) + keys := sortedKeys(models) + check(len(keys) > 0, "Missing packaging model") + packaging := rows(get(obj(models[keys[0]]), "inputs")) + inputs := []Object{} + for _, f := range packaging { + if inputPathPattern.MatchString(str(f["path"])) { + inputs = append(inputs, f) + } + } + check(len(inputs) == 12, "Expected twelve immutable input images") + sortRows(inputs, "path") + features := []any{} + for _, f := range inputs { + path := str(f["path"]) + compressed := dockerRead("exec", container, "unsquashfs", "-o", offset, "-cat", image, "akoflow-wfa-shared/"+path) + check("sha256:"+digest(compressed) == f["contentId"], "Input content identity differs from audited dataset") + gz, e := gzip.NewReader(bytes.NewReader(compressed)) + check(e == nil, "Invalid scientific gzip input") + header := fitsHeader(gz) + check(gz.Close() == nil, "Invalid gzip input") + feature := inputFeature(header) + feature["path"] = path + feature["contentId"] = f["contentId"] + feature["compressedBytes"] = f["sizeBytes"] + feature["band"] = strings.Split(path, "_")[1] + features = append(features, feature) + } + return Object{ + "schemaVersion": 1, + "sourceImageSHA256": hashFields[0], + "basis": "immutable-input-FITS-header", + "inputs": features, + "limitations": list("Pixel count is input work, not measured CPU service demand; band and hardware may need separate calibrated coefficients."), + } +} +func supplementRoots(c, p, request, response Object, d string) Object { + check(get(c, "computeWorkBasis") == "immutable-dataset-independent-work-v1", "Independent compute units required") + check(obj(get(response, "run"))["status"] == "failed", "Expected failed training experiment") + work := Object{} + for _, v := range obj(get(p, "activityComputeWork")) { + f := obj(v) + work[str(get(f, "activityName"))] = f + } + roots := []Object{} + for _, a := range rows(get(obj(get(request, "workflow")), "activities")) { + if strings.HasPrefix(str(get(a, "name")), "mprojectid") { + roots = append(roots, a) + } + } + check(len(roots) == 12, "Need twelve Montage roots") + assignments := index(get(obj(get(request, "plan")), "assignments"), "activityId") + samples := []Object{} + for _, a := range roots { + id := str(a["id"]) + tasks, handles := []Object{}, []Object{} + for _, t := range rows(get(response, "activityAttempts")) { + if t["activityId"] == id { + tasks = append(tasks, t) + } + } + for _, h := range rows(get(response, "handles")) { + if h["activityId"] == id { + handles = append(handles, h) + } + } + check(len(tasks) == 1 && num(tasks[0]["attempt"]) == 1 && tasks[0]["status"] == "completed", "Root is not a successful single attempt") + check(len(handles) == 1 && handles[0]["status"] == "completed" && handles[0]["exitCode"] != nil && num(handles[0]["exitCode"]) == 0 && handles[0]["id"] == str(obj(response["run"])["id"])+":"+id, "Missing exact successful root handle") + check(len(arr(get(obj(get(handles[0], "artifacts")), "finalSnapshot"))) >= 2, "Missing physical root output snapshot") + resource := str(get(obj(get(assignments, id)), "resourceId")) + check(strings.Contains(resource, "bora"), "Training roots must preserve Bora resource family") + seconds := num(get(tasks[0], "runtimeSeconds")) + check(seconds > 0 && !math.IsInf(seconds, 0) && !math.IsNaN(seconds), "Invalid observed root runtime") + f := obj(get(work, str(a["name"]))) + samples = append(samples, Object{"activityFamily": get(f, "activityFamily"), "resourceId": resource, "runtimeSeconds": seconds, "secondsPerWorkUnit": seconds / num(get(f, "computeWorkUnits"))}) + } + result := clone(c) + source := str(obj(response["run"])["id"]) + ":sha256:" + d + result["computeSupplementSource"] = source + observations := arr(get(result, "computeObservations")) + for _, r := range rows(observations) { + if strings.HasPrefix(str(r["activityFamily"]), "mproject_") && r["resourceClass"] == "hpc" { + r["allowedResourceIds"] = list("plafrim-execution-ssh-node-diablo03") + } + } + for _, g := range groups(samples, "activityFamily", "resourceId") { + observations = append(observations, Object{ + "activityFamily": g.key[0], + "resourceClass": "hpc", + "allowedResourceIds": list(g.key[1]), + "source": source, + "taskRuntimeFallbackSamples": len(g.rows), + "runtimeSeconds": summary(vals(g.rows, "runtimeSeconds")), + "wallSecondsPerReferenceSecond": summary(vals(g.rows, "secondsPerWorkUnit")), + "interpretation": "Completed single-attempt root observations only; source run failed later and is not end-to-end validation.", + }) + } + result["computeObservations"] = observations + return result +} +func numericString(v any) string { return fmt.Sprintf("%v", v) } diff --git a/internal/montagetools/hpc_stages.go b/internal/montagetools/hpc_stages.go new file mode 100644 index 00000000..fc87ab2f --- /dev/null +++ b/internal/montagetools/hpc_stages.go @@ -0,0 +1,334 @@ +package montagetools + +import ( + "encoding/json" + "fmt" + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain/workflow" + "math" + "sort" + "strings" + "time" +) + +// Calibration accepts only the training reference. Holdout data never enter +// this function. Fixed placement and compute profiles remain unchanged. +func calibrateHPCStages(request, training Object, sha string) Object { + return calibrateRuntimeStages(request, training, sha, "hpc") +} + +func calibrateRuntimeStages(request, training Object, sha, targetClass string) Object { + check(targetClass != "", "target resource class required") + request = clone(request) + check(obj(training["run"])["status"] == "completed", "completed training reference required") + var typed ports.ExecutionRequest + check(json.Unmarshal(encode(request), &typed) == nil, "invalid request") + // Accept historical API envelopes with lower-case keys, but export the + // canonical offline request and force simulation mode. Never submit here. + request = Object{} + check(json.Unmarshal(encode(typed), &request) == nil, "cannot normalize request") + obj(request["Run"])["mode"] = "simulation" + training = clone(training) + alignRuntimeTraining(typed.Workflow, training) + vol := projectedRuntimeTrainingVolumes(typed.Workflow, training) + indexed := indexRuntimeTraining(training) + samples := collectRuntimeStageSamples(training, targetClass, vol, indexed) + return applyRuntimeStageCalibration(request, training, sha, targetClass, samples) +} + +func positiveLine(xs, ys []float64) (float64, float64) { + var sx, sy, sxx, sxy float64 + for i, x := range xs { + x /= 1e9 + sx += x + sy += ys[i] + sxx += x * x + sxy += x * ys[i] + } + n := float64(len(xs)) + candidates := [][2]float64{{sy / n, 0}} + if sxx > 0 { + candidates = append(candidates, [2]float64{0, sxy / sxx}) + } + if d := n*sxx - sx*sx; d > 1e-12 { + b := (n*sxy - sx*sy) / d + a := (sy - b*sx) / n + if a >= 0 && b >= 0 { + candidates = append(candidates, [2]float64{a, b}) + } + } + best := candidates[0] + score := math.Inf(1) + for _, c := range candidates { + var s float64 + for i, x := range xs { + e := c[0] + c[1]*x/1e9 - ys[i] + s += e * e + } + if s < score { + score = s + best = c + } + } + return best[0], best[1] / 1e9 +} + +func alignRuntimeTraining(version workflow.WorkflowVersion, training Object) { + // Historical catalog prefixes differ; align only exact canonical activity + // names, never family-only matches. Event/task attempt IDs remain unchanged. + names := map[string]string{} + for _, a := range version.Activities { + check(names[a.Name] == "", "duplicate canonical activity name") + names[a.Name] = a.ID + } + remap := map[string]string{} + for _, t := range rows(training["activities"]) { + id := str(t["activityId"]) + name := id[strings.LastIndex(id, "-")+1:] + mapped := names[name] + check(mapped != "", "training workflow canonical names mismatch") + remap[id] = mapped + } + for _, key := range []string{"activities", "handles", "artifactTransferRuns", "workspaces"} { + for _, r := range rows(training[key]) { + original, _ := r["activityId"].(string) + if mapped := remap[original]; mapped != "" { + r["activityId"] = mapped + } + route := obj(r["route"]) + for _, field := range []string{"consumerActivityId", "producerActivityId"} { + original, _ := route[field].(string) + if mapped := remap[original]; mapped != "" { + route[field] = mapped + } + } + } + } +} + +func projectedRuntimeTrainingVolumes(version workflow.WorkflowVersion, training Object) map[string]workflow.WorkspaceProjectionVolumes { + types := map[string]string{} + for _, t := range rows(training["activities"]) { + if t["status"] != "completed" { + continue + } + class := resourceClass(str(t["allocatedResourceId"])) + kind := "cloud_vm" + if class == "hpc" { + kind = "hpc_machine" + } + types[str(t["activityId"])] = kind + } + projected, err := workflow.ProjectWorkspaceDependenciesForResourceTypes(version, types) + check(err == nil, fmt.Sprintf("projection failed: %v", err)) + vol := map[string]workflow.WorkspaceProjectionVolumes{} + for _, a := range projected.Activities { + var v workflow.WorkspaceProjectionVolumes + b, _ := json.Marshal(a.Metadata["workspaceProjectionVolumes"]) + check(json.Unmarshal(b, &v) == nil, "invalid projected volume") + vol[a.ID] = v + } + return vol +} + +type runtimeTrainingIndex struct { + handles map[string][]Object + completed map[string]float64 + merge map[string]Object + gateway map[string]bool +} + +func indexRuntimeTraining(training Object) runtimeTrainingIndex { + handles := map[string][]Object{} + for _, h := range rows(training["handles"]) { + handles[str(h["activityId"])] = append(handles[str(h["activityId"])], h) + } + completed := map[string]float64{} + for _, e := range rows(training["events"]) { + if e["eventType"] == "activity.completed" { + at, err := time.Parse(time.RFC3339Nano, str(e["occurredAt"])) + check(err == nil, "invalid event timestamp") + completed[str(e["aggregateId"])] = float64(at.UnixNano()) / 1e9 + } + } + merge := map[string]Object{} + gateway := map[string]bool{} + for _, t := range rows(training["artifactTransferRuns"]) { + if t["status"] != "completed" { + continue + } + id := str(t["activityId"]) + if t["strategy"] == "gateway" { + gateway[id] = true + } + r := obj(t["route"]) + // These named fields measure the single consumer merge, independent + // of peer/relay transfer time. They can be reused across route types. + if r["mergeCheckSeconds"] != nil { + if merge[id] == nil || num(t["startedAt"]) > num(merge[id]["at"]) { + merge[id] = Object{"at": t["startedAt"], "route": r} + } + } + } + return runtimeTrainingIndex{handles: handles, completed: completed, merge: merge, gateway: gateway} +} + +type runtimeStageSamples struct { + checkTime, copyTime, verifyTime, totalBytes, uniqueBytes, commandTime float64 + mergeN int + prepX, prepY, pubX, pubY, overhead, admission []float64 + runtimeIDs map[string]bool +} + +func collectRuntimeStageSamples(training Object, targetClass string, vol map[string]workflow.WorkspaceProjectionVolumes, indexed runtimeTrainingIndex) runtimeStageSamples { + samples := runtimeStageSamples{prepX: []float64{}, prepY: []float64{}, pubX: []float64{}, pubY: []float64{}, overhead: []float64{}, admission: []float64{}, runtimeIDs: map[string]bool{}} + for _, t := range rows(training["activities"]) { + id := str(t["activityId"]) + hs := indexed.handles[id] + if t["status"] != "completed" || len(hs) != 1 || hs[0]["status"] != "completed" || math.Abs(num(t["startedAt"])-num(hs[0]["startedAt"])) > .001 { + continue + } + m := obj(t["metadata"]) + if resourceClass(str(t["allocatedResourceId"])) != targetClass { + continue + } + samples.runtimeIDs[str(t["runtimeId"])] = true + v := vol[id] + hm := obj(hs[0]["metadata"]) + if submitted := num(hm["submittedAt"]); submitted > 0 && num(hs[0]["startedAt"]) >= submitted { + samples.admission = append(samples.admission, num(hs[0]["startedAt"])-submitted) + } + snapshot := num(hm["remoteWorkspaceSnapshotSeconds"]) + // Initial scan is moved into preparation below, not charged twice. + samples.overhead = append(samples.overhead, num(m["launchWaitSeconds"])+math.Max(0, num(t["overheadSeconds"])-snapshot)) + if snapshot > 0 && v.InputUniqueBytes > 0 { + samples.prepX = append(samples.prepX, float64(v.InputUniqueBytes)) + samples.prepY = append(samples.prepY, snapshot) + } + if finish, ok := indexed.completed[str(t["id"])]; ok && finish >= num(t["finishedAt"]) { + samples.pubX = append(samples.pubX, float64(v.PreExecutionBytes+v.CreatedOutputBytes)) + samples.pubY = append(samples.pubY, finish-num(t["finishedAt"])) + } + if row := indexed.merge[id]; row != nil { + r := obj(row["route"]) + samples.checkTime += num(r["mergeCheckSeconds"]) + samples.copyTime += num(r["mergeCopySeconds"]) + samples.verifyTime += num(r["mergeVerifySeconds"]) + samples.commandTime += num(r["mergeCommandOverheadSeconds"]) + samples.totalBytes += float64(v.InputTotalBytes) + samples.uniqueBytes += float64(v.InputUniqueBytes) + samples.mergeN++ + // Without a separately timed scan, residual inference is permitted + // only for proven internal storage. A cloud peer/relay residual + // includes network/receiver work already modeled by its route. + if snapshot == 0 && !indexed.gateway[id] && r["sharedFilesystem"] != nil && r["sharedFilesystem"] != "" { + residual := num(m["transferElapsedSeconds"]) - num(r["mergeExecutionSeconds"]) - num(r["mergeSshChannelWaitSeconds"]) + samples.prepX = append(samples.prepX, float64(v.InputUniqueBytes)) + samples.prepY = append(samples.prepY, math.Max(0, residual)) + } + } + } + return samples +} + +func applyRuntimeAdmissionProfile(request, model Object, sha, targetClass string, samples runtimeStageSamples) { + if len(samples.admission) > 0 { + sort.Float64s(samples.admission) + var runtimeID string + for id := range samples.runtimeIDs { + runtimeID = id + } + profiles := []any{} + for _, p := range rows(model["admissionProfiles"]) { + if p["targetClass"] != targetClass || p["runtimeId"] != runtimeID { + profiles = append(profiles, p) + } + } + model["admissionProfiles"] = append(profiles, Object{ + "targetClass": targetClass, "runtimeId": runtimeID, + "source": "training-native-submit-to-start:" + sha, + "samples": len(samples.admission), "fixedSeconds": samples.admission[len(samples.admission)/2], + }) + for _, r := range rows(request["Resources"]) { + if r["schedulable"] == true && resourceClass(str(r["id"])) == targetClass { + obj(obj(model["resources"])[str(r["id"])])["executionRuntimeId"] = runtimeID + } + } + } +} + +func applyRuntimeOverhead(request Object, targetClass string, fixedOverhead float64) { + for _, r := range rows(request["Resources"]) { + if r["schedulable"] != true { + continue + } + if resourceClass(str(r["id"])) == targetClass { + r["containerOverheadSeconds"] = fixedOverhead + } + } + for _, a := range rows(obj(request["Plan"])["assignments"]) { + if resourceClass(str(a["resourceId"])) != targetClass { + continue + } + m := obj(a["metadata"]) + m["containerOverheadSeconds"] = fixedOverhead + } +} + +func applyRuntimeStageCalibration(request, training Object, sha, targetClass string, samples runtimeStageSamples) Object { + check(samples.mergeN > 0 && samples.uniqueBytes > 0 && len(samples.pubX) > 0 && len(samples.overhead) > 0 && len(samples.prepX) > 0, "insufficient single-attempt runtime evidence") + check(len(samples.runtimeIDs) == 1, "cannot mix runtime drivers in one class calibration") + prepFixed, prepRate := positiveLine(samples.prepX, samples.prepY) + pubFixed, pubRate := positiveLine(samples.pubX, samples.pubY) + sort.Float64s(samples.overhead) + fixedOverhead := samples.overhead[len(samples.overhead)/2] + model := obj(obj(obj(request["NetworkTopology"])["metadata"])["workspaceMaterializationModel"]) + model["schemaVersion"] = 2 + applyRuntimeAdmissionProfile(request, model, sha, targetClass, samples) + replace := func(key string, profile Object) { + p := []any{} + for _, old := range rows(model[key]) { + if old["targetClass"] != targetClass { + p = append(p, old) + } + } + model[key] = append(p, profile) + } + source := "runtime-stage-training:" + sha + replace("mergeProfiles", Object{ + "targetClass": targetClass, + "source": source, + "samples": samples.mergeN, + "commandOverheadSeconds": samples.commandTime / float64(samples.mergeN), + "checkSecondsPerInputByte": samples.checkTime / samples.totalBytes, + "copySecondsPerUniqueByte": samples.copyTime / samples.uniqueBytes, + "verifySecondsPerUniqueByte": samples.verifyTime / samples.uniqueBytes, + }) + replace("inputObservationProfiles", Object{"targetClass": targetClass, "source": source + ":effective-preparation-residual-not-queue-free", "samples": len(samples.prepX), "fixedSeconds": prepFixed, "secondsPerUniqueByte": prepRate}) + replace("publicationProfiles", Object{ + "targetClass": targetClass, "source": source + ":publication-and-poll-wall", + "samples": len(samples.pubX), "fixedSeconds": pubFixed, + "scanSecondsPerWorkspaceByte": pubRate, "cleanupSecondsPerInputByte": 0, + }) + applyRuntimeOverhead(request, targetClass, fixedOverhead) + obj(obj(request["NetworkTopology"])["metadata"])["runtimeStageCalibration"] = Object{ + "issue": "AKO-166", + "targetClass": targetClass, + "runtimeIds": samples.runtimeIDs, + "trainingRunId": obj(training["run"])["id"], + "trainingSHA256": sha, + "mergeSamples": samples.mergeN, + "preparationResidualSamples": len(samples.prepX), + "publicationSamples": len(samples.pubX), + "fixedLaunchAndRuntimeOverheadSeconds": fixedOverhead, + "holdoutUsed": false, + "limitations": list( + "Recovered/multi-handle training activities excluded.", + "Preparation scans/residuals and publication contain unidentified waiting/poll delay; exploratory effective-wall fit, not queue-free service.", + "Scheduler wait and gateway fallback not injected into the internal-route baseline; evaluate residuals separately.", + "Fixed overhead median does not capture family-specific wrapper cost.", + "Merge is fitted from component timings for every route. Residual inference requires proven same-storage; cloud uses direct scan telemetry and excludes network residual. Coefficients are not borrowed across runtimes.", + ), + } + return request +} diff --git a/internal/montagetools/hpc_stages_test.go b/internal/montagetools/hpc_stages_test.go new file mode 100644 index 00000000..87ff614a --- /dev/null +++ b/internal/montagetools/hpc_stages_test.go @@ -0,0 +1,15 @@ +package montagetools + +import ( + "github.com/stretchr/testify/require" + "testing" +) + +func TestPositiveStageLineIsVolumeBasedNotMakespanMultiplier(t *testing.T) { + a, b := positiveLine([]float64{0, 1e9, 2e9}, []float64{5, 15, 25}) + require.InDelta(t, 5, a, 1e-9) + require.InDelta(t, 1e-8, b, 1e-15) + a, b = positiveLine([]float64{0, 1e9, 2e9}, []float64{10, 4, 1}) + require.GreaterOrEqual(t, a, 0.0) + require.GreaterOrEqual(t, b, 0.0) +} diff --git a/internal/montagetools/json.go b/internal/montagetools/json.go new file mode 100644 index 00000000..4eccdf02 --- /dev/null +++ b/internal/montagetools/json.go @@ -0,0 +1,236 @@ +// Package montagetools contains scientific experiment tooling, not runtime +// execution. JSON contracts and allowlists are kept separate from raw API data. +package montagetools + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "os" + "reflect" + "regexp" + "sort" + "strconv" + "strings" +) + +type Object = map[string]any + +func obj(v any) Object { + if v == nil { + return Object{} + } + m, ok := v.(map[string]any) + check(ok, "object required") + return m +} +func arr(v any) []any { + if v == nil { + return []any{} + } + a, ok := v.([]any) + check(ok, "array required") + return a +} +func get(m Object, k string) any { v, ok := m[k]; check(ok, "missing field "+k); return v } +func str(v any) string { s, ok := v.(string); check(ok, "string required"); return s } +func num(v any) float64 { + if v == nil { + return 0 + } + switch x := v.(type) { + case float64: + return x + case int: + return float64(x) + case int64: + return float64(x) + case json.Number: + f, e := x.Float64() + check(e == nil, "invalid number") + return f + case string: + f, e := strconv.ParseFloat(x, 64) + check(e == nil, "invalid number") + return f + } + panic("number required") +} +func check(ok bool, message string) { + if !ok { + panic(message) + } +} +func eq(a, b any) bool { return reflect.DeepEqual(normal(a), normal(b)) } +func normal(v any) any { + p, e := json.Marshal(v) + check(e == nil, "invalid JSON") + var out any + check(json.Unmarshal(p, &out) == nil, "invalid JSON") + return out +} +func clone(m Object) Object { return obj(normal(m)) } +func sortedKeys(m Object) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} +func list(values ...string) []any { + a := []any{} + for _, v := range values { + a = append(a, v) + } + return a +} +func pick(m Object, keys ...string) Object { + o := Object{} + for _, k := range keys { + if v, ok := m[k]; ok { + o[k] = v + } + } + return o +} +func rows(v any) []Object { + out := []Object{} + for _, r := range arr(v) { + out = append(out, obj(r)) + } + return out +} +func sortRows(rs []Object, key string) { + sort.SliceStable(rs, func(i, j int) bool { return str(rs[i][key]) < str(rs[j][key]) }) +} +func index(v any, key string) Object { + out := Object{} + for _, r := range rows(v) { + out[str(get(r, key))] = r + } + return out +} + +type group struct { + key []string + rows []Object +} + +func groups(v []Object, keys ...string) []group { + out := []group{} + positions := map[string]int{} + for _, r := range v { + parts := []string{} + for _, k := range keys { + parts = append(parts, str(get(r, k))) + } + key := strings.Join(parts, "\x00") + i, ok := positions[key] + if !ok { + i = len(out) + positions[key] = i + out = append(out, group{key: parts}) + } + out[i].rows = append(out[i].rows, r) + } + return out +} +func vals(rs []Object, key string) []any { + out := []any{} + for _, r := range rs { + out = append(out, r[key]) + } + return out +} +func sum(rs []Object, key string) float64 { + v := 0.0 + for _, r := range rs { + v += num(r[key]) + } + return v +} +func digest(p []byte) string { h := sha256.Sum256(p); return hex.EncodeToString(h[:]) } +func encode(v any) []byte { + p, e := json.MarshalIndent(v, "", " ") + check(e == nil, "encode JSON: "+fmt.Sprint(e)) + return append(p, '\n') +} +func read(path string) (Object, []byte) { + p, e := os.ReadFile(path) + check(e == nil, "read input: "+fmt.Sprint(e)) + d := json.NewDecoder(bytes.NewReader(p)) + d.UseNumber() + var m Object + check(d.Decode(&m) == nil, "invalid input JSON") + return m, p +} +func write(path string, v any) { + check(os.WriteFile(path, encode(v), 0600) == nil, "write output failed") +} +func resourceClass(id string) string { + switch { + case strings.Contains(id, "fog-e2-micro"): + return "e2-micro" + case strings.Contains(id, "e2-medium"): + return "e2-medium" + case strings.Contains(id, "bora") || strings.Contains(id, "diablo"): + return "hpc" + } + panic("Unclassified resource " + id) +} + +var familyPattern = regexp.MustCompile(`-([a-z]+)id\d+$`) + +func family(id string) string { + m := familyPattern.FindStringSubmatch(id) + check(len(m) == 2, "Unclassified activity "+id) + return m[1] +} +func summary(values []any) any { + vs := []float64{} + for _, v := range values { + if v == nil { + continue + } + f := num(v) + check(!math.IsNaN(f) && !math.IsInf(f, 0) && f >= 0, "Invalid non-finite or negative measurement") + vs = append(vs, f) + } + if len(vs) == 0 { + return nil + } + sort.Float64s(vs) + n := len(vs) + median := vs[n/2] + if n%2 == 0 { + median = (vs[n/2-1] + median) / 2 + } + return Object{"samples": n, "median": median, "minimum": vs[0], "maximum": vs[n-1]} +} +func pooledRate(rs []Object, key string) any { + bytes, seconds := 0.0, 0.0 + n := 0 + for _, r := range rs { + if num(r[key]) > 0 && num(r["seconds"]) > 0 { + bytes += num(r[key]) + seconds += num(r["seconds"]) + n++ + } + } + if n == 0 { + return nil + } + return Object{"samples": n, "bytes": bytes, "serviceSeconds": seconds, "bytesPerSecond": bytes / seconds, "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time"} +} +func attempt(fn func() Object) (result Object, err error) { + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("%v", p) + } + }() + return fn(), nil +} diff --git a/internal/montagetools/models.go b/internal/montagetools/models.go new file mode 100644 index 00000000..efac377b --- /dev/null +++ b/internal/montagetools/models.go @@ -0,0 +1,155 @@ +package montagetools + +import ( + "math" + "strings" +) + +func median(row Object, key string, positive bool) float64 { + s := obj(get(row, key)) + check(num(s["samples"]) > 0, "Missing measured "+key) + v := num(get(s, "median")) + check(!math.IsInf(v, 0) && !math.IsNaN(v) && v >= 0 && (!positive || v > 0), "Invalid measured "+key) + return v +} +func serviceRate(row Object, pooled, legacy string) float64 { + if row[pooled] == nil { + return median(row, legacy, true) + } + r := obj(row[pooled]) + v := num(get(r, "bytesPerSecond")) + check(num(r["samples"]) > 0 && v > 0 && !math.IsInf(v, 0) && !math.IsNaN(v), "Invalid pooled measured service rate") + return v +} +func freezeMaterialization(c Object, d string, resources Object, relay string) Object { + check(get(c, "workspaceVolumeBasis") == "producer-final-snapshot-manifest", "Manifest-based workspace volumes required") + check(get(obj(get(resources, relay)), "domain") == "relay", "Explicit relay resource required") + source := str(get(c, "baselineRunId")) + ":sha256:" + d + profiles := []any{} + for _, g := range groups(rows(get(c, "materializationPhaseObservations")), "strategy", "sourceClass", "targetClass") { + strategy, from, to := g.key[0], g.key[1], g.key[2] + check(strategy == "gateway" || strategy == "direct-runtime", "Unsupported measured strategy") + phases := Object{} + for _, p := range g.rows { + phases[str(p["phase"])] = p + } + uploadName, receiverName := "peer-private-upload", "peer-receiver-verify-and-extract" + if strategy == "gateway" { + uploadName, receiverName = "archive-upload", "receiver-verify-and-extract" + } + upload := obj(get(phases, uploadName)) + targetPool, sourcePool := false, false + for _, id := range sortedKeys(resources) { + r := obj(resources[id]) + if r["bulkServiceGroupId"] != nil && r["bulkServiceGroupId"] != "" { + targetPool = targetPool || r["class"] == to + sourcePool = sourcePool || r["class"] == from + } + } + if targetPool { + check(eq(obj(get(phases, receiverName))["secondsBases"], list("phase-command-execution-excludes-channel-queue")), "Receiver "+from+"->"+to+" requires queue-free command service before adding a bulk pool") + } + p := Object{ + "strategy": strategy, + "sourceClass": from, + "targetClass": to, + "source": source, + "samples": get(obj(get(upload, "seconds")), "samples"), + "archiveToWorkspaceRatio": median(upload, "archiveToWorkspaceRatio", true), + "uploadArchiveBytesPerSecond": serviceRate(upload, "pooledArchiveServiceRate", "effectiveArchiveBytesPerSecond"), + } + if strategy == "gateway" { + pack := obj(get(phases, "pack-and-receive")) + if sourcePool { + check(eq(pack["secondsBases"], list("source-command-execution-excludes-channel-queue")), "Gateway "+from+"->"+to+" requires queue-free source service measurements before adding a bulk pool") + check(pack["sourcePreflightServiceSeconds"] != nil, "Gateway "+from+"->"+to+" requires separately measured preflight service") + } + p["ingressArchiveBytesPerSecond"] = serviceRate(pack, "pooledArchiveServiceRate", "effectiveArchiveBytesPerSecond") + if pack["sourcePreflightServiceSeconds"] != nil { + p["probeSeconds"] = median(pack, "sourcePreflightServiceSeconds", false) + } + p["ingressIncludesSourcePreparation"] = true + p["validationSecondsPerByte"] = 1 / serviceRate(obj(get(phases, "archive-validation")), "pooledWorkspaceServiceRate", "effectiveWorkspaceBytesPerSecond") + } else { + p["probeSeconds"] = median(obj(get(phases, "peer-authenticated-connectivity-probe")), "seconds", false) + p["snapshotSecondsPerByte"] = 1 / serviceRate(obj(get(phases, "peer-snapshot-acquire")), "pooledWorkspaceServiceRate", "effectiveWorkspaceBytesPerSecond") + } + p["receiverSecondsPerByte"] = 1 / serviceRate(obj(get(phases, receiverName)), "pooledWorkspaceServiceRate", "effectiveWorkspaceBytesPerSecond") + profiles = append(profiles, p) + } + merges := []any{} + for _, r := range rows(get(c, "mergeRateObservations")) { + rates := obj(get(r, "observations")) + p := Object{"targetClass": get(r, "targetClass"), "source": source, "samples": get(obj(get(rates, "checkSecondsPerInputByte")), "samples")} + for _, k := range sortedKeys(rates) { + p[k] = median(rates, k, false) + } + if r["commandOverheadSeconds"] != nil { + p["commandOverheadSeconds"] = median(r, "commandOverheadSeconds", false) + } + merges = append(merges, p) + } + return Object{"schemaVersion": 1, "source": source, "relayResourceId": relay, "resources": resources, "profiles": profiles, "mergeProfiles": merges} +} +func freezeCompute(c Object, d string, resources Object, namespace string) Object { + independent := c["computeWorkBasis"] == "immutable-dataset-independent-work-v1" + source := str(get(c, "baselineRunId")) + ":sha256:" + d + families := Object{} + for _, r := range rows(get(c, "computeObservations")) { + families[str(get(r, "activityFamily"))] = true + } + typeIDs := Object{} + for _, f := range sortedKeys(families) { + typeIDs[f] = namespace + "-" + strings.ReplaceAll(f, "_", "-") + } + profiles := []any{} + for _, r := range rows(c["computeObservations"]) { + coefficient := median(r, "wallSecondsPerReferenceSecond", true) + stats := obj(r["wallSecondsPerReferenceSecond"]) + for _, id := range sortedKeys(resources) { + resource := obj(resources[id]) + if resource["class"] != r["resourceClass"] { + continue + } + if r["allowedResourceIds"] != nil { + allowed := false + for _, v := range arr(r["allowedResourceIds"]) { + allowed = allowed || v == id + } + if !allowed { + continue + } + } + modelVersion, units := "reference-work-v1", "wall-seconds-per-reference-second" + if independent { + modelVersion, units = "compute-work-v1", "wall-seconds-per-independent-work-unit" + } + s := r["source"] + if s == nil { + s = source + } + f := str(r["activityFamily"]) + profiles = append(profiles, Object{ + "id": namespace + "-" + f + "-" + id, + "activityTypeId": typeIDs[f], + "resourceId": id, + "runtimeSeconds": coefficient, + "modelVersion": modelVersion, + "source": s, + "sampleSize": get(stats, "samples"), + "metadata": Object{"units": units, "observedRange": []any{stats["minimum"], stats["maximum"]}, "includesPhysicalSharing": true, "taskRuntimeFallbackSamples": get(r, "taskRuntimeFallbackSamples")}, + }) + } + } + return Object{ + "schemaVersion": 1, + "source": source, + "activityFamilyTypeIds": typeIDs, + "activityProfiles": profiles, + "limitations": list( + "Class-level historical wall rates; no per-activity replay.", + "Unobserved family/class pairs have no measured profile.", + "Includes physical sharing: additional interference factors require separate evidence.", + ), + } +} diff --git a/internal/montagetools/native_case.go b/internal/montagetools/native_case.go new file mode 100644 index 00000000..a978b1d8 --- /dev/null +++ b/internal/montagetools/native_case.go @@ -0,0 +1,66 @@ +package montagetools + +import ( + "context" + "encoding/json" + "fmt" + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/provider/simgrid" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +type dockerNativeCommand struct{ root, image string } + +func (d dockerNativeCommand) Run(ctx context.Context, binary string, args []string, _ []byte) ([]byte, error) { + check(strings.HasPrefix(binary, "/case/") || binary == "/build/output/akoflow-simgrid-runner", "native binary must be a case runner or known legacy runner") + mapped := append([]string(nil), args...) + for i, arg := range mapped { + if filepath.IsAbs(arg) { + rel, err := filepath.Rel(d.root, arg) + if err != nil || rel == ".." || strings.HasPrefix(rel, "../") { + return nil, fmt.Errorf("native case path outside repository") + } + mapped[i] = "/case/" + filepath.ToSlash(rel) + } + } + argv := []string{"run", "--rm", "-v", d.root + ":/case", d.image, binary} + argv = append(argv, mapped...) + output, err := exec.CommandContext(ctx, "docker", argv...).CombinedOutput() + if err != nil { + for _, line := range strings.Split(string(output), "\n") { + if strings.HasPrefix(line, "akoflow-simgrid-runner:") { + return output, fmt.Errorf("%w: %s", err, line) + } + } + } + return output, err +} + +// Exercise the production executor capability/trace checks, not only the C++ +// binary. Mode is checked before any container runs. No daemon or real runtime. +func verifyNativeCase(requestPath, image, binary, out string) { + raw, err := os.ReadFile(requestPath) + check(err == nil, "native case request unavailable") + var request ports.ExecutionRequest + check(json.Unmarshal(raw, &request) == nil, "invalid native case request") + check(string(request.Run.Mode) == "simulation", "verification is simulation only") + root, err := os.Getwd() + check(err == nil, "repository path unavailable") + output, err := filepath.Abs(out) + check(err == nil, "invalid verification output") + _, err = os.Stat(output) + check(os.IsNotExist(err), "verification output must be new") + executor, err := simgrid.NewProcessExecutor(dockerNativeCommand{root: root, image: image}, simgrid.ProcessConfig{BinaryPath: binary, Workspace: output, Timeout: 10 * time.Minute, ReferenceFLOPS: 1e9}) + check(err == nil, "cannot create native verification executor") + trace, err := executor.Execute(context.Background(), request) + check(err == nil, fmt.Sprintf("native executor conformance: %v", err)) + payload, err := json.Marshal(trace) + check(err == nil, "cannot encode native verification trace") + var result Object + check(json.Unmarshal(payload, &result) == nil, "cannot decode verification trace") + write(filepath.Join(output, "verified-result.json"), result) +} diff --git a/internal/montagetools/observation.go b/internal/montagetools/observation.go new file mode 100644 index 00000000..ec4f0983 --- /dev/null +++ b/internal/montagetools/observation.go @@ -0,0 +1,46 @@ +package montagetools + +import "encoding/json" + +// This is an exploratory wall-time sensitivity, never a queue-free fit or +// independent validation. It establishes how much a missing input scan changes +// the forecast without scaling the workflow makespan globally. +func observationSensitivity(request, real Object, evidenceDigest string) Object { + check(obj(get(real, "run"))["status"] == "completed", "completed observation reference required") + var seconds, bytes float64 + samples := 0 + for _, h := range rows(get(real, "handles")) { + metadata := obj(h["metadata"]) + if h["status"] != "completed" || metadata["remoteWorkspaceSnapshotSeconds"] == nil { + continue + } + var files []Object + check(json.Unmarshal([]byte(str(get(metadata, "artifactObservationBefore"))), &files) == nil, "invalid observation manifest") + var inputBytes float64 + for _, file := range files { + inputBytes += num(get(file, "size")) + } + if inputBytes == 0 { + continue + } + seconds += num(get(metadata, "remoteWorkspaceSnapshotSeconds")) + bytes += inputBytes + samples++ + } + check(samples > 0 && bytes > 0, "no input observation samples") + network := obj(get(request, "NetworkTopology")) + metadata := obj(get(network, "metadata")) + model := obj(get(metadata, "workspaceMaterializationModel")) + model["inputObservationProfiles"] = []any{Object{"targetClass": "e2-micro", "source": "exploratory-observed-wall:" + evidenceDigest, "samples": samples, "secondsPerUniqueByte": seconds / bytes}} + metadata["inputObservationSensitivity"] = Object{ + "referenceSHA256": evidenceDigest, + "role": "exploratory-wall-time-sensitivity-not-service-calibration", + "samples": samples, + "limitations": list( + "Includes unknown SSH wait and command overhead; not an accepted queue-free calibration.", + "Recovered reference is not independent validation.", + "Terminal observation and pruning are not represented by this input-only sensitivity.", + ), + } + return request +} diff --git a/internal/montagetools/observation_test.go b/internal/montagetools/observation_test.go new file mode 100644 index 00000000..d5f39a64 --- /dev/null +++ b/internal/montagetools/observation_test.go @@ -0,0 +1,15 @@ +package montagetools + +import "testing" + +func TestObservationSensitivityExplicitlyPreservesNonValidationRole(t *testing.T) { + model := Object{} + metadata := Object{"workspaceMaterializationModel": model} + request := Object{"NetworkTopology": Object{"metadata": metadata}} + real := Object{"run": Object{"status": "completed"}, "handles": []any{Object{"status": "completed", "metadata": Object{"remoteWorkspaceSnapshotSeconds": 2.0, "artifactObservationBefore": `[{"size":100}]`}}}} + observationSensitivity(request, real, "digest") + profile := rows(model["inputObservationProfiles"])[0] + if profile["targetClass"] != "e2-micro" || profile["secondsPerUniqueByte"] != .02 || obj(metadata["inputObservationSensitivity"])["role"] != "exploratory-wall-time-sensitivity-not-service-calibration" { + t.Fatal("lost coefficient or provenance role") + } +} diff --git a/internal/montagetools/prepare.go b/internal/montagetools/prepare.go new file mode 100644 index 00000000..cb739459 --- /dev/null +++ b/internal/montagetools/prepare.go @@ -0,0 +1,174 @@ +package montagetools + +import ( + "os" + "path/filepath" + "strings" +) + +func envDefault(key, fallback string) string { + v, ok := os.LookupEnv(key) + if !ok { + return fallback + } + return v +} +func prepareRequests(args []string) { + check(len(args) == 9, "prepare_montage_measured_requests BASELINE PROJECTION COMPUTE MATERIALIZATION GCP HPC SCOPE NETWORK OUTPUT_DIR") + inputs := []Object{} + for _, p := range args[:8] { + m, _ := read(p) + inputs = append(inputs, m) + } + baseline, p, compute, materialization, gcp, scope, network := inputs[0], inputs[1], inputs[2], inputs[3], inputs[4], inputs[6], inputs[7] + _ = inputs[5] + out := args[8] + check(os.MkdirAll(out, 0700) == nil, "create output directory failed") + save := func(name string, value any) { write(filepath.Join(out, name+".json"), value) } + workflowName := envDefault("MONTAGE_WORKFLOW_NAME", "montage-reference-work-v1") + revision := envDefault("MONTAGE_MODEL_REVISION", "v1") + workflow := obj(get(baseline, "workflow")) + sourceActivities := rows(get(workflow, "activities")) + check(len(sourceActivities) == 58, "Expected complete Montage58") + activities := measuredWorkflowImportActivities(sourceActivities, workflow, p, compute, workflowName) + save("workflow-import", Object{"name": workflowName, "spec": Object{"namespace": "montage-measured-20261011", "activities": activities}}) + relay := Object{ + "id": get(materialization, "relayResourceId"), + "environmentVersionId": get(obj(get(gcp, "version")), "id"), + "name": "Existing Akoflow relay controller (model-only, not schedulable)", + "type": "local_machine", + "cpuCores": 8, + "cpuCapacity": 8, + "memoryBytes": int64(8) * 1024 * 1024 * 1024, + "computeSpeedup": 1, + "schedulable": false, + "metadata": Object{"modelOnly": true, "role": "transfer-relay", "provision": false}, + } + save("relay-resource", relay) + prepareMeasuredNetwork(network, materialization, relay, revision) + scope["id"] = "montage-reference-scope-20261011-" + revision + scope["networkTopologyId"] = network["id"] + network["executionScopeId"] = scope["id"] + save("scope", scope) + save("network", network) + units := "reference-work-v1" + if p["activityComputeWork"] != nil { + units = "compute-work-v1" + } + save("planning-session", Object{ + "id": envDefault("MONTAGE_PLANNING_ID", "montage-reference-planning-20261011-v2"), + "workflowVersionId": workflowName + "-v1", + "executionScopeId": scope["id"], + "networkTopologyId": network["id"], + "algorithms": []any{Object{"id": "prism-time", "configuration": Object{"requireMeasuredProfiles": true}}}, + "configuration": Object{ + "calibrationSource": compute["source"], + "observeWorkspaceStorageBudgets": true, + "activityProfileSnapshot": compute["activityProfiles"], + "requireMeasuredProfiles": true, + "computeCalibrationUnits": units, + "computeCalibrationIncludesPhysicalSharing": true, + "additionalPairwiseSlowdown": "not-applied-to-contention-conditioned-wall-profiles", + }, + }) +} + +func measuredWorkflowImportActivities(sourceActivities []Object, workflow, p, compute Object, workflowName string) []any { + names := Object{} + for _, a := range sourceActivities { + names[str(a["id"])] = get(a, "name") + } + parents := Object{} + for _, d := range rows(get(workflow, "dependencies")) { + id := str(d["activityId"]) + parents[id] = append(arr(parents[id]), d) + } + activities := []any{} + for _, a := range sourceActivities { + id := str(a["id"]) + f := family(id) + var work Object + if p["activityComputeWork"] != nil { + work = obj(get(obj(p["activityComputeWork"]), id)) + f = str(get(work, "activityFamily")) + } + check(get(obj(get(compute, "activityFamilyTypeIds")), f) == workflowName+"-"+strings.ReplaceAll(f, "_", "-"), "Unclassified family "+id) + command := clone(obj(get(a, "command"))) + delete(command, "resolvedExecutable") + metadata := clone(obj(a["metadata"])) + metadata["workspaceProjection"] = get(obj(get(p, "activityModels")), id) + metadata["calibrationSource"] = get(compute, "source") + delete(metadata, "workspaceProjectionVolumes") + if work != nil { + metadata["computeWorkUnits"] = get(work, "computeWorkUnits") + metadata["computeWorkUnit"] = get(work, "unit") + metadata["computeWorkSource"] = get(work, "sourceImageSHA256") + } + depends := []any{} + for _, d := range rows(parents[id]) { + depends = append(depends, get(names, str(d["dependsOnActivityId"]))) + } + seed, runtime, selector := metadata["workspaceSeedPath"], metadata["runtime"], metadata["resourceSelector"] + if seed == nil { + seed = "" + } + if runtime == nil { + runtime = "" + } + if selector == nil { + selector = "" + } + resources := obj(get(a, "resources")) + activities = append(activities, Object{ + "name": get(a, "name"), + "activityType": f, + "metadata": metadata, + "command": command, + "simulation": get(a, "simulation"), + "cpuLimit": numericString(get(resources, "cpu")), + "memoryLimit": numericString(get(resources, "memoryBytes")), + "dependsOn": depends, + "workspaceSeedPath": seed, + "expectedOutputs": metadata["expectedOutputs"], + "runtime": runtime, + "resourceSelector": selector, + }) + } + return activities +} + +func prepareMeasuredNetwork(network, materialization, relay Object, revision string) { + network["id"] = "montage-reference-network-20261011-" + revision + network["name"] = "Measured workspace materialization with explicit controller relay" + meta := obj(network["metadata"]) + meta["workspaceMaterializationModel"] = materialization + meta["calibrationSource"] = get(materialization, "source") + meta["linkBandwidthBasis"] = "Inherited topology assumptions; phase flow caps use measured historical effective rates" + network["metadata"] = meta + links := arr(get(network, "links")) + for _, l := range rows(links) { + m := obj(l["metadata"]) + m["sourceTopologyLinkId"] = get(l, "id") + l["metadata"] = m + l["id"] = str(network["id"]) + "-" + str(l["id"]) + l["topologyId"] = network["id"] + } + for _, id := range sortedKeys(obj(get(materialization, "resources"))) { + if id == relay["id"] { + continue + } + links = append(links, Object{ + "id": str(network["id"]) + "-relay-" + id, + "topologyId": network["id"], + "sourceResourceId": relay["id"], + "targetResourceId": id, + "bandwidthBitsPerSecond": 200000000, + "latencySeconds": 0.05, + "pricePerByte": 0, + "bidirectional": true, + "sharingPolicy": "SHARED", + "metadata": Object{"measurementStatus": "assumed-inherited-200Mbps", "networkCostCalibrated": false}, + }) + } + network["links"] = links +} diff --git a/internal/montagetools/publication.go b/internal/montagetools/publication.go new file mode 100644 index 00000000..0e76a688 --- /dev/null +++ b/internal/montagetools/publication.go @@ -0,0 +1,152 @@ +package montagetools + +import ( + "math" + "time" +) + +func publicationReference(real Object, referenceDigest string) Object { + check(obj(get(real, "run"))["status"] == "completed", "completed reference required") + completed := map[string]float64{} + for _, event := range rows(get(real, "events")) { + if event["eventType"] != "activity.completed" { + continue + } + timestamp, err := time.Parse(time.RFC3339Nano, str(get(event, "occurredAt"))) + check(err == nil, "invalid completion timestamp") + completed[str(get(event, "aggregateId"))] = float64(timestamp.UnixNano()) / 1e9 + } + latest := map[string]Object{} + for _, handle := range rows(get(real, "handles")) { + id := str(get(handle, "activityId")) + if latest[id] == nil || num(handle["startedAt"]) > num(latest[id]["startedAt"]) { + latest[id] = handle + } + } + observations := []any{} + cleanup := map[string]float64{} + for _, workspace := range rows(real["workspaces"]) { + // Released intermediate workspaces are reclaimed later, after their + // consumers; that is not publication cleanup of the producer. + if workspace["isFinal"] == true && workspace["retention"] == "final" { + cleanup[str(get(workspace, "activityId"))] = num(workspace["reclaimedBytes"]) + } + } + var totalSeconds, totalBytes float64 + for _, task := range rows(get(real, "activities")) { + handle := latest[str(get(task, "activityId"))] + if task["status"] != "completed" || handle == nil || handle["status"] != "completed" { + continue + } + metadata := obj(handle["metadata"]) + if metadata["executionTarget"] != "remote-docker" { + continue + } + check(math.Abs(num(task["startedAt"])-num(handle["startedAt"])) < .001, "latest task/handle mismatch") + at, ok := completed[str(get(task, "id"))] + check(ok, "completed publication event missing") + observed := num(get(metadata, "terminalObservedAt")) + check(at >= observed, "publication precedes terminal observation") + var scanBytes float64 + for _, file := range rows(get(obj(get(handle, "artifacts")), "finalSnapshot")) { + scanBytes += num(get(file, "sizeBytes")) + } + check(scanBytes > 0, "terminal snapshot volume missing") + wall := at - observed + totalSeconds += wall + totalBytes += scanBytes + observations = append(observations, Object{ + "activityId": task["activityId"], + "targetClass": resourceClass(str(get(handle, "resourceId"))), + "terminalScanBytes": scanBytes, + "publicationCleanupBytes": cleanup[str(task["activityId"])], + "publicationWallSeconds": wall, + "pollDelayExcludedSeconds": observed - num(get(handle, "finishedAt")), + }) + } + check(len(observations) > 0, "no publication observations") + return Object{ + "runId": obj(real["run"])["id"], + "referenceSHA256": referenceDigest, + "role": "publication-wall-reference-not-queue-free-service-fit", + "samples": len(observations), + "weightedWallSecondsPerScanByte": totalSeconds / totalBytes, + "summedWorkSecondsNotMakespan": totalSeconds, + "observations": observations, + "limitations": list( + "Includes observation SSH transport, sealing, pruning and scheduling between terminal observation and completion event.", + "Excludes initial polling delay but does not prove zero admission wait.", + "Recovered reference is not independent validation; no global makespan multiplier is applied.", + ), + } +} + +func combinedObservationSensitivity(request, real Object, digest string) Object { + request = observationSensitivity(request, real, digest) + reference := publicationReference(real, digest) + volumes := map[string]float64{} + cleanup := map[string]float64{} + for _, row := range rows(reference["observations"]) { + volumes[str(get(row, "activityId"))] = num(get(row, "terminalScanBytes")) + cleanup[str(get(row, "activityId"))] = num(row["publicationCleanupBytes"]) + } + for _, activity := range rows(get(obj(get(request, "Workflow")), "activities")) { + bytes, ok := volumes[str(get(activity, "id"))] + if !ok { + continue + } + projection := obj(get(obj(get(activity, "metadata")), "workspaceProjection")) + projection["terminalObservationBytes"] = int64(bytes) + projection["publicationCleanupBytes"] = int64(cleanup[str(activity["id"])]) + projection["terminalObservationSource"] = "observed-pre-prune-snapshot:" + digest + } + metadata := obj(get(obj(get(request, "NetworkTopology")), "metadata")) + model := obj(get(metadata, "workspaceMaterializationModel")) + scan, prune := fitPublicationWall(rows(reference["observations"])) + model["publicationProfiles"] = []any{Object{"targetClass": "e2-micro", "source": "combined-observed-publication-wall:" + digest, "samples": reference["samples"], "scanSecondsPerWorkspaceByte": scan, "cleanupSecondsPerInputByte": prune}} + metadata["publicationObservationSensitivity"] = reference + return request +} + +// Two-feature nonnegative least squares; no total-makespan scaling. Features +// are normalized to GB for numeric stability. The fitted wall coefficients +// remain exploratory until transport/admission wait is independently measured. +func fitPublicationWall(rows []Object) (float64, float64) { + var xx, xz, zz, xy, zy float64 + for _, row := range rows { + x := num(row["terminalScanBytes"]) / 1e9 + z := num(row["publicationCleanupBytes"]) / 1e9 + y := num(row["publicationWallSeconds"]) + xx += x * x + xz += x * z + zz += z * z + xy += x * y + zy += z * y + } + candidates := [][2]float64{{0, 0}} + if xx > 0 { + candidates = append(candidates, [2]float64{xy / xx, 0}) + } + if zz > 0 { + candidates = append(candidates, [2]float64{0, zy / zz}) + } + if determinant := xx*zz - xz*xz; determinant > 1e-12 { + a, b := (xy*zz-zy*xz)/determinant, (zy*xx-xy*xz)/determinant + if a >= 0 && b >= 0 { + candidates = append(candidates, [2]float64{a, b}) + } + } + best, errorSquared := candidates[0], math.Inf(1) + for _, candidate := range candidates { + var residual float64 + for _, row := range rows { + predicted := candidate[0]*num(row["terminalScanBytes"])/1e9 + candidate[1]*num(row["publicationCleanupBytes"])/1e9 + delta := predicted - num(row["publicationWallSeconds"]) + residual += delta * delta + } + if residual < errorSquared { + best, errorSquared = candidate, residual + } + } + return best[0] / 1e9, best[1] / 1e9 +} diff --git a/internal/montagetools/publication_test.go b/internal/montagetools/publication_test.go new file mode 100644 index 00000000..84b14077 --- /dev/null +++ b/internal/montagetools/publication_test.go @@ -0,0 +1,57 @@ +package montagetools + +import ( + "math" + "testing" +) + +func TestPublicationFitSeparatesScanAndCleanupWithoutNegativeRates(t *testing.T) { + scan, cleanup := fitPublicationWall([]Object{ + {"terminalScanBytes": 1e9, "publicationCleanupBytes": 0.0, "publicationWallSeconds": 2.0}, + {"terminalScanBytes": 1e9, "publicationCleanupBytes": 1e9, "publicationWallSeconds": 5.0}, + {"terminalScanBytes": 2e9, "publicationCleanupBytes": 1e9, "publicationWallSeconds": 7.0}, + }) + if scan != 2e-9 || cleanup != 3e-9 { + t.Fatalf("fit %g %g", scan, cleanup) + } +} + +func TestPublicationReferenceExcludesPollingAndRejectsWrongAttempt(t *testing.T) { + handle := Object{ + "activityId": "a", + "status": "completed", + "resourceId": "montage-gcp-fog-e2-micro-01", + "startedAt": 90.0, + "finishedAt": 98.0, + "metadata": Object{"executionTarget": "remote-docker", "terminalObservedAt": 100.0}, + "artifacts": Object{"finalSnapshot": []any{Object{"sizeBytes": 100.0}}}, + } + task := Object{"id": "task-a", "activityId": "a", "status": "completed", "startedAt": 90.0} + real := Object{ + "run": Object{"id": "reference", "status": "completed"}, + "handles": []any{handle}, + "activities": []any{task}, + "events": []any{Object{"aggregateId": "task-a", "eventType": "activity.completed", "occurredAt": "1970-01-01T00:01:45Z"}}, + } + report := publicationReference(real, "digest") + row := rows(report["observations"])[0] + if row["publicationWallSeconds"] != 5.0 || row["pollDelayExcludedSeconds"] != 2.0 { + t.Fatal("poll wait counted as publication") + } + obj(handle["metadata"])["remoteWorkspaceSnapshotSeconds"] = 2.0 + obj(handle["metadata"])["artifactObservationBefore"] = `[{"size":100}]` + projection := Object{} + model := Object{} + request := Object{"Workflow": Object{"activities": []any{Object{"id": "a", "metadata": Object{"workspaceProjection": projection}}}}, "NetworkTopology": Object{"metadata": Object{"workspaceMaterializationModel": model}}} + combinedObservationSensitivity(request, real, "digest") + if projection["terminalObservationBytes"] != int64(100) || math.Abs(num(rows(model["publicationProfiles"])[0]["scanSecondsPerWorkspaceByte"])-.05) > 1e-10 { + t.Fatal("publication scan used exported size or wrong wall coefficient") + } + task["startedAt"] = 89.0 + defer func() { + if recover() == nil { + t.Fatal("wrong handle attempt accepted") + } + }() + publicationReference(real, "digest") +} diff --git a/internal/montagetools/round.go b/internal/montagetools/round.go new file mode 100644 index 00000000..305ed844 --- /dev/null +++ b/internal/montagetools/round.go @@ -0,0 +1,66 @@ +package montagetools + +import ( + "os" + "path/filepath" +) + +// Prepare only: no API submission. A frozen exploratory forecast is not a +// passed validation gate. The independent run collects missing service data. +func prepareInstrumentedRound(request, simulation Object, runID, planID string) Object { + check(runPattern.MatchString(runID) && runPattern.MatchString(planID), "invalid fresh round identity") + request = clone(request) + check(len(arr(request["RuntimeAllocations"])) == 0, "fresh round must not reuse concrete allocations") + activities := rows(obj(get(request, "Workflow"))["activities"]) + assignments := rows(obj(get(request, "Plan"))["assignments"]) + tasks := rows(get(simulation, "tasks")) + check(len(activities) == 58 && len(assignments) == 58 && len(tasks) == 58, "complete Montage58 forecast required") + planned := map[string]string{} + for _, assignment := range assignments { + id := str(get(assignment, "activityId")) + check(planned[id] == "", "duplicate assignment") + planned[id] = str(get(assignment, "resourceId")) + } + seen := map[string]bool{} + for _, task := range tasks { + id := str(get(task, "activityId")) + check(!seen[id] && planned[id] != "" && planned[id] == task["allocatedResourceId"] && task["status"] == "completed", "simulation does not match complete planned placement") + seen[id] = true + } + for _, activity := range activities { + check(seen[str(get(activity, "id"))], "unplanned workflow activity") + } + plan := obj(request["Plan"]) + plan["id"] = planID + for _, assignment := range assignments { + assignment["planId"] = planID + assignment["id"] = planID + "-" + str(assignment["activityId"]) + } + for _, action := range rows(plan["lifecycleActions"]) { + action["planId"] = planID + } + metadata := obj(plan["metadata"]) + metadata["experimentRole"] = "fresh-instrumented-round-not-yet-validated" + metadata["dominantTransferGateAccepted"] = false + plan["metadata"] = metadata + request["Run"] = Object{"id": runID, "schedulePlanId": planID, "mode": "real", "seed": 1} + return request +} + +func writeInstrumentedRound(request, simulation Object, rawRequest, rawSimulation []byte, runID, planID, out string) { + _, err := os.Stat(out) + check(os.IsNotExist(err), "round output already exists or inaccessible") + prepared := prepareInstrumentedRound(request, simulation, runID, planID) + check(os.Mkdir(out, 0700) == nil, "create round directory failed") + write(filepath.Join(out, "real-request.json"), prepared) + write(filepath.Join(out, "simulation-reference.json"), simulation) + write(filepath.Join(out, "frozen-provenance.json"), Object{ + "issue": "AKO-158", "runId": runID, "planId": planID, + "inputRequestSHA256": digest(rawRequest), "simulationSHA256": digest(rawSimulation), + "realRequestSHA256": digest(encode(prepared)), + "planMakespanSeconds": obj(obj(prepared["Plan"])["predicted"])["makespanSeconds"], + "simgridMakespanSeconds": obj(simulation["executed"])["makespanSeconds"], + "role": "frozen-exploratory-forecast-for-new-measurements", + "dominantTransferGateAccepted": false, "realSubmitted": false, + }) +} diff --git a/internal/montagetools/round_test.go b/internal/montagetools/round_test.go new file mode 100644 index 00000000..3e134c50 --- /dev/null +++ b/internal/montagetools/round_test.go @@ -0,0 +1,57 @@ +package montagetools + +import ( + "fmt" + "testing" +) + +func roundFixture() (Object, Object) { + activities, assignments, tasks := []any{}, []any{}, []any{} + for i := 0; i < 58; i++ { + id := fmt.Sprintf("activity-%d", i) + activities = append(activities, Object{"id": id}) + assignments = append(assignments, Object{"activityId": id, "resourceId": "micro", "planId": "old", "id": id}) + tasks = append(tasks, Object{"activityId": id, "allocatedResourceId": "micro", "status": "completed"}) + } + return Object{"Workflow": Object{"activities": activities}, "RuntimeAllocations": []any{}, "Plan": Object{"id": "old", "assignments": assignments}, "Run": Object{"id": "old"}}, Object{"tasks": tasks} +} + +func TestInstrumentedRoundPreservesForecastAndStartsFresh(t *testing.T) { + request, simulation := roundFixture() + prepared := prepareInstrumentedRound(request, simulation, "montage-fresh", "montage-plan") + if obj(request["Run"])["id"] != "old" || obj(prepared["Run"])["mode"] != "real" || obj(prepared["Plan"])["id"] != "montage-plan" { + t.Fatal("source mutated or identity not renewed") + } + for _, assignment := range rows(obj(prepared["Plan"])["assignments"]) { + if assignment["planId"] != "montage-plan" || assignment["resourceId"] != "micro" { + t.Fatal("placement changed or stale plan relation") + } + } + if obj(obj(prepared["Plan"])["metadata"])["dominantTransferGateAccepted"] != false { + t.Fatal("preparation cannot accept the scientific gate") + } +} + +func TestInstrumentedRoundRejectsReusedOrMismatchedForecast(t *testing.T) { + for _, mode := range []string{"allocation", "placement", "failed", "duplicate"} { + t.Run(mode, func(t *testing.T) { + request, simulation := roundFixture() + switch mode { + case "allocation": + request["RuntimeAllocations"] = []any{Object{"instanceId": "old"}} + case "placement": + obj(arr(simulation["tasks"])[0])["allocatedResourceId"] = "other" + case "failed": + obj(arr(simulation["tasks"])[0])["status"] = "failed" + case "duplicate": + obj(arr(simulation["tasks"])[1])["activityId"] = "activity-0" + } + defer func() { + if recover() == nil { + t.Fatal("invalid fresh round accepted") + } + }() + prepareInstrumentedRound(request, simulation, "montage-fresh", "montage-plan") + }) + } +} diff --git a/internal/montagetools/runtime_coverage.go b/internal/montagetools/runtime_coverage.go new file mode 100644 index 00000000..20183534 --- /dev/null +++ b/internal/montagetools/runtime_coverage.go @@ -0,0 +1,100 @@ +package montagetools + +import ( + "sort" + "strings" +) + +func runtimeReferenceCoverage(references []Object) Object { + result := []any{} + for _, reference := range references { + run := obj(reference["run"]) + handles := map[string][]Object{} + for _, h := range rows(reference["handles"]) { + id, _ := h["activityId"].(string) + handles[id] = append(handles[id], h) + } + latest := map[string]Object{} + for _, t := range rows(reference["activities"]) { + id, _ := t["activityId"].(string) + if latest[id] == nil || num(t["queuedAt"]) > num(latest[id]["queuedAt"]) || (num(t["queuedAt"]) == num(latest[id]["queuedAt"]) && num(t["startedAt"]) > num(latest[id]["startedAt"])) { + latest[id] = t + } + } + groups := map[string]Object{} + completed := 0 + terminalHandles := true + for _, h := range rows(reference["handles"]) { + if h["status"] != "completed" { + terminalHandles = false + } + } + for id, t := range latest { + if t["status"] != "completed" { + continue + } + completed++ + runtimeID, _ := t["runtimeId"].(string) + resource, _ := t["allocatedResourceId"].(string) + class := referenceResourceClass(resource) + key := runtimeID + "/" + class + if groups[key] == nil { + groups[key] = Object{"runtimeId": runtimeID, "resourceClass": class, "completedActivities": 0, "singleHandleActivities": 0, "scanTelemetrySamples": 0, "nativeAdmissionSamples": 0} + } + g := groups[key] + g["completedActivities"] = int(num(g["completedActivities"])) + 1 + hs := handles[id] + if len(hs) != 1 || hs[0]["status"] != "completed" { + continue + } + g["singleHandleActivities"] = int(num(g["singleHandleActivities"])) + 1 + m := obj(hs[0]["metadata"]) + if num(m["remoteWorkspaceSnapshotSeconds"]) > 0 { + g["scanTelemetrySamples"] = int(num(g["scanTelemetrySamples"])) + 1 + } + if submitted := num(m["submittedAt"]); submitted > 0 && num(hs[0]["startedAt"]) >= submitted { + g["nativeAdmissionSamples"] = int(num(g["nativeAdmissionSamples"])) + 1 + } + } + keys := []string{} + for key := range groups { + keys = append(keys, key) + } + sort.Strings(keys) + classes := []any{} + for _, key := range keys { + classes = append(classes, groups[key]) + } + result = append(result, Object{ + "runId": run["id"], + "status": run["status"], + "distinctActivities": len(latest), + "completedDistinctActivities": completed, + "handleCount": len(rows(reference["handles"])), + "eligibleCleanWholeRunReference": run["status"] == "completed" && completed == 58 && len(latest) == 58 && len(rows(reference["handles"])) == 58 && terminalHandles, + "classes": classes, + }) + } + return Object{ + "issue": "AKO-168", + "references": result, + "limitations": list( + "Eligibility is not independence: a holdout cannot be the same run used to calibrate its case.", + "Recovered references may supply single-attempt stage samples but not a clean whole-run makespan.", + "Unknown resources remain explicitly unclassified rather than receiving borrowed coefficients.", + ), + } +} + +func referenceResourceClass(resource string) string { + switch { + case strings.Contains(resource, "fog-e2-micro"): + return "e2-micro" + case strings.Contains(resource, "e2-medium"): + return "e2-medium" + case strings.Contains(resource, "bora") || strings.Contains(resource, "diablo"): + return "hpc" + default: + return "unclassified" + } +} diff --git a/internal/montagetools/runtime_coverage_test.go b/internal/montagetools/runtime_coverage_test.go new file mode 100644 index 00000000..df658ff2 --- /dev/null +++ b/internal/montagetools/runtime_coverage_test.go @@ -0,0 +1,19 @@ +package montagetools + +import ( + "github.com/stretchr/testify/require" + "testing" +) + +func TestCoverageDoesNotTreatRecoveredCountersAsIndependentActivities(t *testing.T) { + _, training := stageFixture() + obj(training["run"])["completedActivityCount"] = 58 + h := clone(rows(training["handles"])[0]) + h["status"] = "failed" + training["handles"] = append(arr(training["handles"]), h) + r := runtimeReferenceCoverage([]Object{training}) + row := rows(r["references"])[0] + require.Equal(t, 2, row["distinctActivities"]) + require.Equal(t, false, row["eligibleCleanWholeRunReference"]) + require.Equal(t, 1, rows(row["classes"])[0]["singleHandleActivities"]) +} diff --git a/internal/montagetools/runtime_stage_calibration_test.go b/internal/montagetools/runtime_stage_calibration_test.go new file mode 100644 index 00000000..e2c57a59 --- /dev/null +++ b/internal/montagetools/runtime_stage_calibration_test.go @@ -0,0 +1,102 @@ +package montagetools + +import ( + "github.com/stretchr/testify/require" + "testing" +) + +func stageFixture() (Object, Object) { + list := func(v ...any) []any { return v } + rid := "montage-gcp-fog-e2-micro-01" + file := Object{"path": "data", "sizeBytes": 10, "contentId": "fixture-data"} + projection := Object{"contract": "whole-workspace-v1", "source": "fixture", "inputs": list(file)} + req := Object{ + "Run": Object{"mode": "real"}, + "Workflow": Object{ + "activities": list(Object{ + "id": "new-rootid0000001", + "name": "rootid0000001", + "metadata": Object{"workspaceProjection": projection}, + }, Object{ + "id": "new-childid0000002", + "name": "childid0000002", + "metadata": Object{"workspaceProjection": Object{"contract": "whole-workspace-v1", "source": "fixture", "outputs": list(Object{"path": "out", "sizeBytes": 2, "contentId": "fixture-out"})}}, + }), + "dependencies": list(Object{"activityId": "new-childid0000002", "dependsOnActivityId": "new-rootid0000001"}), + }, + "Resources": list(Object{"id": rid, "type": "cloud_vm", "schedulable": true, "metadata": Object{}}), + "Plan": Object{"assignments": list(Object{"activityId": "new-rootid0000001", "resourceId": rid, "metadata": Object{}}, Object{"activityId": "new-childid0000002", "resourceId": rid, "metadata": Object{}})}, + "NetworkTopology": Object{"metadata": Object{"workspaceMaterializationModel": Object{"mergeProfiles": list(Object{"targetClass": "hpc", "source": "unchanged"})}}}, + } + training := Object{ + "run": Object{"id": "training-run", "status": "completed"}, + "activities": list(Object{ + "id": "root-task", + "activityId": "old-rootid0000001", + "allocatedResourceId": rid, + "runtimeId": "cloud", + "status": "completed", + "startedAt": 1, + "finishedAt": 8, + "overheadSeconds": 1, + "metadata": Object{"launchWaitSeconds": 2}, + }, Object{ + "id": "child-task", + "activityId": "old-childid0000002", + "allocatedResourceId": rid, + "runtimeId": "cloud", + "status": "completed", + "startedAt": 2, + "finishedAt": 15, + "overheadSeconds": 7, + "metadata": Object{"launchWaitSeconds": 2, "transferElapsedSeconds": 1000}, + }), + "handles": list(Object{ + "activityId": "old-rootid0000001", + "status": "completed", + "startedAt": 1, + "metadata": Object{}, + }, Object{ + "activityId": "old-childid0000002", + "status": "completed", + "startedAt": 2, + "metadata": Object{"remoteWorkspaceSnapshotSeconds": 3}, + }), + "events": list(Object{ + "eventType": "activity.completed", + "aggregateId": "root-task", + "occurredAt": "1970-01-01T00:00:10Z", + }, Object{ + "eventType": "activity.completed", + "aggregateId": "child-task", + "occurredAt": "1970-01-01T00:00:20Z", + }), + "artifactTransferRuns": list(Object{ + "activityId": "old-childid0000002", + "strategy": "gateway", + "status": "completed", + "startedAt": 0, + "route": Object{"mergeCheckSeconds": 1, "mergeCopySeconds": 2, "mergeVerifySeconds": 3, "mergeCommandOverheadSeconds": 4, "mergeExecutionSeconds": 10}, + }), + } + return req, training +} + +func TestCloudStageFitDoesNotChargeNetworkResidualOrInitialScanTwice(t *testing.T) { + r, training := stageFixture() + out := calibrateRuntimeStages(r, training, "training-hash", "e2-micro") + model := obj(obj(obj(out["NetworkTopology"])["metadata"])["workspaceMaterializationModel"]) + p := rows(model["inputObservationProfiles"])[0] + require.Equal(t, 3.0, num(p["fixedSeconds"])+10*num(p["secondsPerUniqueByte"])) + require.Equal(t, 6.0, num(rows(out["Resources"])[0]["containerOverheadSeconds"])) // 2 launch + (7 overhead - 3 scan) + require.Equal(t, "simulation", obj(out["Run"])["mode"]) + require.Equal(t, "real", obj(r["Run"])["mode"]) + require.Equal(t, "hpc", rows(model["mergeProfiles"])[0]["targetClass"]) + require.Equal(t, "old-childid0000002", rows(training["activities"])[1]["activityId"]) +} + +func TestStageFitRejectsMixingRuntimeDrivers(t *testing.T) { + r, training := stageFixture() + rows(training["activities"])[0]["runtimeId"] = "other-driver" + require.Panics(t, func() { calibrateRuntimeStages(r, training, "sha", "e2-micro") }) +} diff --git a/internal/montagetools/testdata/calibration.json b/internal/montagetools/testdata/calibration.json new file mode 100644 index 00000000..d6dd3e60 --- /dev/null +++ b/internal/montagetools/testdata/calibration.json @@ -0,0 +1,3161 @@ +{ + "schemaVersion": 1, + "baselineRunId": "montage-docker-image-real-20261010-v2", + "baselineSHA256": "ce8343153d9016ce2149fa5e70b3624ca148c4747e6dd9887dae1fa2b6ab46c6", + "computeWorkBasis": null, + "synchronizationContract": "whole-workspace-v1", + "workspaceVolumeBasis": "producer-final-snapshot-manifest", + "computeObservations": [ + { + "activityFamily": "madd", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 3, + "median": 8.217379093170166, + "minimum": 0.9578084945678711, + "maximum": 8.437352895736694 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 3, + "median": 0.9374836550818549, + "minimum": 0.10642316606309679, + "maximum": 4.108689546585083 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mbackground", + "resourceClass": "e2-medium", + "runtimeSeconds": { + "samples": 2, + "median": 0.5800764560699463, + "minimum": 0.5333530902862549, + "maximum": 0.6267998218536377 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 0.32499454332434613, + "minimum": 0.02318926479505456, + "maximum": 0.6267998218536377 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mbackground", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 8, + "median": 2.1703304052352905, + "minimum": 0.9790773391723633, + "maximum": 23.867364168167114 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 8, + "median": 0.13403182718488904, + "minimum": 0.09645940860112508, + "maximum": 13.19267725944519 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mbackground", + "resourceClass": "hpc", + "runtimeSeconds": { + "samples": 2, + "median": 3.2587720155715942, + "minimum": 2.595179796218872, + "maximum": 3.9223642349243164 + }, + "taskRuntimeFallbackSamples": 2, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 2.042281486093998, + "minimum": 0.1621987372636795, + "maximum": 3.9223642349243164 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mbgmodel", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 2, + "median": 0.5460176467895508, + "minimum": 0.47063517570495605, + "maximum": 0.6214001178741455 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 0.07053212135557144, + "minimum": 0.05229279730055067, + "maximum": 0.08877144541059222 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mbgmodel", + "resourceClass": "e2-medium", + "runtimeSeconds": { + "samples": 1, + "median": 0.5198588371276855, + "minimum": 0.5198588371276855, + "maximum": 0.5198588371276855 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 1, + "median": 0.025992941856384278, + "minimum": 0.025992941856384278, + "maximum": 0.025992941856384278 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "hpc", + "runtimeSeconds": { + "samples": 1, + "median": 2.8328795433044434, + "minimum": 2.8328795433044434, + "maximum": 2.8328795433044434 + }, + "taskRuntimeFallbackSamples": 1, + "wallSecondsPerReferenceSecond": { + "samples": 1, + "median": 0.16663997313555548, + "minimum": 0.16663997313555548, + "maximum": 0.16663997313555548 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 2, + "median": 3.633373498916626, + "minimum": 2.559312343597412, + "maximum": 4.70743465423584 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 0.5174553818172878, + "minimum": 0.5118624687194824, + "maximum": 0.5230482949150933 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mdifffit", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 14, + "median": 7.9425482749938965, + "minimum": 0.6820909976959229, + "maximum": 52.214394092559814 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 14, + "median": 0.5160782317320506, + "minimum": 0.02842045823733012, + "maximum": 2.6107197046279906 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mdifffit", + "resourceClass": "hpc", + "runtimeSeconds": { + "samples": 3, + "median": 2.2217273712158203, + "minimum": 1.3592419624328613, + "maximum": 2.9261977672576904 + }, + "taskRuntimeFallbackSamples": 3, + "wallSecondsPerReferenceSecond": { + "samples": 3, + "median": 0.15401040880303635, + "minimum": 0.08495262265205383, + "maximum": 0.20197521556507458 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mdifffit", + "resourceClass": "e2-medium", + "runtimeSeconds": { + "samples": 1, + "median": 4.188046932220459, + "minimum": 4.188046932220459, + "maximum": 4.188046932220459 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 1, + "median": 0.3221574563246507, + "minimum": 0.3221574563246507, + "maximum": 0.3221574563246507 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mimgtbl", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 3, + "median": 1.4470241069793701, + "minimum": 0.1340923309326172, + "maximum": 1.9225947856903076 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 3, + "median": 0.06577382304451683, + "minimum": 0.01489914788140191, + "maximum": 0.0769037914276123 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mproject", + "resourceClass": "e2-medium", + "runtimeSeconds": { + "samples": 4, + "median": 723.2229771614075, + "minimum": 540.6099560260773, + "maximum": 1009.9635989665985 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 4, + "median": 0.5422618222257024, + "minimum": 0.482687460737569, + "maximum": 0.6058569879823626 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mproject", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 6, + "median": 2135.8616498708725, + "minimum": 1800.6897740364075, + "maximum": 3373.3067605495453 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 6, + "median": 4.142489493875232, + "minimum": 2.4189968622063263, + "maximum": 7.97978781444439 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mproject", + "resourceClass": "hpc", + "runtimeSeconds": { + "samples": 2, + "median": 432.66202092170715, + "minimum": 430.53262972831726, + "maximum": 434.79141211509705 + }, + "taskRuntimeFallbackSamples": 2, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 0.280406867515572, + "minimum": 0.2784816492421198, + "maximum": 0.2823320857890241 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mviewer", + "resourceClass": "e2-medium", + "runtimeSeconds": { + "samples": 2, + "median": 2.066543698310852, + "minimum": 1.2472658157348633, + "maximum": 2.885821580886841 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 0.24281440161857284, + "minimum": 0.07336857739616842, + "maximum": 0.41226022584097727 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + }, + { + "activityFamily": "mviewer", + "resourceClass": "e2-micro", + "runtimeSeconds": { + "samples": 2, + "median": 14.030364871025085, + "minimum": 1.6501014232635498, + "maximum": 26.41062831878662 + }, + "taskRuntimeFallbackSamples": 0, + "wallSecondsPerReferenceSecond": { + "samples": 2, + "median": 13.308445498347282, + "minimum": 0.20626267790794373, + "maximum": 26.41062831878662 + }, + "interpretation": "observed wall runtime; includes physical sharing, not clean CPU service demand" + } + ], + "materializationPhaseObservations": [ + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-micro", + "phase": "peer-authenticated-connectivity-probe", + "seconds": { + "samples": 20, + "median": 2.4162210225000003, + "minimum": 2.286490792, + "maximum": 13.712441924 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 20, + "bytes": 4719444744, + "serviceSeconds": 82.095890459, + "bytesPerSecond": 57486979.16075307, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 20, + "bytes": 7521007531, + "serviceSeconds": 82.095890459, + "bytesPerSecond": 91612472.79187636, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 18, + "median": 196610182.0, + "minimum": 18236443.0, + "maximum": 740939386.0 + }, + "archiveBytes": { + "samples": 20, + "median": 143339660.0, + "minimum": 142917128.0, + "maximum": 426325775.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 196938502.0, + "minimum": 195579142.0, + "maximum": 740939386.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.7256941966584066, + "minimum": 0.5719966097845853, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 83676394.3242459, + "minimum": 35750036.62047911, + "maximum": 311227613.81359726 + }, + "files": { + "samples": 18, + "median": 47.0, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 60827976.14251813, + "minimum": 26046095.920719363, + "maximum": 179075854.47277406 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-micro", + "phase": "peer-snapshot-acquire", + "seconds": { + "samples": 20, + "median": 6.8133940655, + "minimum": 3.5541e-05, + "maximum": 23.889355677 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 20, + "bytes": 4719444744, + "serviceSeconds": 196.708563674, + "bytesPerSecond": 23992065.49960587, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 20, + "bytes": 7521007531, + "serviceSeconds": 196.708563674, + "bytesPerSecond": 38234265.91363033, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 18, + "median": 196610182.0, + "minimum": 18236443.0, + "maximum": 740939386.0 + }, + "archiveBytes": { + "samples": 20, + "median": 143339660.0, + "minimum": 142917128.0, + "maximum": 426325775.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 196938502.0, + "minimum": 195579142.0, + "maximum": 740939386.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.7256941966584066, + "minimum": 0.5719966097845853, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 30510114.773531586, + "minimum": 18470614.9184961, + "maximum": 17811951886553.56 + }, + "files": { + "samples": 18, + "median": 47.0, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 21221299.827622443, + "minimum": 13456976.645619227, + "maximum": 10350951070594.525 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-micro", + "phase": "peer-private-upload", + "seconds": { + "samples": 20, + "median": 6.318401003, + "minimum": 3.399642126, + "maximum": 50.113527148 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 20, + "bytes": 4719444744, + "serviceSeconds": 204.741662843, + "bytesPerSecond": 23050729.775595132, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 20, + "bytes": 7521007531, + "serviceSeconds": 204.741662843, + "bytesPerSecond": 36734133.28076396, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 18, + "median": 196610182.0, + "minimum": 18236443.0, + "maximum": 740939386.0 + }, + "archiveBytes": { + "samples": 20, + "median": 143339660.0, + "minimum": 142917128.0, + "maximum": 426325775.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 196938502.0, + "minimum": 195579142.0, + "maximum": 740939386.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.7256941966584066, + "minimum": 0.5719966097845853, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 54899955.45290358, + "minimum": 7649470.226514639, + "maximum": 113099800.15949468 + }, + "files": { + "samples": 18, + "median": 47.0, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 39840579.06897718, + "minimum": 5610082.049398552, + "maximum": 64692702.25854507 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-micro", + "phase": "peer-receiver-verify-and-extract", + "seconds": { + "samples": 20, + "median": 11.3101169635, + "minimum": 2.66509171, + "maximum": 237.244030355 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 20, + "bytes": 4719444744, + "serviceSeconds": 703.761369402, + "bytesPerSecond": 6706029.84078283, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 20, + "bytes": 7521007531, + "serviceSeconds": 703.761369402, + "bytesPerSecond": 10686871.797738414, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 18, + "median": 196610182.0, + "minimum": 18236443.0, + "maximum": 740939386.0 + }, + "archiveBytes": { + "samples": 20, + "median": 143339660.0, + "minimum": 142917128.0, + "maximum": 426325775.0 + }, + "workspaceBytes": { + "samples": 20, + "median": 196938502.0, + "minimum": 195579142.0, + "maximum": 740939386.0 + }, + "archiveToWorkspaceRatio": { + "samples": 20, + "median": 0.7256941966584066, + "minimum": 0.5719966097845853, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 20, + "median": 45867943.07818958, + "minimum": 3123110.768651568, + "maximum": 73402810.59221035 + }, + "files": { + "samples": 18, + "median": 47.0, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 20, + "median": 29496458.305427484, + "minimum": 1796992.633964562, + "maximum": 53733886.32843708 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-authenticated-connectivity-probe", + "seconds": { + "samples": 45, + "median": 2.444578959, + "minimum": 2.267388085, + "maximum": 14.119102382 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 45, + "bytes": 14768952722, + "serviceSeconds": 176.824679621, + "bytesPerSecond": 83523141.41699716, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 45, + "bytes": 24933404926, + "serviceSeconds": 176.824679621, + "bytesPerSecond": 141006362.79642305, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 44, + "median": 106346880.0, + "minimum": 2594.0, + "maximum": 1109057816.0 + }, + "archiveBytes": { + "samples": 45, + "median": 363111632.0, + "minimum": 141375291.0, + "maximum": 619720534.0 + }, + "workspaceBytes": { + "samples": 45, + "median": 627864822.0, + "minimum": 195625222.0, + "maximum": 1109057816.0 + }, + "archiveToWorkspaceRatio": { + "samples": 45, + "median": 0.578327721631775, + "minimum": 0.5587810888300886, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 45, + "median": 134248084.80919027, + "minimum": 26694732.13031702, + "maximum": 481931462.75104815 + }, + "files": { + "samples": 44, + "median": 2.5, + "minimum": 1.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 45, + "median": 85519694.35317792, + "minimum": 19647136.301396284, + "maximum": 269499698.2431896 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-snapshot-acquire", + "seconds": { + "samples": 45, + "median": 47.297451689, + "minimum": 0.000196, + "maximum": 724.592180869 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 45, + "bytes": 14768952722, + "serviceSeconds": 4994.773932149, + "bytesPerSecond": 2956881.1166686104, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 45, + "bytes": 24933404926, + "serviceSeconds": 4994.773932149, + "bytesPerSecond": 4991898.585342462, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 44, + "median": 106346880.0, + "minimum": 2594.0, + "maximum": 1109057816.0 + }, + "archiveBytes": { + "samples": 45, + "median": 363111632.0, + "minimum": 141375291.0, + "maximum": 619720534.0 + }, + "workspaceBytes": { + "samples": 45, + "median": 627864822.0, + "minimum": 195625222.0, + "maximum": 1109057816.0 + }, + "archiveToWorkspaceRatio": { + "samples": 45, + "median": 0.578327721631775, + "minimum": 0.5587810888300886, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 45, + "median": 13591501.74435183, + "minimum": 1446547.9612862365, + "maximum": 998293581632.6531 + }, + "files": { + "samples": 44, + "median": 2.5, + "minimum": 1.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 45, + "median": 7600469.9877143195, + "minimum": 810355.8022055948, + "maximum": 725908500000.0 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-private-upload", + "seconds": { + "samples": 45, + "median": 21.295400718, + "minimum": 3.692133835, + "maximum": 204.790461176 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 45, + "bytes": 14768952722, + "serviceSeconds": 1391.840169336, + "bytesPerSecond": 10611098.204649296, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 45, + "bytes": 24933404926, + "serviceSeconds": 1391.840169336, + "bytesPerSecond": 17913985.72574241, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 44, + "median": 106346880.0, + "minimum": 2594.0, + "maximum": 1109057816.0 + }, + "archiveBytes": { + "samples": 45, + "median": 363111632.0, + "minimum": 141375291.0, + "maximum": 619720534.0 + }, + "workspaceBytes": { + "samples": 45, + "median": 627864822.0, + "minimum": 195625222.0, + "maximum": 1109057816.0 + }, + "archiveToWorkspaceRatio": { + "samples": 45, + "median": 0.578327721631775, + "minimum": 0.5587810888300886, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 45, + "median": 27154065.76248039, + "minimum": 5056332.734069341, + "maximum": 127839669.3181365 + }, + "files": { + "samples": 44, + "median": 2.5, + "minimum": 1.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 45, + "median": 17623724.802639503, + "minimum": 2912047.4390039076, + "maximum": 71566066.50891274 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-receiver-verify-and-extract", + "seconds": { + "samples": 45, + "median": 12.808673423, + "minimum": 2.726851626, + "maximum": 467.73053142 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 45, + "bytes": 14768952722, + "serviceSeconds": 2264.991643906, + "bytesPerSecond": 6520532.983746818, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 45, + "bytes": 24933404926, + "serviceSeconds": 2264.991643906, + "bytesPerSecond": 11008166.406743163, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 44, + "median": 106346880.0, + "minimum": 2594.0, + "maximum": 1109057816.0 + }, + "archiveBytes": { + "samples": 45, + "median": 363111632.0, + "minimum": 141375291.0, + "maximum": 619720534.0 + }, + "workspaceBytes": { + "samples": 45, + "median": 627864822.0, + "minimum": 195625222.0, + "maximum": 1109057816.0 + }, + "archiveToWorkspaceRatio": { + "samples": 45, + "median": 0.578327721631775, + "minimum": 0.5587810888300886, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 45, + "median": 45018555.18656611, + "minimum": 1584029.7270111442, + "maximum": 71755111.32852522 + }, + "files": { + "samples": 44, + "median": 2.5, + "minimum": 1.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 45, + "median": 28348867.599979248, + "minimum": 911255.8500425805, + "maximum": 52176680.47773714 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "pack-and-receive", + "seconds": { + "samples": 11, + "median": 42.532719728000004, + "minimum": 11.220321047, + "maximum": 102.588095879 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 11, + "bytes": 2603160156, + "serviceSeconds": 535.036846578, + "bytesPerSecond": 4865384.828445642, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 11, + "bytes": 4560861956, + "serviceSeconds": 535.036846578, + "bytesPerSecond": 8524388.526080882, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 10, + "median": 106346880.0, + "minimum": 18391965.0, + "maximum": 739073142.0 + }, + "archiveBytes": { + "samples": 11, + "median": 208894084.0, + "minimum": 51992429.0, + "maximum": 426351589.0 + }, + "workspaceBytes": { + "samples": 11, + "median": 321020707.0, + "minimum": 106018560.0, + "maximum": 740899066.0 + }, + "archiveToWorkspaceRatio": { + "samples": 11, + "median": 0.5736604862310096, + "minimum": 0.4904087454121241, + "maximum": 0.6507184098874967 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 11, + "median": 7633347.82034076, + "minimum": 2678102.132784162, + "maximum": 12998257.441814132 + }, + "files": { + "samples": 10, + "median": 2.5, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 11, + "median": 4565984.029781632, + "minimum": 1313364.7070242148, + "maximum": 7479868.674241965 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "archive-validation", + "seconds": { + "samples": 11, + "median": 1.732665959, + "minimum": 0.502052333, + "maximum": 3.854221543 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 11, + "bytes": 2603160156, + "serviceSeconds": 24.244881968, + "bytesPerSecond": 107369471.19131464, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 11, + "bytes": 4560861956, + "serviceSeconds": 24.244881968, + "bytesPerSecond": 188116484.21385294, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 10, + "median": 106346880.0, + "minimum": 18391965.0, + "maximum": 739073142.0 + }, + "archiveBytes": { + "samples": 11, + "median": 208894084.0, + "minimum": 51992429.0, + "maximum": 426351589.0 + }, + "workspaceBytes": { + "samples": 11, + "median": 321020707.0, + "minimum": 106018560.0, + "maximum": 740899066.0 + }, + "archiveToWorkspaceRatio": { + "samples": 11, + "median": 0.5736604862310096, + "minimum": 0.4904087454121241, + "maximum": 0.6507184098874967 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 11, + "median": 193524669.92053956, + "minimum": 164995666.03024346, + "maximum": 211170336.30037928 + }, + "files": { + "samples": 10, + "median": 2.5, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 11, + "median": 108211123.48062727, + "minimum": 89732925.24604906, + "maximum": 120562237.00531535 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "archive-upload", + "seconds": { + "samples": 11, + "median": 33.194828307, + "minimum": 5.755920378, + "maximum": 88.282822748 + }, + "secondsBases": [ + "phase-command-execution-excludes-channel-queue" + ], + "pooledArchiveServiceRate": { + "samples": 11, + "bytes": 2603160156, + "serviceSeconds": 384.441166964, + "bytesPerSecond": 6771283.566111343, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 11, + "bytes": 4560861956, + "serviceSeconds": 384.441166964, + "bytesPerSecond": 11863614.898523837, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": { + "samples": 11, + "median": 9.708e-06, + "minimum": 6.75e-06, + "maximum": 0.00033975 + }, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 10, + "median": 106346880.0, + "minimum": 18391965.0, + "maximum": 739073142.0 + }, + "archiveBytes": { + "samples": 11, + "median": 208894084.0, + "minimum": 51992429.0, + "maximum": 426351589.0 + }, + "workspaceBytes": { + "samples": 11, + "median": 321020707.0, + "minimum": 106018560.0, + "maximum": 740899066.0 + }, + "archiveToWorkspaceRatio": { + "samples": 11, + "median": 0.5736604862310096, + "minimum": 0.4904087454121241, + "maximum": 0.6507184098874967 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 11, + "median": 15118533.681912266, + "minimum": 2078076.62672509, + "maximum": 20852720.505293377 + }, + "files": { + "samples": 10, + "median": 2.5, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 11, + "median": 8139339.578303657, + "minimum": 1064879.7098892068, + "maximum": 11962381.784305941 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "phase": "receiver-verify-and-extract", + "seconds": { + "samples": 11, + "median": 7.516678671, + "minimum": 1.272753501, + "maximum": 209.987703305 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 11, + "bytes": 2603160156, + "serviceSeconds": 505.157891857, + "bytesPerSecond": 5153161.413415872, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 11, + "bytes": 4560861956, + "serviceSeconds": 505.157891857, + "bytesPerSecond": 9028586.96166047, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 10, + "median": 106346880.0, + "minimum": 18391965.0, + "maximum": 739073142.0 + }, + "archiveBytes": { + "samples": 11, + "median": 208894084.0, + "minimum": 51992429.0, + "maximum": 426351589.0 + }, + "workspaceBytes": { + "samples": 11, + "median": 321020707.0, + "minimum": 106018560.0, + "maximum": 740899066.0 + }, + "archiveToWorkspaceRatio": { + "samples": 11, + "median": 0.5736604862310096, + "minimum": 0.4904087454121241, + "maximum": 0.6507184098874967 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 11, + "median": 46798115.17037942, + "minimum": 3528297.392366206, + "maximum": 98548224.98363979 + }, + "files": { + "samples": 10, + "median": 2.5, + "minimum": 2.0, + "maximum": 75.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 11, + "median": 23981003.71521475, + "minimum": 2030364.5513029818, + "maximum": 57403030.977419436 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-medium", + "phase": "peer-authenticated-connectivity-probe", + "seconds": { + "samples": 7, + "median": 3.238551043, + "minimum": 2.347021292, + "maximum": 5.747049585 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 7, + "bytes": 3584464125, + "serviceSeconds": 23.434317592, + "bytesPerSecond": 152957905.04365543, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 7, + "bytes": 6328189016, + "serviceSeconds": 23.434317592, + "bytesPerSecond": 270039398.0390671, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 7, + "median": 1009451800.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveBytes": { + "samples": 7, + "median": 615130007.0, + "minimum": 363111632.0, + "maximum": 624504559.0 + }, + "workspaceBytes": { + "samples": 7, + "median": 1100003102.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveToWorkspaceRatio": { + "samples": 7, + "median": 0.5616915015784198, + "minimum": 0.5587810888300886, + "maximum": 0.5824735682180756 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 7, + "median": 333062072.17141193, + "minimum": 109249939.94113939, + "maximum": 471436381.27734333 + }, + "files": { + "samples": 7, + "median": 73.0, + "minimum": 39.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 7, + "median": 186108787.3359471, + "minimum": 63182268.8545674, + "maximum": 264801808.89836743 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-medium", + "phase": "peer-snapshot-acquire", + "seconds": { + "samples": 7, + "median": 222.943990226, + "minimum": 7.5167e-05, + "maximum": 362.706896956 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 7, + "bytes": 3584464125, + "serviceSeconds": 1370.467591624, + "bytesPerSecond": 2615504.4795713997, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 7, + "bytes": 6328189016, + "serviceSeconds": 1370.467591624, + "bytesPerSecond": 4617540.067839995, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 7, + "median": 1009451800.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveBytes": { + "samples": 7, + "median": 615130007.0, + "minimum": 363111632.0, + "maximum": 624504559.0 + }, + "workspaceBytes": { + "samples": 7, + "median": 1100003102.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveToWorkspaceRatio": { + "samples": 7, + "median": 0.5616915015784198, + "minimum": 0.5587810888300886, + "maximum": 0.5824735682180756 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 7, + "median": 3071406.804890141, + "minimum": 1914116.5272692032, + "maximum": 14754584006279.354 + }, + "files": { + "samples": 7, + "median": 73.0, + "minimum": 39.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 7, + "median": 1725183.10019692, + "minimum": 1106986.6501533238, + "maximum": 8244582516263.787 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-medium", + "phase": "peer-reused-source-manifest-check", + "seconds": { + "samples": 3, + "median": 14.740183174, + "minimum": 8.123189962, + "maximum": 54.534579608 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 3, + "bytes": 1613834969, + "serviceSeconds": 77.39795274400001, + "bytesPerSecond": 20851132.514291298, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 3, + "bytes": 2855438394, + "serviceSeconds": 77.39795274400001, + "bytesPerSecond": 36892944.745510176, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 3, + "median": 1018506514.0, + "minimum": 634552186.0, + "maximum": 1111828392.0 + }, + "archiveBytes": { + "samples": 3, + "median": 619720534.0, + "minimum": 369609876.0, + "maximum": 624504559.0 + }, + "workspaceBytes": { + "samples": 3, + "median": 1109057816.0, + "minimum": 634552186.0, + "maximum": 1111828392.0 + }, + "archiveToWorkspaceRatio": { + "samples": 3, + "median": 0.5616915015784198, + "minimum": 0.5587810888300886, + "maximum": 0.5824735682180756 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 3, + "median": 43049138.43399705, + "minimum": 20387585.271435726, + "maximum": 136529838.79339692 + }, + "files": { + "samples": 3, + "median": 73.0, + "minimum": 39.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 3, + "median": 25074985.27236416, + "minimum": 11451533.38467081, + "maximum": 76290291.97877079 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-medium", + "phase": "peer-private-upload", + "seconds": { + "samples": 7, + "median": 27.623296388, + "minimum": 7.566221212, + "maximum": 98.32883392 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 7, + "bytes": 3584464125, + "serviceSeconds": 277.584645627, + "bytesPerSecond": 12913048.979721189, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 7, + "bytes": 6328189016, + "serviceSeconds": 277.584645627, + "bytesPerSecond": 22797330.888767548, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 7, + "median": 1009451800.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveBytes": { + "samples": 7, + "median": 615130007.0, + "minimum": 363111632.0, + "maximum": 624504559.0 + }, + "workspaceBytes": { + "samples": 7, + "median": 1100003102.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveToWorkspaceRatio": { + "samples": 7, + "median": 0.5616915015784198, + "minimum": 0.5587810888300886, + "maximum": 0.5824735682180756 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 7, + "median": 40249663.77593501, + "minimum": 6385358.159650593, + "maximum": 146580146.80314106 + }, + "files": { + "samples": 7, + "median": 73.0, + "minimum": 39.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 7, + "median": 22607894.084331468, + "minimum": 3692829.636273592, + "maximum": 81906214.03153339 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-medium", + "phase": "peer-receiver-verify-and-extract", + "seconds": { + "samples": 7, + "median": 10.862616296, + "minimum": 5.661688502, + "maximum": 12.127251798 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 7, + "bytes": 3584464125, + "serviceSeconds": 66.142443239, + "bytesPerSecond": 54193101.27459079, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 7, + "bytes": 6328189016, + "serviceSeconds": 66.142443239, + "bytesPerSecond": 95675162.6657279, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 7, + "median": 1009451800.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveBytes": { + "samples": 7, + "median": 615130007.0, + "minimum": 363111632.0, + "maximum": 624504559.0 + }, + "workspaceBytes": { + "samples": 7, + "median": 1100003102.0, + "minimum": 627864822.0, + "maximum": 1111828392.0 + }, + "archiveToWorkspaceRatio": { + "samples": 7, + "median": 0.5616915015784198, + "minimum": 0.5587810888300886, + "maximum": 0.5824735682180756 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 7, + "median": 94960134.68711175, + "minimum": 83793313.37409675, + "maximum": 110897097.53162962 + }, + "files": { + "samples": 7, + "median": 73.0, + "minimum": 39.0, + "maximum": 84.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 7, + "median": 53061927.45591616, + "minimum": 48807390.23382553, + "maximum": 64134865.75104411 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-micro", + "targetClass": "hpc", + "phase": "pack-and-receive", + "seconds": { + "samples": 10, + "median": 38.603155225500004, + "minimum": 11.782335589, + "maximum": 199.5754098 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 10, + "bytes": 1991885972, + "serviceSeconds": 595.318555566, + "bytesPerSecond": 3345916.154261665, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 10, + "bytes": 3038347972, + "serviceSeconds": 595.318555566, + "bytesPerSecond": 5103734.704038053, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": null, + "archiveBytes": { + "samples": 10, + "median": 203193258.0, + "minimum": 143696217.0, + "maximum": 369609876.0 + }, + "workspaceBytes": { + "samples": 10, + "median": 312516065.5, + "minimum": 196610182.0, + "maximum": 634552186.0 + }, + "archiveToWorkspaceRatio": { + "samples": 10, + "median": 0.6522022259028002, + "minimum": 0.5824735682180756, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 10, + "median": 12514547.202893108, + "minimum": 1678160.6127510006, + "maximum": 19765033.306060687 + }, + "files": { + "samples": 9, + "median": 5.0, + "minimum": 2.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 10, + "median": 7741110.27791067, + "minimum": 1075263.5668645387, + "maximum": 12819134.471322034 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-micro", + "targetClass": "hpc", + "phase": "archive-validation", + "seconds": { + "samples": 10, + "median": 1.6895398135000002, + "minimum": 1.133418042, + "maximum": 3.341227627 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 10, + "bytes": 1991885972, + "serviceSeconds": 16.802207758999998, + "bytesPerSecond": 118549062.1572072, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 10, + "bytes": 3038347972, + "serviceSeconds": 16.802207758999998, + "bytesPerSecond": 180830282.28076324, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": null, + "archiveBytes": { + "samples": 10, + "median": 203193258.0, + "minimum": 143696217.0, + "maximum": 369609876.0 + }, + "workspaceBytes": { + "samples": 10, + "median": 312516065.5, + "minimum": 196610182.0, + "maximum": 634552186.0 + }, + "archiveToWorkspaceRatio": { + "samples": 10, + "median": 0.6522022259028002, + "minimum": 0.5824735682180756, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 10, + "median": 179804359.97152635, + "minimum": 167484361.96237248, + "maximum": 189915880.28073013 + }, + "files": { + "samples": 9, + "median": 5.0, + "minimum": 2.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 10, + "median": 120277383.91281037, + "minimum": 110620980.44839375, + "maximum": 127857225.33963335 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-micro", + "targetClass": "hpc", + "phase": "archive-upload", + "seconds": { + "samples": 10, + "median": 76.61885259799999, + "minimum": 24.609660678, + "maximum": 171.383548828 + }, + "secondsBases": [ + "phase-command-execution-excludes-channel-queue" + ], + "pooledArchiveServiceRate": { + "samples": 10, + "bytes": 1991885972, + "serviceSeconds": 952.487866393, + "bytesPerSecond": 2091245.5079801935, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 10, + "bytes": 3038347972, + "serviceSeconds": 952.487866393, + "bytesPerSecond": 3189907.272526206, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": { + "samples": 10, + "median": 1.72295e-05, + "minimum": 6.959e-06, + "maximum": 4.3083e-05 + }, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": null, + "archiveBytes": { + "samples": 10, + "median": 203193258.0, + "minimum": 143696217.0, + "maximum": 369609876.0 + }, + "workspaceBytes": { + "samples": 10, + "median": 312516065.5, + "minimum": 196610182.0, + "maximum": 634552186.0 + }, + "archiveToWorkspaceRatio": { + "samples": 10, + "median": 0.6522022259028002, + "minimum": 0.5824735682180756, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 10, + "median": 5540640.911608172, + "minimum": 1852594.8200816612, + "maximum": 8000849.12085028 + }, + "files": { + "samples": 9, + "median": 5.0, + "minimum": 2.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 10, + "median": 3704148.753778026, + "minimum": 1207509.6908446734, + "maximum": 5895535.127377915 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-micro", + "targetClass": "hpc", + "phase": "receiver-verify-and-extract", + "seconds": { + "samples": 10, + "median": 7.1363985240000005, + "minimum": 3.683395543, + "maximum": 10.001879504 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 10, + "bytes": 1991885972, + "serviceSeconds": 65.596834363, + "bytesPerSecond": 30365580.76838425, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 10, + "bytes": 3038347972, + "serviceSeconds": 65.596834363, + "bytesPerSecond": 46318515.23789058, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": null, + "archiveBytes": { + "samples": 10, + "median": 203193258.0, + "minimum": 143696217.0, + "maximum": 369609876.0 + }, + "workspaceBytes": { + "samples": 10, + "median": 312516065.5, + "minimum": 196610182.0, + "maximum": 634552186.0 + }, + "archiveToWorkspaceRatio": { + "samples": 10, + "median": 0.6522022259028002, + "minimum": 0.5824735682180756, + "maximum": 0.7368636801329126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 10, + "median": 46164951.51762764, + "minimum": 34201395.10569445, + "maximum": 63443294.40743881 + }, + "files": { + "samples": 9, + "median": 5.0, + "minimum": 2.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 10, + "median": 31838848.70171778, + "minimum": 21772903.484861646, + "maximum": 39342960.6753477 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "phase": "peer-reused-source-manifest-check", + "seconds": { + "samples": 6, + "median": 27.712840638, + "minimum": 0.474361584, + "maximum": 50.339336731 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 6, + "bytes": 1520224865, + "serviceSeconds": 159.848747074, + "bytesPerSecond": 9510395.876273155, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 6, + "bytes": 2473056372, + "serviceSeconds": 159.848747074, + "bytesPerSecond": 15471227.752915256, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 6, + "median": 411765182.0, + "minimum": 106346880.0, + "maximum": 627864822.0 + }, + "archiveBytes": { + "samples": 6, + "median": 254013659.0, + "minimum": 142278066.0, + "maximum": 363111632.0 + }, + "workspaceBytes": { + "samples": 6, + "median": 412381502.0, + "minimum": 195665542.0, + "maximum": 627864822.0 + }, + "archiveToWorkspaceRatio": { + "samples": 6, + "median": 0.6527385212944301, + "minimum": 0.578327721631775, + "maximum": 0.7359930118603126 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 6, + "median": 25591306.483499434, + "minimum": 12472647.888770213, + "maximum": 412481846.3377085 + }, + "files": { + "samples": 6, + "median": 60.0, + "minimum": 2.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 6, + "median": 17731894.794842176, + "minimum": 7213278.0362278465, + "maximum": 299935894.47158945 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-medium", + "targetClass": "hpc", + "phase": "pack-and-receive", + "seconds": { + "samples": 1, + "median": 30.418417181, + "minimum": 30.418417181, + "maximum": 30.418417181 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 367883152, + "serviceSeconds": 30.418417181, + "bytesPerSecond": 12094092.529896256, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 633054582, + "serviceSeconds": 30.418417181, + "bytesPerSecond": 20811555.651732583, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": null, + "archiveBytes": { + "samples": 1, + "median": 367883152.0, + "minimum": 367883152.0, + "maximum": 367883152.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 633054582.0, + "minimum": 633054582.0, + "maximum": 633054582.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5811239069429878, + "minimum": 0.5811239069429878, + "maximum": 0.5811239069429878 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 20811555.651732583, + "minimum": 20811555.651732583, + "maximum": 20811555.651732583 + }, + "files": { + "samples": 1, + "median": 73.0, + "minimum": 73.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 12094092.529896256, + "minimum": 12094092.529896256, + "maximum": 12094092.529896256 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-medium", + "targetClass": "hpc", + "phase": "archive-validation", + "seconds": { + "samples": 1, + "median": 3.350000627, + "minimum": 3.350000627, + "maximum": 3.350000627 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 367883152, + "serviceSeconds": 3.350000627, + "bytesPerSecond": 109815845.7150641, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 633054582, + "serviceSeconds": 3.350000627, + "bytesPerSecond": 188971481.64623314, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": null, + "archiveBytes": { + "samples": 1, + "median": 367883152.0, + "minimum": 367883152.0, + "maximum": 367883152.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 633054582.0, + "minimum": 633054582.0, + "maximum": 633054582.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5811239069429878, + "minimum": 0.5811239069429878, + "maximum": 0.5811239069429878 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 188971481.64623314, + "minimum": 188971481.64623314, + "maximum": 188971481.64623314 + }, + "files": { + "samples": 1, + "median": 73.0, + "minimum": 73.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 109815845.7150641, + "minimum": 109815845.7150641, + "maximum": 109815845.7150641 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-medium", + "targetClass": "hpc", + "phase": "archive-upload", + "seconds": { + "samples": 1, + "median": 109.214267425, + "minimum": 109.214267425, + "maximum": 109.214267425 + }, + "secondsBases": [ + "phase-command-execution-excludes-channel-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 367883152, + "serviceSeconds": 109.214267425, + "bytesPerSecond": 3368453.2311919224, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 633054582, + "serviceSeconds": 109.214267425, + "bytesPerSecond": 5796445.802602974, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": { + "samples": 1, + "median": 2.1416e-05, + "minimum": 2.1416e-05, + "maximum": 2.1416e-05 + }, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": null, + "archiveBytes": { + "samples": 1, + "median": 367883152.0, + "minimum": 367883152.0, + "maximum": 367883152.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 633054582.0, + "minimum": 633054582.0, + "maximum": 633054582.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5811239069429878, + "minimum": 0.5811239069429878, + "maximum": 0.5811239069429878 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 5796445.802602974, + "minimum": 5796445.802602974, + "maximum": 5796445.802602974 + }, + "files": { + "samples": 1, + "median": 73.0, + "minimum": 73.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 3368453.2311919224, + "minimum": 3368453.2311919224, + "maximum": 3368453.2311919224 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "gateway", + "sourceClass": "e2-medium", + "targetClass": "hpc", + "phase": "receiver-verify-and-extract", + "seconds": { + "samples": 1, + "median": 10.328649046, + "minimum": 10.328649046, + "maximum": 10.328649046 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 367883152, + "serviceSeconds": 10.328649046, + "bytesPerSecond": 35617741.52278617, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 633054582, + "serviceSeconds": 10.328649046, + "bytesPerSecond": 61291131.02600426, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": null, + "archiveBytes": { + "samples": 1, + "median": 367883152.0, + "minimum": 367883152.0, + "maximum": 367883152.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 633054582.0, + "minimum": 633054582.0, + "maximum": 633054582.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.5811239069429878, + "minimum": 0.5811239069429878, + "maximum": 0.5811239069429878 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 61291131.02600426, + "minimum": 61291131.02600426, + "maximum": 61291131.02600426 + }, + "files": { + "samples": 1, + "median": 73.0, + "minimum": 73.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 35617741.52278617, + "minimum": 35617741.52278617, + "maximum": 35617741.52278617 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-micro", + "phase": "peer-reused-source-manifest-check", + "seconds": { + "samples": 4, + "median": 0.66461273, + "minimum": 0.495965208, + "maximum": 0.82791425 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 4, + "bytes": 1022120167, + "serviceSeconds": 2.653104918, + "bytesPerSecond": 385254333.5415882, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 4, + "bytes": 1658626808, + "serviceSeconds": 2.653104918, + "bytesPerSecond": 625164424.0478544, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 4, + "median": 414996542.0, + "minimum": 105027840.0, + "maximum": 633054582.0 + }, + "archiveBytes": { + "samples": 4, + "median": 255659943.5, + "minimum": 142917128.0, + "maximum": 367883152.0 + }, + "workspaceBytes": { + "samples": 4, + "median": 414996542.0, + "minimum": 195579142.0, + "maximum": 633054582.0 + }, + "archiveToWorkspaceRatio": { + "samples": 4, + "median": 0.6534090518006972, + "minimum": 0.5811239069429878, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 4, + "median": 579489154.6585486, + "minimum": 392106576.9312505, + "maximum": 765512906.0901998 + }, + "files": { + "samples": 4, + "median": 60.0, + "minimum": 2.0, + "maximum": 73.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 4, + "median": 366778296.9747619, + "minimum": 284549467.35060155, + "maximum": 444857850.8024174 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-medium", + "phase": "peer-authenticated-connectivity-probe", + "seconds": { + "samples": 1, + "median": 4.352586419, + "minimum": 4.352586419, + "maximum": 4.352586419 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 143436735, + "serviceSeconds": 4.352586419, + "bytesPerSecond": 32954368.091088787, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 195579142, + "serviceSeconds": 4.352586419, + "bytesPerSecond": 44934005.479191385, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveBytes": { + "samples": 1, + "median": 143436735.0, + "minimum": 143436735.0, + "maximum": 143436735.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.7333948473912417, + "minimum": 0.7333948473912417, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 44934005.479191385, + "minimum": 44934005.479191385, + "maximum": 44934005.479191385 + }, + "files": { + "samples": 1, + "median": 47.0, + "minimum": 47.0, + "maximum": 47.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 32954368.091088787, + "minimum": 32954368.091088787, + "maximum": 32954368.091088787 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-medium", + "phase": "peer-snapshot-acquire", + "seconds": { + "samples": 1, + "median": 15.533482091, + "minimum": 15.533482091, + "maximum": 15.533482091 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 143436735, + "serviceSeconds": 15.533482091, + "bytesPerSecond": 9234036.139463304, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 195579142, + "serviceSeconds": 15.533482091, + "bytesPerSecond": 12590811.310318973, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveBytes": { + "samples": 1, + "median": 143436735.0, + "minimum": 143436735.0, + "maximum": 143436735.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.7333948473912417, + "minimum": 0.7333948473912417, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 12590811.310318973, + "minimum": 12590811.310318973, + "maximum": 12590811.310318973 + }, + "files": { + "samples": 1, + "median": 47.0, + "minimum": 47.0, + "maximum": 47.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 9234036.139463304, + "minimum": 9234036.139463304, + "maximum": 9234036.139463304 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-medium", + "phase": "peer-private-upload", + "seconds": { + "samples": 1, + "median": 5.817277252, + "minimum": 5.817277252, + "maximum": 5.817277252 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 143436735, + "serviceSeconds": 5.817277252, + "bytesPerSecond": 24657022.312403273, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 195579142, + "serviceSeconds": 5.817277252, + "bytesPerSecond": 33620392.07135249, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveBytes": { + "samples": 1, + "median": 143436735.0, + "minimum": 143436735.0, + "maximum": 143436735.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.7333948473912417, + "minimum": 0.7333948473912417, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 33620392.07135249, + "minimum": 33620392.07135249, + "maximum": 33620392.07135249 + }, + "files": { + "samples": 1, + "median": 47.0, + "minimum": 47.0, + "maximum": 47.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 24657022.312403273, + "minimum": 24657022.312403273, + "maximum": 24657022.312403273 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-medium", + "phase": "peer-receiver-verify-and-extract", + "seconds": { + "samples": 1, + "median": 5.928769003, + "minimum": 5.928769003, + "maximum": 5.928769003 + }, + "secondsBases": [ + "combined-phase-wall-includes-unseparated-service-queue" + ], + "pooledArchiveServiceRate": { + "samples": 1, + "bytes": 143436735, + "serviceSeconds": 5.928769003, + "bytesPerSecond": 24193341.809643783, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "pooledWorkspaceServiceRate": { + "samples": 1, + "bytes": 195579142, + "serviceSeconds": 5.928769003, + "bytesPerSecond": 32988153.510625146, + "interpretation": "byte-weighted per-command service coefficient; summed service is not concurrent wall time" + }, + "phaseChannelWaitSeconds": null, + "sourceChannelWaitSeconds": null, + "sourcePreflightSeconds": null, + "sourcePreflightServiceSeconds": null, + "sourcePreflightChannelWaitSeconds": null, + "logicalBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveBytes": { + "samples": 1, + "median": 143436735.0, + "minimum": 143436735.0, + "maximum": 143436735.0 + }, + "workspaceBytes": { + "samples": 1, + "median": 195579142.0, + "minimum": 195579142.0, + "maximum": 195579142.0 + }, + "archiveToWorkspaceRatio": { + "samples": 1, + "median": 0.7333948473912417, + "minimum": 0.7333948473912417, + "maximum": 0.7333948473912417 + }, + "effectiveWorkspaceBytesPerSecond": { + "samples": 1, + "median": 32988153.510625146, + "minimum": 32988153.510625146, + "maximum": 32988153.510625146 + }, + "files": { + "samples": 1, + "median": 47.0, + "minimum": 47.0, + "maximum": 47.0 + }, + "effectiveArchiveBytesPerSecond": { + "samples": 1, + "median": 24193341.809643783, + "minimum": 24193341.809643783, + "maximum": 24193341.809643783 + }, + "interpretation": "concurrency-conditioned effective phase rates; not isolated link bandwidth" + } + ], + "consumerMergeObservations": [ + { + "activityFamily": "madd", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 3, + "median": 236.587417607, + "minimum": 66.450399947, + "maximum": 243.573326361 + }, + "checkSeconds": { + "samples": 3, + "median": 142.49716386900036, + "minimum": 25.46066779800003, + "maximum": 145.75149784399946 + }, + "copySeconds": { + "samples": 3, + "median": 53.56884224699934, + "minimum": 33.81956537900078, + "maximum": 63.3139910280006 + }, + "verifySeconds": { + "samples": 3, + "median": 35.92816998099988, + "minimum": 6.668966102999548, + "maximum": 37.1964132789999 + } + }, + { + "activityFamily": "mbackground", + "resourceClass": "e2-medium", + "mergeSeconds": { + "samples": 2, + "median": 4.83583319, + "minimum": 4.176778294, + "maximum": 5.494888086 + }, + "checkSeconds": { + "samples": 2, + "median": 1.6694268080004804, + "minimum": 1.6326302810002744, + "maximum": 1.7062233350006863 + }, + "copySeconds": { + "samples": 2, + "median": 2.1372344224996596, + "minimum": 1.4865193229998113, + "maximum": 2.787949521999508 + }, + "verifySeconds": { + "samples": 2, + "median": 0.5077693425000689, + "minimum": 0.5032371529996453, + "maximum": 0.5123015320004924 + } + }, + { + "activityFamily": "mbackground", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 8, + "median": 19.92376553, + "minimum": 19.626282426, + "maximum": 138.21304698 + }, + "checkSeconds": { + "samples": 8, + "median": 5.024046718499903, + "minimum": 4.7745690230003675, + "maximum": 53.48430508499996 + }, + "copySeconds": { + "samples": 8, + "median": 10.670739221000076, + "minimum": 10.32359564699982, + "maximum": 40.23239413300007 + }, + "verifySeconds": { + "samples": 8, + "median": 3.8808005935002257, + "minimum": 3.5650197659997502, + "maximum": 44.77779699300004 + } + }, + { + "activityFamily": "mbackground", + "resourceClass": "hpc", + "mergeSeconds": { + "samples": 2, + "median": 22.751472198000002, + "minimum": 21.729738135, + "maximum": 23.773206261 + }, + "checkSeconds": { + "samples": 2, + "median": 9.362968396861106, + "minimum": 8.317466377280653, + "maximum": 10.40847041644156 + }, + "copySeconds": { + "samples": 2, + "median": 5.138671131338924, + "minimum": 4.891942439600825, + "maximum": 5.385399823077023 + }, + "verifySeconds": { + "samples": 2, + "median": 7.284067759755999, + "minimum": 7.006542180664837, + "maximum": 7.56159333884716 + } + }, + { + "activityFamily": "mbgmodel", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 2, + "median": 16.8738145285, + "minimum": 16.747609883, + "maximum": 17.000019174 + }, + "checkSeconds": { + "samples": 2, + "median": 3.261755618000052, + "minimum": 2.698455793999983, + "maximum": 3.825055442000121 + }, + "copySeconds": { + "samples": 2, + "median": 9.35986830599984, + "minimum": 9.022188792999941, + "maximum": 9.697547818999738 + }, + "verifySeconds": { + "samples": 2, + "median": 3.6908348800000113, + "minimum": 3.6229606209999474, + "maximum": 3.758709139000075 + } + }, + { + "activityFamily": "mbgmodel", + "resourceClass": "e2-medium", + "mergeSeconds": { + "samples": 1, + "median": 3.603590835, + "minimum": 3.603590835, + "maximum": 3.603590835 + }, + "checkSeconds": { + "samples": 1, + "median": 0.5051906540002165, + "minimum": 0.5051906540002165, + "maximum": 0.5051906540002165 + }, + "copySeconds": { + "samples": 1, + "median": 2.142047676000402, + "minimum": 2.142047676000402, + "maximum": 2.142047676000402 + }, + "verifySeconds": { + "samples": 1, + "median": 0.4960838049992162, + "minimum": 0.4960838049992162, + "maximum": 0.4960838049992162 + } + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "hpc", + "mergeSeconds": { + "samples": 1, + "median": 51.168297815, + "minimum": 51.168297815, + "maximum": 51.168297815 + }, + "checkSeconds": { + "samples": 1, + "median": 29.774229486472905, + "minimum": 29.774229486472905, + "maximum": 29.774229486472905 + }, + "copySeconds": { + "samples": 1, + "median": 10.907653647474945, + "minimum": 10.907653647474945, + "maximum": 10.907653647474945 + }, + "verifySeconds": { + "samples": 1, + "median": 9.319769462570548, + "minimum": 9.319769462570548, + "maximum": 9.319769462570548 + } + }, + { + "activityFamily": "mconcatfit", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 2, + "median": 129.158736309, + "minimum": 114.001598677, + "maximum": 144.315873941 + }, + "checkSeconds": { + "samples": 2, + "median": 65.35185085299986, + "minimum": 64.67154081999979, + "maximum": 66.03216088599993 + }, + "copySeconds": { + "samples": 2, + "median": 33.53280334749991, + "minimum": 26.555731558999923, + "maximum": 40.50987513599989 + }, + "verifySeconds": { + "samples": 2, + "median": 29.360318618000065, + "minimum": 22.22765749600012, + "maximum": 36.49297974000001 + } + }, + { + "activityFamily": "mdifffit", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 14, + "median": 8.3427407115, + "minimum": 5.830948001, + "maximum": 38.347655976 + }, + "checkSeconds": { + "samples": 14, + "median": 2.511359688500079, + "minimum": 1.1038579400000117, + "maximum": 15.438960124000005 + }, + "copySeconds": { + "samples": 14, + "median": 4.124673134000005, + "minimum": 2.378768721999677, + "maximum": 13.515580894999857 + }, + "verifySeconds": { + "samples": 14, + "median": 1.6997197335000465, + "minimum": 1.0764374750000059, + "maximum": 13.448123979999764 + } + }, + { + "activityFamily": "mdifffit", + "resourceClass": "hpc", + "mergeSeconds": { + "samples": 3, + "median": 7.47857792, + "minimum": 6.511636503, + "maximum": 10.95796063 + }, + "checkSeconds": { + "samples": 3, + "median": 1.7308137826621532, + "minimum": 1.710929460823536, + "maximum": 3.6601696284487844 + }, + "copySeconds": { + "samples": 3, + "median": 2.0938274255022407, + "minimum": 1.2903437670320272, + "maximum": 2.483007412403822 + }, + "verifySeconds": { + "samples": 3, + "median": 2.644647597335279, + "minimum": 2.5150117529556155, + "maximum": 2.732031609863043 + } + }, + { + "activityFamily": "mdifffit", + "resourceClass": "e2-medium", + "mergeSeconds": { + "samples": 1, + "median": 3.536212793, + "minimum": 3.536212793, + "maximum": 3.536212793 + }, + "checkSeconds": { + "samples": 1, + "median": 0.7207421489999888, + "minimum": 0.7207421489999888, + "maximum": 0.7207421489999888 + }, + "copySeconds": { + "samples": 1, + "median": 1.7899585740000248, + "minimum": 1.7899585740000248, + "maximum": 1.7899585740000248 + }, + "verifySeconds": { + "samples": 1, + "median": 0.5052870060000032, + "minimum": 0.5052870060000032, + "maximum": 0.5052870060000032 + } + }, + { + "activityFamily": "mimgtbl", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 3, + "median": 53.028995149, + "minimum": 52.357549732, + "maximum": 131.204556309 + }, + "checkSeconds": { + "samples": 3, + "median": 18.724648188000174, + "minimum": 18.695452503000524, + "maximum": 55.52439420599967 + }, + "copySeconds": { + "samples": 3, + "median": 27.288482144000227, + "minimum": 27.0868461150003, + "maximum": 57.700238978000016 + }, + "verifySeconds": { + "samples": 3, + "median": 6.4975390220006375, + "minimum": 6.045547994999652, + "maximum": 16.76125427399984 + } + }, + { + "activityFamily": "mviewer", + "resourceClass": "e2-medium", + "mergeSeconds": { + "samples": 2, + "median": 30.185459264000002, + "minimum": 7.251629086, + "maximum": 53.119289442 + }, + "checkSeconds": { + "samples": 2, + "median": 4.712208465500225, + "minimum": 0.8757597130006616, + "maximum": 8.54865721799979 + }, + "copySeconds": { + "samples": 2, + "median": 17.59759918700047, + "minimum": 5.020064412000465, + "maximum": 30.175133962000473 + }, + "verifySeconds": { + "samples": 2, + "median": 7.402032238000174, + "minimum": 0.8792642919997888, + "maximum": 13.92480018400056 + } + }, + { + "activityFamily": "mviewer", + "resourceClass": "e2-micro", + "mergeSeconds": { + "samples": 2, + "median": 42.396445477, + "minimum": 30.327887263, + "maximum": 54.465003691 + }, + "checkSeconds": { + "samples": 2, + "median": 7.072915739999644, + "minimum": 6.825928993999696, + "maximum": 7.319902485999592 + }, + "copySeconds": { + "samples": 2, + "median": 16.937191215999974, + "minimum": 16.770350947000225, + "maximum": 17.104031484999723 + }, + "verifySeconds": { + "samples": 2, + "median": 17.893067211000016, + "minimum": 6.250859729000695, + "maximum": 29.535274692999337 + } + } + ], + "mergeRateObservations": [ + { + "targetClass": "e2-micro", + "source": "baseline:montage-docker-image-real-20261010-v2", + "observations": { + "checkSecondsPerInputByte": { + "samples": 34, + "median": 6.35527429607893e-09, + "minimum": 2.8968348057689955e-09, + "maximum": 7.236672806194324e-08 + }, + "copySecondsPerUniqueByte": { + "samples": 34, + "median": 1.699528122472209e-08, + "minimum": 7.844377436225745e-09, + "maximum": 6.452023672950758e-08 + }, + "verifySecondsPerUniqueByte": { + "samples": 34, + "median": 6.138832298139974e-09, + "minimum": 3.5569571742792556e-09, + "maximum": 7.073291666499625e-08 + } + }, + "commandOverheadSeconds": null, + "channelWaitSeconds": null + }, + { + "targetClass": "e2-medium", + "source": "baseline:montage-docker-image-real-20261010-v2", + "observations": { + "checkSecondsPerInputByte": { + "samples": 6, + "median": 1.912526350228284e-09, + "minimum": 7.876752557337657e-10, + "maximum": 2.5742072137899076e-09 + }, + "copySecondsPerUniqueByte": { + "samples": 6, + "median": 4.454356662161575e-09, + "minimum": 2.3675786107344793e-09, + "maximum": 9.610568101437293e-09 + }, + "verifySecondsPerUniqueByte": { + "samples": 6, + "median": 8.04424495753747e-10, + "minimum": 7.83635464283875e-10, + "maximum": 4.434950997591924e-09 + } + }, + "commandOverheadSeconds": null, + "channelWaitSeconds": null + }, + { + "targetClass": "hpc", + "source": "baseline:montage-docker-image-real-20261010-v2", + "observations": { + "checkSecondsPerInputByte": { + "samples": 6, + "median": 9.649068875587286e-09, + "minimum": 5.653559042397685e-09, + "maximum": 1.5362090273490482e-08 + }, + "copySecondsPerUniqueByte": { + "samples": 6, + "median": 7.94870229763006e-09, + "minimum": 6.085461673088312e-09, + "maximum": 1.718957664580683e-08 + }, + "verifySecondsPerUniqueByte": { + "samples": 6, + "median": 1.1464509411470224e-08, + "minimum": 8.721156347585568e-09, + "maximum": 1.4687200077643863e-08 + } + }, + "commandOverheadSeconds": null, + "channelWaitSeconds": null + } + ], + "limitations": [ + "Single baseline; ranges are observations, not confidence intervals.", + "Do not replay per-activity durations or total makespan as a holdout prediction.", + "No destination cache hit or shared-storage route may be assumed without evidence.", + "Workspace manifests and topology-specific contention are required before applying these observations.", + "Compute samples are contention-conditioned; do not apply interference slowdown a second time." + ] +} diff --git a/internal/montagetools/testdata/compute.json b/internal/montagetools/testdata/compute.json new file mode 100644 index 00000000..92cb33d0 --- /dev/null +++ b/internal/montagetools/testdata/compute.json @@ -0,0 +1,1785 @@ +{ + "schemaVersion": 1, + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "activityFamilyTypeIds": { + "madd": "migration-fixture-madd", + "mbackground": "migration-fixture-mbackground", + "mbgmodel": "migration-fixture-mbgmodel", + "mconcatfit": "migration-fixture-mconcatfit", + "mdifffit": "migration-fixture-mdifffit", + "mimgtbl": "migration-fixture-mimgtbl", + "mproject": "migration-fixture-mproject", + "mviewer": "migration-fixture-mviewer" + }, + "activityProfiles": [ + { + "id": "migration-fixture-madd-montage-gcp-fog-e2-micro-01", + "activityTypeId": "migration-fixture-madd", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-madd-montage-gcp-fog-e2-micro-02", + "activityTypeId": "migration-fixture-madd", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-madd-montage-gcp-fog-e2-micro-03", + "activityTypeId": "migration-fixture-madd", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-madd-montage-gcp-fog-e2-micro-04", + "activityTypeId": "migration-fixture-madd", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-madd-montage-gcp-fog-e2-micro-05", + "activityTypeId": "migration-fixture-madd", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-madd-montage-gcp-fog-e2-micro-06", + "activityTypeId": "migration-fixture-madd", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-madd-montage-gcp-fog-e2-micro-07", + "activityTypeId": "migration-fixture-madd", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-madd-montage-gcp-fog-e2-micro-08", + "activityTypeId": "migration-fixture-madd", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-madd-montage-gcp-fog-e2-micro-09", + "activityTypeId": "migration-fixture-madd", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-madd-montage-gcp-fog-e2-micro-10", + "activityTypeId": "migration-fixture-madd", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 0.9374836550818549, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.10642316606309679, + 4.108689546585083 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbackground-montage-gcp-e2-medium-1", + "activityTypeId": "migration-fixture-mbackground", + "resourceId": "montage-gcp-e2-medium-1", + "runtimeSeconds": 0.32499454332434613, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02318926479505456, + 0.6267998218536377 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbackground-montage-gcp-e2-medium-2", + "activityTypeId": "migration-fixture-mbackground", + "resourceId": "montage-gcp-e2-medium-2", + "runtimeSeconds": 0.32499454332434613, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02318926479505456, + 0.6267998218536377 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbackground-montage-gcp-fog-e2-micro-01", + "activityTypeId": "migration-fixture-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbackground-montage-gcp-fog-e2-micro-02", + "activityTypeId": "migration-fixture-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbackground-montage-gcp-fog-e2-micro-03", + "activityTypeId": "migration-fixture-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbackground-montage-gcp-fog-e2-micro-04", + "activityTypeId": "migration-fixture-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbackground-montage-gcp-fog-e2-micro-05", + "activityTypeId": "migration-fixture-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbackground-montage-gcp-fog-e2-micro-06", + "activityTypeId": "migration-fixture-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbackground-montage-gcp-fog-e2-micro-07", + "activityTypeId": "migration-fixture-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbackground-montage-gcp-fog-e2-micro-08", + "activityTypeId": "migration-fixture-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbackground-montage-gcp-fog-e2-micro-09", + "activityTypeId": "migration-fixture-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbackground-montage-gcp-fog-e2-micro-10", + "activityTypeId": "migration-fixture-mbackground", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 0.13403182718488904, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 8, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.09645940860112508, + 13.19267725944519 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbackground-plafrim-execution-ssh-node-bora001", + "activityTypeId": "migration-fixture-mbackground", + "resourceId": "plafrim-execution-ssh-node-bora001", + "runtimeSeconds": 2.042281486093998, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.1621987372636795, + 3.9223642349243164 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 2 + } + }, + { + "id": "migration-fixture-mbackground-plafrim-execution-ssh-node-diablo03", + "activityTypeId": "migration-fixture-mbackground", + "resourceId": "plafrim-execution-ssh-node-diablo03", + "runtimeSeconds": 2.042281486093998, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.1621987372636795, + 3.9223642349243164 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 2 + } + }, + { + "id": "migration-fixture-mbgmodel-montage-gcp-fog-e2-micro-01", + "activityTypeId": "migration-fixture-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbgmodel-montage-gcp-fog-e2-micro-02", + "activityTypeId": "migration-fixture-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbgmodel-montage-gcp-fog-e2-micro-03", + "activityTypeId": "migration-fixture-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbgmodel-montage-gcp-fog-e2-micro-04", + "activityTypeId": "migration-fixture-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbgmodel-montage-gcp-fog-e2-micro-05", + "activityTypeId": "migration-fixture-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbgmodel-montage-gcp-fog-e2-micro-06", + "activityTypeId": "migration-fixture-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbgmodel-montage-gcp-fog-e2-micro-07", + "activityTypeId": "migration-fixture-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbgmodel-montage-gcp-fog-e2-micro-08", + "activityTypeId": "migration-fixture-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbgmodel-montage-gcp-fog-e2-micro-09", + "activityTypeId": "migration-fixture-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbgmodel-montage-gcp-fog-e2-micro-10", + "activityTypeId": "migration-fixture-mbgmodel", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 0.07053212135557144, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.05229279730055067, + 0.08877144541059222 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbgmodel-montage-gcp-e2-medium-1", + "activityTypeId": "migration-fixture-mbgmodel", + "resourceId": "montage-gcp-e2-medium-1", + "runtimeSeconds": 0.025992941856384278, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 1, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.025992941856384278, + 0.025992941856384278 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mbgmodel-montage-gcp-e2-medium-2", + "activityTypeId": "migration-fixture-mbgmodel", + "resourceId": "montage-gcp-e2-medium-2", + "runtimeSeconds": 0.025992941856384278, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 1, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.025992941856384278, + 0.025992941856384278 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mconcatfit-plafrim-execution-ssh-node-bora001", + "activityTypeId": "migration-fixture-mconcatfit", + "resourceId": "plafrim-execution-ssh-node-bora001", + "runtimeSeconds": 0.16663997313555548, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 1, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.16663997313555548, + 0.16663997313555548 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 1 + } + }, + { + "id": "migration-fixture-mconcatfit-plafrim-execution-ssh-node-diablo03", + "activityTypeId": "migration-fixture-mconcatfit", + "resourceId": "plafrim-execution-ssh-node-diablo03", + "runtimeSeconds": 0.16663997313555548, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 1, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.16663997313555548, + 0.16663997313555548 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 1 + } + }, + { + "id": "migration-fixture-mconcatfit-montage-gcp-fog-e2-micro-01", + "activityTypeId": "migration-fixture-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mconcatfit-montage-gcp-fog-e2-micro-02", + "activityTypeId": "migration-fixture-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mconcatfit-montage-gcp-fog-e2-micro-03", + "activityTypeId": "migration-fixture-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mconcatfit-montage-gcp-fog-e2-micro-04", + "activityTypeId": "migration-fixture-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mconcatfit-montage-gcp-fog-e2-micro-05", + "activityTypeId": "migration-fixture-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mconcatfit-montage-gcp-fog-e2-micro-06", + "activityTypeId": "migration-fixture-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mconcatfit-montage-gcp-fog-e2-micro-07", + "activityTypeId": "migration-fixture-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mconcatfit-montage-gcp-fog-e2-micro-08", + "activityTypeId": "migration-fixture-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mconcatfit-montage-gcp-fog-e2-micro-09", + "activityTypeId": "migration-fixture-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mconcatfit-montage-gcp-fog-e2-micro-10", + "activityTypeId": "migration-fixture-mconcatfit", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 0.5174553818172878, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.5118624687194824, + 0.5230482949150933 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mdifffit-montage-gcp-fog-e2-micro-01", + "activityTypeId": "migration-fixture-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mdifffit-montage-gcp-fog-e2-micro-02", + "activityTypeId": "migration-fixture-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mdifffit-montage-gcp-fog-e2-micro-03", + "activityTypeId": "migration-fixture-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mdifffit-montage-gcp-fog-e2-micro-04", + "activityTypeId": "migration-fixture-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mdifffit-montage-gcp-fog-e2-micro-05", + "activityTypeId": "migration-fixture-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mdifffit-montage-gcp-fog-e2-micro-06", + "activityTypeId": "migration-fixture-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mdifffit-montage-gcp-fog-e2-micro-07", + "activityTypeId": "migration-fixture-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mdifffit-montage-gcp-fog-e2-micro-08", + "activityTypeId": "migration-fixture-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mdifffit-montage-gcp-fog-e2-micro-09", + "activityTypeId": "migration-fixture-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mdifffit-montage-gcp-fog-e2-micro-10", + "activityTypeId": "migration-fixture-mdifffit", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 0.5160782317320506, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 14, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.02842045823733012, + 2.6107197046279906 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mdifffit-plafrim-execution-ssh-node-bora001", + "activityTypeId": "migration-fixture-mdifffit", + "resourceId": "plafrim-execution-ssh-node-bora001", + "runtimeSeconds": 0.15401040880303635, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.08495262265205383, + 0.20197521556507458 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 3 + } + }, + { + "id": "migration-fixture-mdifffit-plafrim-execution-ssh-node-diablo03", + "activityTypeId": "migration-fixture-mdifffit", + "resourceId": "plafrim-execution-ssh-node-diablo03", + "runtimeSeconds": 0.15401040880303635, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.08495262265205383, + 0.20197521556507458 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 3 + } + }, + { + "id": "migration-fixture-mdifffit-montage-gcp-e2-medium-1", + "activityTypeId": "migration-fixture-mdifffit", + "resourceId": "montage-gcp-e2-medium-1", + "runtimeSeconds": 0.3221574563246507, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 1, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.3221574563246507, + 0.3221574563246507 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mdifffit-montage-gcp-e2-medium-2", + "activityTypeId": "migration-fixture-mdifffit", + "resourceId": "montage-gcp-e2-medium-2", + "runtimeSeconds": 0.3221574563246507, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 1, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.3221574563246507, + 0.3221574563246507 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mimgtbl-montage-gcp-fog-e2-micro-01", + "activityTypeId": "migration-fixture-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mimgtbl-montage-gcp-fog-e2-micro-02", + "activityTypeId": "migration-fixture-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mimgtbl-montage-gcp-fog-e2-micro-03", + "activityTypeId": "migration-fixture-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mimgtbl-montage-gcp-fog-e2-micro-04", + "activityTypeId": "migration-fixture-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mimgtbl-montage-gcp-fog-e2-micro-05", + "activityTypeId": "migration-fixture-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mimgtbl-montage-gcp-fog-e2-micro-06", + "activityTypeId": "migration-fixture-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mimgtbl-montage-gcp-fog-e2-micro-07", + "activityTypeId": "migration-fixture-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mimgtbl-montage-gcp-fog-e2-micro-08", + "activityTypeId": "migration-fixture-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mimgtbl-montage-gcp-fog-e2-micro-09", + "activityTypeId": "migration-fixture-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mimgtbl-montage-gcp-fog-e2-micro-10", + "activityTypeId": "migration-fixture-mimgtbl", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 0.06577382304451683, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 3, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.01489914788140191, + 0.0769037914276123 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mproject-montage-gcp-e2-medium-1", + "activityTypeId": "migration-fixture-mproject", + "resourceId": "montage-gcp-e2-medium-1", + "runtimeSeconds": 0.5422618222257024, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 4, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.482687460737569, + 0.6058569879823626 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mproject-montage-gcp-e2-medium-2", + "activityTypeId": "migration-fixture-mproject", + "resourceId": "montage-gcp-e2-medium-2", + "runtimeSeconds": 0.5422618222257024, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 4, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.482687460737569, + 0.6058569879823626 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mproject-montage-gcp-fog-e2-micro-01", + "activityTypeId": "migration-fixture-mproject", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mproject-montage-gcp-fog-e2-micro-02", + "activityTypeId": "migration-fixture-mproject", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mproject-montage-gcp-fog-e2-micro-03", + "activityTypeId": "migration-fixture-mproject", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mproject-montage-gcp-fog-e2-micro-04", + "activityTypeId": "migration-fixture-mproject", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mproject-montage-gcp-fog-e2-micro-05", + "activityTypeId": "migration-fixture-mproject", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mproject-montage-gcp-fog-e2-micro-06", + "activityTypeId": "migration-fixture-mproject", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mproject-montage-gcp-fog-e2-micro-07", + "activityTypeId": "migration-fixture-mproject", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mproject-montage-gcp-fog-e2-micro-08", + "activityTypeId": "migration-fixture-mproject", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mproject-montage-gcp-fog-e2-micro-09", + "activityTypeId": "migration-fixture-mproject", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mproject-montage-gcp-fog-e2-micro-10", + "activityTypeId": "migration-fixture-mproject", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 4.142489493875232, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 6, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 2.4189968622063263, + 7.97978781444439 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mproject-plafrim-execution-ssh-node-bora001", + "activityTypeId": "migration-fixture-mproject", + "resourceId": "plafrim-execution-ssh-node-bora001", + "runtimeSeconds": 0.280406867515572, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.2784816492421198, + 0.2823320857890241 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 2 + } + }, + { + "id": "migration-fixture-mproject-plafrim-execution-ssh-node-diablo03", + "activityTypeId": "migration-fixture-mproject", + "resourceId": "plafrim-execution-ssh-node-diablo03", + "runtimeSeconds": 0.280406867515572, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.2784816492421198, + 0.2823320857890241 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 2 + } + }, + { + "id": "migration-fixture-mviewer-montage-gcp-e2-medium-1", + "activityTypeId": "migration-fixture-mviewer", + "resourceId": "montage-gcp-e2-medium-1", + "runtimeSeconds": 0.24281440161857284, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.07336857739616842, + 0.41226022584097727 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mviewer-montage-gcp-e2-medium-2", + "activityTypeId": "migration-fixture-mviewer", + "resourceId": "montage-gcp-e2-medium-2", + "runtimeSeconds": 0.24281440161857284, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.07336857739616842, + 0.41226022584097727 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mviewer-montage-gcp-fog-e2-micro-01", + "activityTypeId": "migration-fixture-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-01", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mviewer-montage-gcp-fog-e2-micro-02", + "activityTypeId": "migration-fixture-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-02", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mviewer-montage-gcp-fog-e2-micro-03", + "activityTypeId": "migration-fixture-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-03", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mviewer-montage-gcp-fog-e2-micro-04", + "activityTypeId": "migration-fixture-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-04", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mviewer-montage-gcp-fog-e2-micro-05", + "activityTypeId": "migration-fixture-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-05", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mviewer-montage-gcp-fog-e2-micro-06", + "activityTypeId": "migration-fixture-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-06", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mviewer-montage-gcp-fog-e2-micro-07", + "activityTypeId": "migration-fixture-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-07", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mviewer-montage-gcp-fog-e2-micro-08", + "activityTypeId": "migration-fixture-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-08", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mviewer-montage-gcp-fog-e2-micro-09", + "activityTypeId": "migration-fixture-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-09", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + }, + { + "id": "migration-fixture-mviewer-montage-gcp-fog-e2-micro-10", + "activityTypeId": "migration-fixture-mviewer", + "resourceId": "montage-gcp-fog-e2-micro-10", + "runtimeSeconds": 13.308445498347282, + "modelVersion": "reference-work-v1", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "sampleSize": 2, + "metadata": { + "units": "wall-seconds-per-reference-second", + "observedRange": [ + 0.20626267790794373, + 26.41062831878662 + ], + "includesPhysicalSharing": true, + "taskRuntimeFallbackSamples": 0 + } + } + ], + "limitations": [ + "Class-level historical wall rates; no per-activity replay.", + "Unobserved family/class pairs have no measured profile.", + "Includes physical sharing: additional interference factors require separate evidence." + ] +} diff --git a/internal/montagetools/testdata/materialization.json b/internal/montagetools/testdata/materialization.json new file mode 100644 index 00000000..de16069d --- /dev/null +++ b/internal/montagetools/testdata/materialization.json @@ -0,0 +1,182 @@ +{ + "schemaVersion": 1, + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "relayResourceId": "montage-relay-controller", + "resources": { + "montage-relay-controller": { + "class": "relay", + "domain": "relay" + }, + "montage-gcp-e2-medium-1": { + "class": "e2-medium", + "domain": "cloud" + }, + "montage-gcp-e2-medium-2": { + "class": "e2-medium", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-01": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-02": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-03": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-04": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-05": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-06": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-07": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-08": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-09": { + "class": "e2-micro", + "domain": "cloud" + }, + "montage-gcp-fog-e2-micro-10": { + "class": "e2-micro", + "domain": "cloud" + }, + "plafrim-execution-ssh-node-bora001": { + "class": "hpc", + "domain": "hpc" + }, + "plafrim-execution-ssh-node-diablo03": { + "class": "hpc", + "domain": "hpc" + } + }, + "profiles": [ + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-micro", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "samples": 20, + "archiveToWorkspaceRatio": 0.7256941966584066, + "uploadArchiveBytesPerSecond": 23050729.775595132, + "probeSeconds": 2.4162210225000003, + "snapshotSecondsPerByte": 2.6154549488643507e-08, + "receiverSecondsPerByte": 9.357275158963007e-08 + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-micro", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "samples": 45, + "archiveToWorkspaceRatio": 0.578327721631775, + "uploadArchiveBytesPerSecond": 10611098.204649296, + "probeSeconds": 2.444578959, + "snapshotSecondsPerByte": 2.0032458250178902e-07, + "receiverSecondsPerByte": 9.084165001243439e-08 + }, + { + "strategy": "gateway", + "sourceClass": "hpc", + "targetClass": "e2-micro", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "samples": 11, + "archiveToWorkspaceRatio": 0.5736604862310096, + "uploadArchiveBytesPerSecond": 6771283.566111343, + "ingressArchiveBytesPerSecond": 4865384.828445642, + "ingressIncludesSourcePreparation": true, + "validationSecondsPerByte": 5.315855248831829e-09, + "receiverSecondsPerByte": 1.1075930311647432e-07 + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-micro", + "targetClass": "e2-medium", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "samples": 7, + "archiveToWorkspaceRatio": 0.5616915015784198, + "uploadArchiveBytesPerSecond": 12913048.979721189, + "probeSeconds": 3.238551043, + "snapshotSecondsPerByte": 2.1656552738215494e-07, + "receiverSecondsPerByte": 1.0452033444602787e-08 + }, + { + "strategy": "gateway", + "sourceClass": "e2-micro", + "targetClass": "hpc", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "samples": 10, + "archiveToWorkspaceRatio": 0.6522022259028002, + "uploadArchiveBytesPerSecond": 2091245.5079801935, + "ingressArchiveBytesPerSecond": 3345916.154261665, + "ingressIncludesSourcePreparation": true, + "validationSecondsPerByte": 5.530047221003427e-09, + "receiverSecondsPerByte": 2.158963850339391e-08 + }, + { + "strategy": "gateway", + "sourceClass": "e2-medium", + "targetClass": "hpc", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "samples": 1, + "archiveToWorkspaceRatio": 0.5811239069429878, + "uploadArchiveBytesPerSecond": 3368453.2311919224, + "ingressArchiveBytesPerSecond": 12094092.529896256, + "ingressIncludesSourcePreparation": true, + "validationSecondsPerByte": 5.291803775302269e-09, + "receiverSecondsPerByte": 1.631557426433729e-08 + }, + { + "strategy": "direct-runtime", + "sourceClass": "e2-medium", + "targetClass": "e2-medium", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "samples": 1, + "archiveToWorkspaceRatio": 0.7333948473912417, + "uploadArchiveBytesPerSecond": 24657022.312403273, + "probeSeconds": 4.352586419, + "snapshotSecondsPerByte": 7.94229994679085e-08, + "receiverSecondsPerByte": 3.0313912528566057e-08 + } + ], + "mergeProfiles": [ + { + "targetClass": "e2-micro", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "samples": 34, + "checkSecondsPerInputByte": 6.35527429607893e-09, + "copySecondsPerUniqueByte": 1.699528122472209e-08, + "verifySecondsPerUniqueByte": 6.138832298139974e-09 + }, + { + "targetClass": "e2-medium", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "samples": 6, + "checkSecondsPerInputByte": 1.912526350228284e-09, + "copySecondsPerUniqueByte": 4.454356662161575e-09, + "verifySecondsPerUniqueByte": 8.04424495753747e-10 + }, + { + "targetClass": "hpc", + "source": "montage-docker-image-real-20261010-v2:sha256:migration-fixture-digest", + "samples": 6, + "checkSecondsPerInputByte": 9.649068875587286e-09, + "copySecondsPerUniqueByte": 7.94870229763006e-09, + "verifySecondsPerUniqueByte": 1.1464509411470224e-08 + } + ] +} diff --git a/internal/montagetools/tools_test.go b/internal/montagetools/tools_test.go new file mode 100644 index 00000000..c0cb2f37 --- /dev/null +++ b/internal/montagetools/tools_test.go @@ -0,0 +1,405 @@ +package montagetools + +import ( + "fmt" + "github.com/stretchr/testify/require" + "math" + "os" + "path/filepath" + "strings" + "testing" +) + +func scientific(t *testing.T, name string) Object { + t.Helper() + m, _ := read(filepath.Join("..", "..", "experiments", "montage-micro-hpc-e2", name)) + return m +} +func bad(t *testing.T, fn func() Object) { t.Helper(); _, err := attempt(fn); require.Error(t, err) } +func TestGoldenMigrationParity(t *testing.T) { + a := scientific(t, "docker-image-real-v2-results.json") + p := scientific(t, "workspace-projection-v2.json") + c := calibration(a, str(p["auditedEvidenceSHA256"]), p) + want, _ := read("testdata/calibration.json") + equivalent(t, normal(want), normal(c)) + resources := scientific(t, "materialization-resource-model.json") + m := freezeMaterialization(c, "migration-fixture-digest", resources, "montage-relay-controller") + want, _ = read("testdata/materialization.json") + equivalent(t, normal(want), normal(m)) + compute := freezeCompute(c, "migration-fixture-digest", resources, "migration-fixture") + want, _ = read("testdata/compute.json") + sortScientificRows(compute, "activityProfiles", "id") + sortScientificRows(want, "activityProfiles", "id") + equivalent(t, normal(want), normal(compute)) +} +func equivalent(t *testing.T, want, got any) { + t.Helper() + switch w := want.(type) { + case float64: + g, ok := got.(float64) + require.True(t, ok) + require.InDelta(t, w, g, math.Max(1, math.Abs(w))*1e-12) + case map[string]any: + g, ok := got.(map[string]any) + require.True(t, ok) + require.Len(t, g, len(w)) + for k, v := range w { + require.Contains(t, g, k) + equivalent(t, v, g[k]) + } + case []any: + g, ok := got.([]any) + require.True(t, ok) + require.Len(t, g, len(w)) + for i, v := range w { + equivalent(t, v, g[i]) + } + default: + require.Equal(t, want, got) + } +} +func sortScientificRows(m Object, key, id string) { + rs := rows(m[key]) + sortRows(rs, id) + out := []any{} + for _, r := range rs { + out = append(out, r) + } + m[key] = out +} +func TestRootComputeSupplement(t *testing.T) { + c := Object{"computeWorkBasis": "immutable-dataset-independent-work-v1", "computeObservations": []any{}} + p := Object{"activityComputeWork": Object{}} + request := Object{"workflow": Object{"activities": []any{}}, "plan": Object{"assignments": []any{}}} + response := Object{"run": Object{"id": "training", "status": "failed"}, "activityAttempts": []any{}, "handles": []any{}} + for i := 0; i < 12; i++ { + id := fmt.Sprintf("mprojectid%07d", i) + obj(p["activityComputeWork"])[id] = Object{"activityName": id, "activityFamily": "mproject_red", "computeWorkUnits": 6.0} + w := obj(request["workflow"]) + w["activities"] = append(arr(w["activities"]), Object{"id": id, "name": id}) + plan := obj(request["plan"]) + plan["assignments"] = append(arr(plan["assignments"]), Object{"activityId": id, "resourceId": "plafrim-bora"}) + response["activityAttempts"] = append(arr(response["activityAttempts"]), Object{"activityId": id, "attempt": 1, "status": "completed", "runtimeSeconds": 300.0}) + response["handles"] = append(arr(response["handles"]), Object{"id": "training:" + id, "activityId": id, "status": "completed", "exitCode": 0, "artifacts": Object{"finalSnapshot": []any{Object{}, Object{}}}}) + } + r := supplementRoots(c, p, request, response, "digest") + row := rows(r["computeObservations"])[0] + require.Equal(t, list("plafrim-bora"), row["allowedResourceIds"]) + require.Equal(t, 50.0, obj(row["wallSecondsPerReferenceSecond"])["median"]) + require.Empty(t, c["computeObservations"]) + rows(response["handles"])[0]["exitCode"] = nil + bad(t, func() Object { return supplementRoots(c, p, request, response, "digest") }) +} +func TestCalibrationSafetyAndProvenance(t *testing.T) { + for _, name := range []string{"barrier_violation_count", "unreleased_lease_count", "cross_cloud_hpc_non_relay_count"} { + t.Run(name, func(t *testing.T) { + a := scientific(t, "docker-image-real-v2-results.json") + obj(a["audit"])[name] = 1 + bad(t, func() Object { return calibration(a, "d", nil) }) + }) + } + a := scientific(t, "docker-image-real-v2-results.json") + arr(a["activities"])[1] = arr(a["activities"])[0] + bad(t, func() Object { return calibration(a, "d", nil) }) + a = scientific(t, "docker-image-real-v2-results.json") + p := scientific(t, "workspace-projection-v2.json") + p["baselineRunId"] = "other" + bad(t, func() Object { return calibration(a, str(p["auditedEvidenceSHA256"]), p) }) +} +func TestSummaryAndPooledService(t *testing.T) { + require.Equal(t, normal(Object{"samples": 4, "median": 2.5, "minimum": 1, "maximum": 4}), normal(summary([]any{4, 2, 1, 3}))) + require.Nil(t, summary([]any{nil})) + for _, v := range []float64{math.Inf(1), math.NaN(), -1} { + bad(t, func() Object { return Object{"summary": summary([]any{v})} }) + } + rate := obj(pooledRate([]Object{{"archiveBytes": 100, "seconds": 10}, {"archiveBytes": 1000, "seconds": 20}, {"archiveBytes": nil, "seconds": 999}}, "archiveBytes")) + require.Equal(t, 2, rate["samples"]) + require.InDelta(t, 1100.0/30, num(rate["bytesPerSecond"]), 1e-12) + require.Nil(t, pooledRate(nil, "archiveBytes")) +} +func TestQueueSeparation(t *testing.T) { + for _, mode := range []string{"pack", "upload", "receiver"} { + t.Run(mode, func(t *testing.T) { + a := scientific(t, "docker-image-real-v2-results.json") + for _, r := range rows(a["routes"]) { + switch mode { + case "pack": + if r["strategy"] == "gateway" { + r["source_pack_execution_seconds"] = 10 + r["source_pack_ssh_channel_wait_seconds"] = 20 + } + case "upload": + if r["strategy"] == "gateway" { + r["rsync_execution_seconds"] = 7 + r["ssh_channel_wait_seconds"] = 19 + } + case "receiver": + for _, p := range rows(r["phases"]) { + if strings.HasSuffix(str(p["name"]), "receiver-verify-and-extract") { + p["executionSeconds"] = 3 + p["sshChannelWaitSeconds"] = 12 + p["durationSeconds"] = 15 + } + } + } + } + c := calibration(a, "d", nil) + seen := 0 + for _, p := range rows(c["materializationPhaseObservations"]) { + if mode == "pack" && p["phase"] == "pack-and-receive" { + require.Equal(t, 10.0, num(obj(p["seconds"])["median"])) + require.Equal(t, 20.0, num(obj(p["sourceChannelWaitSeconds"])["median"])) + seen++ + } + if mode == "upload" && p["phase"] == "archive-upload" { + require.Equal(t, 7.0, num(obj(p["seconds"])["median"])) + require.Equal(t, 19.0, num(obj(p["phaseChannelWaitSeconds"])["median"])) + seen++ + } + if mode == "receiver" && strings.HasSuffix(str(p["phase"]), "receiver-verify-and-extract") { + require.Equal(t, 3.0, num(obj(p["seconds"])["median"])) + require.Equal(t, 12.0, num(obj(p["phaseChannelWaitSeconds"])["median"])) + seen++ + } + } + require.Positive(t, seen) + }) + } +} +func TestFreezeRejectsUnmeasuredAndMixedQueue(t *testing.T) { + resources := Object{ + "relay": Object{"domain": "relay", "class": "relay"}, + "hpc": Object{"domain": "hpc", "class": "hpc", "bulkServiceGroupId": "group", "bulkServiceSlots": 3, "bulkServiceSource": "fixture"}, + "micro": Object{"domain": "cloud", "class": "e2-micro"}, + } + c := scientific(t, "execution-calibration-v2.json") + bad(t, func() Object { return freezeMaterialization(c, "d", resources, "relay") }) + for _, p := range rows(c["materializationPhaseObservations"]) { + if p["sourceClass"] == "hpc" && p["phase"] == "pack-and-receive" { + p["secondsBases"] = list("source-command-execution-excludes-channel-queue") + p["sourcePreflightServiceSeconds"] = Object{"samples": 2, "median": 0.4} + } + if p["targetClass"] == "hpc" && strings.HasSuffix(str(p["phase"]), "receiver-verify-and-extract") { + p["secondsBases"] = list("phase-command-execution-excludes-channel-queue") + } + } + m := freezeMaterialization(c, "d", resources, "relay") + require.Len(t, rows(m["profiles"]), 7) + for _, p := range rows(c["materializationPhaseObservations"]) { + if p["sourceClass"] == "hpc" && p["phase"] == "pack-and-receive" { + p["secondsBases"] = append(arr(p["secondsBases"]), "combined-phase-wall-includes-unseparated-service-queue") + break + } + } + bad(t, func() Object { return freezeMaterialization(c, "d", resources, "relay") }) + c["workspaceVolumeBasis"] = "output-only" + bad(t, func() Object { return freezeMaterialization(c, "d", resources, "relay") }) +} +func TestPooledRateAndMergeOverhead(t *testing.T) { + r := Object{"pooledArchiveServiceRate": Object{"samples": 2, "bytesPerSecond": 36.5}, "legacy": Object{"samples": 2, "median": 999}} + require.Equal(t, 36.5, serviceRate(r, "pooledArchiveServiceRate", "legacy")) + delete(r, "pooledArchiveServiceRate") + require.Equal(t, 999.0, serviceRate(r, "pooledArchiveServiceRate", "legacy")) + r["pooledArchiveServiceRate"] = Object{"samples": 2, "bytesPerSecond": 0} + bad(t, func() Object { return Object{"rate": serviceRate(r, "pooledArchiveServiceRate", "legacy")} }) + c := scientific(t, "execution-calibration-v2.json") + for _, m := range rows(c["mergeRateObservations"]) { + m["commandOverheadSeconds"] = Object{"samples": 3, "median": 2.5} + m["channelWaitSeconds"] = Object{"samples": 3, "median": 30} + } + result := freezeMaterialization(c, "d", scientific(t, "materialization-resource-model.json"), "montage-relay-controller") + for _, m := range rows(result["mergeProfiles"]) { + require.Equal(t, 2.5, m["commandOverheadSeconds"]) + require.NotContains(t, m, "channelWaitSeconds") + } +} +func TestWorkspaceFilesAndPackaging(t *testing.T) { + for _, p := range []string{"/absolute", "a/../b", "a\\b", "a\x00b"} { + bad(t, func() Object { + return files([]any{Object{"path": p, "sizeBytes": 1, "checksum": "sha256:" + strings.Repeat("a", 64)}}) + }) + } + bad(t, func() Object { return files([]any{Object{"path": "safe", "sizeBytes": 1, "checksum": "bad"}}) }) + seed := Object{"path": "runtime.hdr", "sizeBytes": 10, "contentId": "immutable"} + input := Object{"path": "parent.fits", "sizeBytes": 100, "contentId": "parent"} + output := Object{"path": "difference.fits", "sizeBytes": 50, "contentId": "output"} + m := activityModel(Object{"parent.fits": input}, Object{"parent.fits": input, "runtime.hdr": seed, "difference.fits": output}, Object{"runtime.hdr": seed}, "audited") + require.Equal(t, []any{output}, m["outputs"]) + require.Empty(t, obj(m["inputsByResourceType"])["hpc_machine"]) + changed := Object{"path": "runtime.hdr", "sizeBytes": 20, "contentId": "changed"} + m = activityModel(Object{}, Object{"runtime.hdr": changed}, Object{"runtime.hdr": seed}, "audited") + require.Equal(t, []any{changed}, m["outputs"]) + m = activityModel(Object{"runtime.hdr": seed}, Object{"runtime.hdr": seed}, Object{"runtime.hdr": seed}, "audited") + require.Empty(t, m["outputs"]) + require.Empty(t, m["removedPaths"]) + bad(t, func() Object { return unionFiles([]Object{{"runtime.hdr": seed}, {"runtime.hdr": changed}}) }) +} +func TestInputWorkAndHeaders(t *testing.T) { + cards := "" + for _, c := range []string{"SIMPLE = T", "BITPIX = 16", "NAXIS = 2", "NAXIS1 = 2531", "NAXIS2 = 2536", "END"} { + cards += fmt.Sprintf("%-80s", c) + } + feature := inputFeature(fitsHeader(strings.NewReader(cards))) + require.InDelta(t, 6.418616, num(feature["computeWorkUnits"]), 1e-12) + require.Equal(t, 16, feature["bitpix"]) + bad(t, func() Object { return fitsHeader(strings.NewReader("short")) }) + bad(t, func() Object { return inputFeature(Object{"NAXIS": 3}) }) + bad(t, func() Object { return inputFeature(Object{"NAXIS": 2, "NAXIS1": 0, "NAXIS2": 1}) }) + a := Object{"id": "workflow-mprojectid0000001", "name": "mprojectid0000001", "command": Object{"arguments": list("-c", "mProject -X 'input.fits' output.fits region.hdr")}, "simulation": Object{"durationSeconds": 999}} + request := Object{"workflow": Object{"activities": []any{a}, "dependencies": []any{}}} + inputs := Object{"sourceImageSHA256": "source", "inputs": []any{Object{"path": "input.fits", "band": "red", "unit": "input-megapixels", "computeWorkUnits": 6.0}}} + work := workFeatures(request, inputs) + obj(a["simulation"])["durationSeconds"] = 1 + require.Equal(t, work, workFeatures(request, inputs)) + obj(a["command"])["arguments"] = list("-c", "mProject missing.fits out hdr") + bad(t, func() Object { return workFeatures(request, inputs) }) +} +func auditFixture() (Object, Object, Object) { + activities, assignments, tasks := []any{}, []any{}, []any{} + for i := 1; i <= 58; i++ { + id := fmt.Sprintf("activity-%d", i) + activities = append(activities, Object{"id": id}) + assignments = append(assignments, Object{"activityId": id, "resourceId": "micro"}) + tasks = append(tasks, Object{"activityId": id, "attempt": 2, "status": "completed", "startedAt": 20, "metadata": Object{"address": "secret-address"}}) + } + return Object{ + "workflow": Object{"activities": activities, "dependencies": []any{}}, + "plan": Object{"predicted": Object{"makespanSeconds": 10, "secret": "never-export"}, "assignments": assignments}, + }, Object{ + "run": Object{"id": "run", "status": "completed"}, + "activityAttempts": tasks, + "artifactTransferRuns": []any{Object{"activityId": "activity-1", "status": "completed", "finishedAt": 19}}, + "workspaceLeases": []any{Object{"released": true}}, + }, Object{ + "run": Object{}, + } +} +func TestAuditAllowlistBarrierLatestAndTimes(t *testing.T) { + request, response, sim := auditFixture() + response["activityAttempts"] = append(arr(response["activityAttempts"]), Object{"activityId": "activity-1", "attempt": 1, "status": "failed"}) + r := audit(request, response, sim, false) + require.Len(t, rows(r["activities"]), 58) + require.NotContains(t, string(encode(r)), "secret") + require.Equal(t, "multiple_attempts", r["attempt_history"]) + obj(arr(response["artifactTransferRuns"])[0])["finishedAt"] = 21 + bad(t, func() Object { return audit(request, response, sim, false) }) + obj(arr(response["artifactTransferRuns"])[0])["finishedAt"] = 19 + obj(response["run"])["status"] = "running" + bad(t, func() Object { return audit(request, response, sim, false) }) + request, response, sim = auditFixture() + obj(response["run"])["startedAt"] = "2026-10-08T00:00:00Z" + obj(response["run"])["finishedAt"] = "2026-10-08T00:01:00Z" + obj(response["run"])["makespanSeconds"] = 100 + r = audit(request, response, sim, false) + require.Equal(t, 60.0, r["real_api_run_timestamp_wall_seconds"]) +} +func TestAuditStrictAndMicroOverlap(t *testing.T) { + request, response, sim := auditFixture() + handles := []any{} + for _, t := range rows(response["activityAttempts"]) { + t["attempt"] = 1 + handles = append(handles, Object{"id": "run:" + str(t["activityId"]), "runId": "run", "activityId": t["activityId"], "status": "completed", "exitCode": 0}) + } + response["handles"] = handles + response["cloudBilling"] = Object{"status": "estimated", "currency": "USD", "estimatedCost": 0.2} + audit(request, response, sim, true) + obj(handles[0])["exitCode"] = nil + bad(t, func() Object { return audit(request, response, sim, true) }) + obj(handles[0])["exitCode"] = 0 + obj(handles[0])["id"] = "wrong" + bad(t, func() Object { return audit(request, response, sim, true) }) + request, response, sim = auditFixture() + request["resources"] = []any{Object{"id": "micro", "cpuCapacity": 1}} + for i, t := range rows(response["activityAttempts"]) { + t["startedAt"] = 20 + i*10 + t["finishedAt"] = 25 + i*10 + } + audit(request, response, sim, false) + obj(arr(response["activityAttempts"])[1])["startedAt"] = 24 + bad(t, func() Object { return audit(request, response, sim, false) }) +} +func TestNativeTimingExactAttempt(t *testing.T) { + request, response, sim := auditFixture() + h := Object{"activityId": "activity-1", "startedAt": 20, "finishedAt": 25, "metadata": Object{"executionAttempt": 2, "terminalObservedAt": 28, "address": "never-export"}} + response["handles"] = []any{h} + r := audit(request, response, sim, false) + require.Equal(t, 5.0, rows(r["activities"])[0]["validated_handle_runtime_seconds"]) + require.Equal(t, 3.0, rows(r["activities"])[0]["terminal_observation_delay_seconds"]) + obj(h["metadata"])["executionAttempt"] = 1 + require.Nil(t, rows(audit(request, response, sim, false)["activities"])[0]["validated_handle_runtime_seconds"]) + obj(h["metadata"])["executionAttempt"] = 2 + obj(h["metadata"])["terminalObservedAt"] = 24 + require.Nil(t, rows(audit(request, response, sim, false)["activities"])[0]["validated_handle_runtime_seconds"]) +} +func TestFixedPlacementAndLocalComparison(t *testing.T) { + observed := Object{ + "run_id": "observed", + "activities": []any{Object{"activity_id": "a", "resource_id": "vm"}, Object{"activity_id": "b", "resource_id": "vm"}}, + "routes": []any{ + Object{"producer_activity_id": "a", "consumer_activity_id": "b", "strategy": "direct-runtime", "network_bytes": 0, "phases": []any{Object{"name": "shared-conflict-check-copy-and-verify", "durationSeconds": 1000}}}, + }, + "real_api_reported": Object{"makespanSeconds": 100}, + } + request := Object{"run": Object{"mode": "simulation"}, "plan": Object{"assignments": []any{Object{"activityId": "a", "resourceId": "vm"}, Object{"activityId": "b", "resourceId": "vm"}}}, "networkTopology": Object{"metadata": Object{}}} + prepared := fixedRequest(request, Object{"source": "old"}, observed, "sim-only") + require.Equal(t, normal(request["plan"]), normal(prepared["plan"])) + require.NotContains(t, obj(request["run"]), "id") + obj(request["run"])["mode"] = "real" + bad(t, func() Object { return fixedRequest(request, Object{}, observed, "bad") }) + sim := Object{"transfers": []any{}, "executed": Object{"makespanSeconds": 90}} + r := compare(observed, sim, "calibration-fit-not-independent-validation", Object{"source": "observed"}) + require.Equal(t, 0, r["routeMismatchCount"]) + require.Equal(t, -10.0, r["makespanErrorPercent"]) + require.Equal(t, false, r["accepted"]) + obj(arr(observed["routes"])[0])["network_bytes"] = 10 + r = compare(observed, sim, "original-frozen-forecast", Object{"source": "old"}) + require.Equal(t, 1, r["routeMismatchCount"]) + require.Equal(t, -100.0, r["networkBytesErrorPercent"]) + bad(t, func() Object { return compare(observed, Object{}, "unlabeled", Object{}) }) +} +func TestSourceCapacityNotNICOrQueue(t *testing.T) { + flows := []any{} + for i := 0; i < 3; i++ { + flows = append(flows, Object{ + "status": "completed", + "route": Object{ + "strategy": "gateway", + "sourceResourceId": "hpc", + "targetResourceId": "montage-gcp-fog-e2-micro-02", + "archiveBytes": 100, + "sourcePackExecutionSeconds": 10, + "sourcePackSshChannelWaitSeconds": 999, + "phases": []any{Object{"name": "pack-and-receive", "finishedAt": 20}}, + }, + }) + } + c := sourceCapacity(Object{"artifactTransferRuns": flows}, "hpc", "e2-micro") + require.Equal(t, 3, c["concurrentCommandsMaximum"]) + require.Equal(t, 10.0, c["busySeconds"]) + require.Equal(t, 30.0, c["durationWeightedMedianBytesPerSecond"]) + request := Object{ + "networkTopology": Object{ + "metadata": Object{"workspaceMaterializationModel": Object{"relayResourceId": "relay"}}, + "links": []any{Object{"id": "physical", "sourceResourceId": "relay", "targetResourceId": "hpc", "bidirectional": true, "bandwidthBitsPerSecond": 8000}}, + }, + } + r := applyCapacity(request, "hpc", c) + links := rows(obj(r["networkTopology"])["links"]) + require.Equal(t, false, links[0]["bidirectional"]) + require.Equal(t, 8000.0, num(links[0]["bandwidthBitsPerSecond"])) + require.Equal(t, 240.0, num(links[1]["bandwidthBitsPerSecond"])) + require.Equal(t, true, rows(obj(request["networkTopology"])["links"])[0]["bidirectional"]) + bad(t, func() Object { return sourceCapacity(Object{"artifactTransferRuns": []any{}}, "hpc", "e2-micro") }) +} +func TestCLIRejectsBadArgsAndWritesPrivateJSON(t *testing.T) { + require.Error(t, Run(nil)) + require.Error(t, Run([]string{"unknown"})) + dir := t.TempDir() + out := filepath.Join(dir, "result.json") + input := filepath.Join("..", "..", "experiments", "montage-micro-hpc-e2", "docker-image-real-v2-results.json") + require.NoError(t, Run([]string{"build_montage_execution_calibration", input, out})) + info, err := os.Stat(out) + require.NoError(t, err) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) + require.Error(t, Run([]string{"export_cloud_workspace", "bad", "bad", out})) +} diff --git a/internal/montagetools/transfer_wall.go b/internal/montagetools/transfer_wall.go new file mode 100644 index 00000000..1a09d69d --- /dev/null +++ b/internal/montagetools/transfer_wall.go @@ -0,0 +1,124 @@ +package montagetools + +import "strings" + +// Exploratory effective route wall profiles, not queue-free service rates. +// Shared consumer merge/barrier phases are not charged per dependency. +func transferWallSensitivity(request, real Object, provenance string) Object { + request = clone(request) + check(obj(real["run"])["status"] == "completed", "completed transfer reference required") + groups := transferWallReferenceGroups(real) + metadata := obj(obj(request["NetworkTopology"])["metadata"]) + model := obj(metadata["workspaceMaterializationModel"]) + updated := []any{} + for _, profile := range rows(model["profiles"]) { + key := str(profile["strategy"]) + "/" + str(profile["sourceClass"]) + "/" + str(profile["targetClass"]) + g := groups[key] + if g == nil { + continue + } + profile["archiveToWorkspaceRatio"] = num(g["archive"]) / num(g["workspace"]) + profile["uploadArchiveBytesPerSecond"] = num(g["archive"]) / num(g["upload"]) + profile["receiverSecondsPerByte"] = num(g["receiver"]) / num(g["workspace"]) + profile["source"], profile["samples"] = "exploratory-effective-transfer-wall:"+provenance, int(num(g["samples"])) + if profile["strategy"] == "direct-runtime" { + profile["probeSeconds"] = num(g["probe"]) / num(g["samples"]) + profile["snapshotSecondsPerByte"] = num(g["snapshot"]) / num(g["workspace"]) + } else { + check(num(g["ingress"]) > 0, "gateway ingress phase missing") + profile["ingressArchiveBytesPerSecond"] = num(g["archive"]) / num(g["ingress"]) + profile["validationSecondsPerByte"] = num(g["validation"]) / num(g["workspace"]) + } + updated = append(updated, clone(profile)) + } + check(len(updated) > 0, "no comparable measured route profiles") + // Wall samples may contain unobserved admission; do not put another SSH + // semaphore queue on top of their effective durations in this sensitivity. + for _, value := range obj(model["resources"]) { + resource := obj(value) + delete(resource, "bulkServiceGroupId") + delete(resource, "bulkServiceSlots") + delete(resource, "bulkServiceSource") + } + metadata["transferWallSensitivity"] = Object{ + "referenceSHA256": provenance, + "profiles": updated, + "role": "effective-wall-route-sensitivity-not-independent-validation", + "limitations": list( + "Unknown SSH/cache waiting and CPU/IO contention remain in effective wall coefficients.", + "Explicit SSH admission groups are disabled for this offline sensitivity to avoid adding another queue on wall-fit samples.", + "Network sharing remains simulated; upload wall samples may already contain sharing, so double-counting risk is not eliminated.", + "Shared merge is retained once per consumer, not replicated per predecessor.", + "No real execution is authorized or submitted.", + ), + } + return request +} + +func transferWallReferenceGroups(real Object) map[string]Object { + latest := map[string]Object{} + workspaceBytes := map[string]float64{} + for _, workspace := range rows(real["workspaces"]) { + id := str(workspace["activityId"]) + check(workspaceBytes[id] == 0, "ambiguous source workspace volume") + workspaceBytes[id] = num(workspace["sizeBytes"]) + } + for _, transfer := range rows(real["artifactTransferRuns"]) { + route := obj(transfer["route"]) + if transfer["status"] != "completed" || route["producerActivityId"] == nil || route["consumerActivityId"] == nil { + continue + } + key := str(route["producerActivityId"]) + "/" + str(route["consumerActivityId"]) + if latest[key] == nil || num(transfer["startedAt"]) > num(latest[key]["startedAt"]) { + latest[key] = transfer + } + } + groups := map[string]Object{} + for _, transfer := range latest { + strategy := transfer["strategy"] + if strategy != "direct-runtime" && strategy != "gateway" { + continue + } + route := obj(transfer["route"]) + // logicalBytes can contain only newly produced outputs (e.g. a tiny + // table), while the archive carries all inherited workspace files. + workspace, archive := workspaceBytes[str(route["producerActivityId"])], num(route["archiveBytes"]) + if workspace <= 0 || archive <= 0 { + continue + } + from, to := resourceClass(str(route["sourceResourceId"])), resourceClass(str(route["targetResourceId"])) + key := str(strategy) + "/" + from + "/" + to + phaseSeconds := map[string]float64{} + for _, phase := range rows(route["phases"]) { + name := str(phase["name"]) + if strings.HasPrefix(name, "shared-") { + continue + } + seconds := num(phase["durationSeconds"]) - num(phase["sshChannelWaitSeconds"]) + check(seconds >= 0, "negative transfer phase service after observed admission subtraction") + phaseSeconds[name] += seconds + } + if phaseSeconds["peer-private-upload"] <= 0 && phaseSeconds["archive-upload"] <= 0 { + continue + } + g := groups[key] + if g == nil { + g = Object{"strategy": strategy, "sourceClass": from, "targetClass": to} + groups[key] = g + } + for name, value := range map[string]float64{ + "workspace": workspace, + "archive": archive, + "samples": 1, + "probe": phaseSeconds["peer-authenticated-connectivity-probe"], + "snapshot": phaseSeconds["peer-snapshot-acquire"], + "upload": phaseSeconds["peer-private-upload"] + phaseSeconds["archive-upload"], + "receiver": phaseSeconds["peer-receiver-verify-and-extract"] + phaseSeconds["receiver-verify-and-extract"], + "ingress": phaseSeconds["pack-and-receive"], + "validation": phaseSeconds["archive-validation"], + } { + g[name] = num(g[name]) + value + } + } + return groups +} diff --git a/internal/montagetools/transfer_wall_test.go b/internal/montagetools/transfer_wall_test.go new file mode 100644 index 00000000..e8696aa2 --- /dev/null +++ b/internal/montagetools/transfer_wall_test.go @@ -0,0 +1,54 @@ +package montagetools + +import "testing" + +func TestTransferWallSensitivityFitsStagesWithoutSharedBarrierDuplication(t *testing.T) { + profile := Object{"strategy": "direct-runtime", "sourceClass": "e2-micro", "targetClass": "e2-micro"} + request := Object{ + "NetworkTopology": Object{"metadata": Object{"workspaceMaterializationModel": Object{"profiles": []any{profile}, "resources": Object{"micro": Object{"bulkServiceGroupId": "old", "bulkServiceSlots": 3, "bulkServiceSource": "old"}}}}}, + } + real := Object{ + "run": Object{ + "status": "completed", + }, + "workspaces": []any{ + Object{"activityId": "p", "sizeBytes": 100.0}, + }, + "artifactTransferRuns": []any{ + Object{ + "status": "completed", + "strategy": "direct-runtime", + "startedAt": 1.0, + "logicalBytes": 1.0, + "route": Object{"producerActivityId": "p", "consumerActivityId": "c", "sourceResourceId": "montage-gcp-fog-e2-micro-01", "targetResourceId": "montage-gcp-fog-e2-micro-02", "archiveBytes": 50.0, "phases": []any{ + Object{ + "name": "peer-snapshot-acquire", + "durationSeconds": 12.0, + "sshChannelWaitSeconds": 2.0, + }, Object{ + "name": "peer-private-upload", + "durationSeconds": 5.0, + }, Object{ + "name": "peer-receiver-verify-and-extract", + "durationSeconds": 2.0, + }, Object{ + "name": "shared-preparation-barrier", + "durationSeconds": 1000.0, + }, + }}, + }, + }, + } + result := transferWallSensitivity(request, real, "digest") + model := obj(obj(obj(result["NetworkTopology"])["metadata"])["workspaceMaterializationModel"]) + fitted := obj(arr(model["profiles"])[0]) + if num(fitted["snapshotSecondsPerByte"]) != .1 || num(fitted["uploadArchiveBytesPerSecond"]) != 10 || num(fitted["receiverSecondsPerByte"]) != .02 { + t.Fatal("wrong phase fit or duplicated barrier", fitted) + } + if obj(obj(model["resources"])["micro"])["bulkServiceGroupId"] != nil { + t.Fatal("admission double charged") + } + if profile["snapshotSecondsPerByte"] != nil { + t.Fatal("source forecast mutated") + } +} diff --git a/internal/montagetools/validation.go b/internal/montagetools/validation.go new file mode 100644 index 00000000..62cb85af --- /dev/null +++ b/internal/montagetools/validation.go @@ -0,0 +1,348 @@ +package montagetools + +import ( + "math" + "sort" + "strings" +) + +func fixedRequest(original, model, observed Object, id string) Object { + r := clone(original) + check(obj(get(r, "run"))["mode"] == "simulation", "Simulation-only request required") + assigned, actual := Object{}, Object{} + for _, a := range rows(obj(r["plan"])["assignments"]) { + assigned[str(a["activityId"])] = a["resourceId"] + } + for _, a := range rows(observed["activities"]) { + actual[str(a["activity_id"])] = a["resource_id"] + } + check(eq(assigned, actual), "Fixed observed placement differs from plan") + obj(r["run"])["id"] = id + obj(get(obj(get(r, "networkTopology")), "metadata"))["workspaceMaterializationModel"] = model + return r +} +func errorPercent(predicted, observed float64) any { + if observed <= 0 { + return nil + } + return 100 * (predicted - observed) / observed +} +func compare(observed, simulated Object, role string, model Object) Object { + check(role == "original-frozen-forecast" || role == "historical-calibration-completed-run-holdout" || role == "calibration-fit-not-independent-validation", "Unknown comparison role") + predicted := Object{} + for _, r := range rows(get(simulated, "transfers")) { + predicted[str(r["producerActivityId"])+"\x00"+str(r["consumerActivityId"])] = r + } + resources := Object{} + for _, a := range rows(observed["activities"]) { + resources[str(a["activity_id"])] = a["resource_id"] + } + resultRows, mismatch := compareDependencyRoutes(observed, predicted, resources) + makespan, real := num(get(obj(get(simulated, "executed")), "makespanSeconds")), num(get(obj(get(observed, "real_api_reported")), "makespanSeconds")) + percent := errorPercent(makespan, real) + dominant := append([]any{}, resultRows...) + sort.SliceStable(dominant, func(i, j int) bool { + return num(obj(dominant[i])["observedDependencyPhaseSeconds"]) > num(obj(dominant[j])["observedDependencyPhaseSeconds"]) + }) + if len(dominant) > 15 { + dominant = dominant[:15] + } + predBytes, realBytes := sum(rows(resultRows), "predictedNetworkBytes"), sum(rows(resultRows), "observedNetworkBytes") + limitation := "Observed comparison run was not used to fit these transfer coefficients; this remains a retrospective test, not a new prospective real validation." + if role == "calibration-fit-not-independent-validation" { + limitation = "This comparison reuses calibration data and is not independent validation." + } + return Object{ + "schemaVersion": 1, + "comparisonRole": role, + "observedRunId": get(observed, "run_id"), + "modelSource": get(model, "source"), + "simulatedMakespanSeconds": makespan, + "observedMakespanSeconds": real, + "makespanErrorPercent": percent, + "withinInitialMakespanTolerance": percent != nil && math.Abs(num(percent)) <= 20, + "predictedNetworkBytes": predBytes, + "observedNetworkBytes": realBytes, + "networkBytesErrorPercent": errorPercent(predBytes, realBytes), + "routeMismatchCount": mismatch, + "dominantTransfers": dominant, + "dependencies": resultRows, + "accepted": false, + "limitations": list( + "Makespan tolerance alone is insufficient; dominant transfer residuals must also be reviewed.", + "Historical receiver phases may include unseparated queues; destination bulk pools must remain disabled until service is separately identified.", + "No summed parallel phase durations are treated as makespan.", + "Compute, ordering and provisioning assumptions retained from the frozen original request.", + limitation, + ), + } +} +func sourceCapacity(diagnostic Object, sourceID, targetClass string) Object { + intervals := []Object{} + boundaries := map[float64]bool{} + for _, t := range rows(get(diagnostic, "artifactTransferRuns")) { + r := obj(t["route"]) + if t["status"] != "completed" || r["strategy"] != "gateway" || r["sourceResourceId"] != sourceID || num(r["sourcePackExecutionSeconds"]) <= 0 || num(r["archiveBytes"]) <= 0 { + continue + } + if resourceClass(str(get(r, "targetResourceId"))) != targetClass { + continue + } + for _, p := range rows(get(r, "phases")) { + if p["name"] == "pack-and-receive" && p["finishedAt"] != nil { + finish, seconds := num(p["finishedAt"]), num(r["sourcePackExecutionSeconds"]) + intervals = append(intervals, Object{"start": finish - seconds, "finish": finish, "rate": num(r["archiveBytes"]) / seconds}) + boundaries[finish-seconds] = true + boundaries[finish] = true + break + } + } + } + check(len(intervals) >= 3, "Insufficient completed source-capacity observations") + points := []float64{} + for p := range boundaries { + points = append(points, p) + } + sort.Float64s(points) + windows := []Object{} + total := 0.0 + maximum := 0 + for i := 1; i < len(points); i++ { + a, b := points[i-1], points[i] + rate, n := 0.0, 0 + for _, v := range intervals { + if num(v["start"]) <= a && num(v["finish"]) >= b { + rate += num(v["rate"]) + n++ + } + } + if n > 0 { + windows = append(windows, Object{"seconds": b - a, "bytesPerSecond": rate, "concurrentCommands": n}) + total += b - a + if n > maximum { + maximum = n + } + } + } + check(total > 0, "Invalid source capacity interval") + sort.SliceStable(windows, func(i, j int) bool { return num(windows[i]["bytesPerSecond"]) < num(windows[j]["bytesPerSecond"]) }) + cumulative, median := 0.0, 0.0 + for _, w := range windows { + cumulative += num(w["seconds"]) + if cumulative >= total/2 { + median = num(w["bytesPerSecond"]) + break + } + } + return Object{ + "samples": len(intervals), + "busySeconds": total, + "concurrentCommandsMaximum": maximum, + "durationWeightedMedianBytesPerSecond": median, + "minimumBytesPerSecond": windows[0]["bytesPerSecond"], + "maximumBytesPerSecond": windows[len(windows)-1]["bytesPerSecond"], + "interpretation": "Aggregate source pack/ingress service, NOT NIC bandwidth; assumes uniform byte progress within each queue-free command.", + "limitations": list("Small auxiliary diagnostic sample; capacity is conditioned on workload and concurrent source work.", "Admission queue excluded from command intervals."), + } +} +func applyCapacity(request Object, source string, capacity Object) Object { + r := clone(request) + topology := obj(get(r, "networkTopology")) + model := obj(get(obj(get(topology, "metadata")), "workspaceMaterializationModel")) + relay := get(model, "relayResourceId") + links := arr(get(topology, "links")) + var physical Object + for _, v := range links { + l := obj(v) + if l["sourceResourceId"] == relay && l["targetResourceId"] == source && l["bidirectional"] == true { + physical = l + break + } + } + check(physical != nil, "Expected explicit bidirectional relay endpoint link") + physical["bidirectional"] = false + measured := clone(physical) + measured["id"] = str(physical["id"]) + "-measured-source-service" + measured["sourceResourceId"] = source + measured["targetResourceId"] = relay + measured["bandwidthBitsPerSecond"] = 8 * num(capacity["durationWeightedMedianBytesPerSecond"]) + measured["sharingPolicy"] = "SHARED" + measured["metadata"] = Object{"measurementBasis": "combined-source-pack-and-ingress-service-not-nic", "evidence": capacity} + topology["links"] = append(links, measured) + return r +} +func supplementSource(c, diagnostic Object, d string) Object { + manifests, snapshots := Object{}, Object{} + for _, h := range rows(get(diagnostic, "handles")) { + if h["status"] == "completed" && h["exitCode"] != nil && num(h["exitCode"]) == 0 { + id := str(get(h, "activityId")) + m := files(get(obj(get(h, "artifacts")), "finalSnapshot")) + manifests[id] = m + snapshots[id] = fileBytes(m) + } + } + samples := []Object{} + completed := []Object{} + for _, t := range rows(get(diagnostic, "artifactTransferRuns")) { + if t["status"] != "completed" { + continue + } + r := obj(t["route"]) + if r["producerActivityId"] != nil { + completed = append(completed, r) + } + if r["strategy"] != "gateway" || r["sourcePackExecutionSeconds"] == nil || r["sourcePackPreflightServiceSeconds"] == nil || r["archiveBytes"] == nil || snapshots[str(r["producerActivityId"])] == nil { + continue + } + if resourceClass(str(r["sourceResourceId"])) != "hpc" || resourceClass(str(r["targetResourceId"])) != "e2-micro" { + continue + } + samples = append(samples, Object{ + "seconds": r["sourcePackExecutionSeconds"], + "archiveBytes": r["archiveBytes"], + "workspaceBytes": snapshots[str(r["producerActivityId"])], + "wait": num(r["sourcePackSshChannelWaitSeconds"]), + "preflight": r["sourcePackPreflightServiceSeconds"], + }) + } + check(len(samples) > 0, "No successful separately measured source service") + updated := clone(c) + var phase Object + for _, p := range rows(get(updated, "materializationPhaseObservations")) { + if p["strategy"] == "gateway" && p["sourceClass"] == "hpc" && p["targetClass"] == "e2-micro" && p["phase"] == "pack-and-receive" { + phase = p + break + } + } + check(phase != nil, "Completed primary reference lacks this route class") + phase["seconds"] = summary(vals(samples, "seconds")) + phase["secondsBases"] = list("source-command-execution-excludes-channel-queue") + phase["pooledArchiveServiceRate"] = pooledRate(samples, "archiveBytes") + phase["pooledWorkspaceServiceRate"] = pooledRate(samples, "workspaceBytes") + phase["sourceChannelWaitSeconds"] = summary(vals(samples, "wait")) + phase["sourcePreflightServiceSeconds"] = summary(vals(samples, "preflight")) + evidence := Object{ + "runId": get(obj(get(diagnostic, "run")), "id"), + "sha256": d, + "completedTransferSamples": len(samples), + "role": "completed commands from interrupted diagnostic; not independent validation", + "queueNotAddedToService": true, + } + updated["auxiliarySourceServiceEvidence"] = evidence + merges := auxiliaryHPCMergeSamples(completed, manifests, snapshots) + check(len(merges) > 0, "Queue-free auxiliary HPC merge service missing") + var merge Object + for _, m := range rows(get(updated, "mergeRateObservations")) { + if m["targetClass"] == "hpc" { + merge = m + break + } + } + check(merge != nil, "Primary HPC merge reference missing") + rates := Object{} + for _, key := range []string{"checkSecondsPerInputByte", "copySecondsPerUniqueByte", "verifySecondsPerUniqueByte"} { + rates[key] = summary(vals(merges, key)) + } + merge["observations"] = rates + merge["commandOverheadSeconds"] = summary(vals(merges, "overhead")) + merge["channelWaitSeconds"] = summary(vals(merges, "wait")) + evidence["completedHPCMergeSamples"] = len(merges) + return updated +} + +func compareDependencyRoutes(observed, predicted, resources Object) ([]any, int) { + resultRows := []any{} + mismatch := 0 + for _, r := range rows(get(observed, "routes")) { + if r["producer_activity_id"] == nil { + continue + } + producer, consumer := str(r["producer_activity_id"]), str(r["consumer_activity_id"]) + estimate := obj(predicted[producer+"\x00"+consumer]) + service, local := 0.0, resources[producer] == resources[consumer] && num(r["network_bytes"]) == 0 + for _, p := range rows(get(r, "phases")) { + if !strings.HasPrefix(str(p["name"]), "shared-") { + service += num(get(p, "durationSeconds")) + local = false + } + } + if r["strategy"] == "runtime-local" { + service = 0 + } + canonical := r["strategy"] + if local { + canonical = "runtime-local" + } + strategy := "runtime-local" + duration, network, wait := 0.0, 0.0, 0.0 + var percent any + if len(estimate) > 0 { + strategy = str(get(estimate, "strategy")) + duration = num(get(estimate, "durationSeconds")) + network = num(estimate["networkBytes"]) + percent = errorPercent(duration, service) + for _, p := range rows(get(obj(get(estimate, "route")), "phases")) { + wait += num(p["serviceWaitSeconds"]) + } + } + if canonical != strategy { + mismatch++ + } + resultRows = append(resultRows, Object{ + "producer": producer, + "consumer": consumer, + "observedStrategy": canonical, + "executorStrategy": r["strategy"], + "predictedStrategy": strategy, + "observedNetworkBytes": num(r["network_bytes"]), + "predictedNetworkBytes": network, + "observedDependencyPhaseSeconds": service, + "predictedDependencySeconds": duration, + "durationErrorPercent": percent, + "predictedServiceWaitSeconds": wait, + }) + } + return resultRows, mismatch +} + +func auxiliaryHPCMergeSamples(completed []Object, manifests, snapshots Object) []Object { + merges := []Object{} + for _, g := range groups(completed, "consumerActivityId") { + consumer := g.key[0] + if manifests[consumer] == nil { + continue + } + r := g.rows[0] + if resourceClass(str(get(r, "targetResourceId"))) != "hpc" || r["mergeCommandOverheadSeconds"] == nil || r["mergeSshChannelWaitSeconds"] == nil { + continue + } + parents := Object{} + valid := true + for _, p := range g.rows { + id := str(p["producerActivityId"]) + parents[id] = true + valid = valid && manifests[id] != nil + } + if !valid { + continue + } + sources := []Object{} + total := 0.0 + for _, p := range sortedKeys(parents) { + sources = append(sources, obj(manifests[p])) + total += num(snapshots[p]) + } + unique := fileBytes(unionFiles(sources)) + if total <= 0 || unique <= 0 { + continue + } + merges = append(merges, Object{ + "checkSecondsPerInputByte": num(get(r, "mergeCheckSeconds")) / total, + "copySecondsPerUniqueByte": num(get(r, "mergeCopySeconds")) / unique, + "verifySecondsPerUniqueByte": num(get(r, "mergeVerifySeconds")) / unique, + "overhead": r["mergeCommandOverheadSeconds"], + "wait": r["mergeSshChannelWaitSeconds"], + }) + } + return merges +} diff --git a/internal/montagetools/workspace.go b/internal/montagetools/workspace.go new file mode 100644 index 00000000..792b8682 --- /dev/null +++ b/internal/montagetools/workspace.go @@ -0,0 +1,169 @@ +package montagetools + +import ( + "regexp" + "strings" +) + +var checksumPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + +func files(snapshot any) Object { + result := Object{} + for _, f := range rows(snapshot) { + name := str(get(f, "path")) + check(!strings.HasPrefix(name, "/") && !strings.Contains(name, "\x00") && !strings.Contains(name, "\\"), "Unsafe scientific manifest path") + for _, p := range strings.Split(name, "/") { + check(p != "..", "Unsafe scientific manifest path") + } + check(checksumPattern.MatchString(str(get(f, "checksum"))), "Missing content identity") + check(num(get(f, "sizeBytes")) >= 0, "Negative manifest size") + check(result[name] == nil, "Duplicate manifest path") + result[name] = Object{"path": name, "sizeBytes": f["sizeBytes"], "contentId": f["checksum"]} + } + return result +} +func fileValues(m Object) []any { + out := []any{} + for _, k := range sortedKeys(m) { + out = append(out, m[k]) + } + return out +} +func fileBytes(m Object) float64 { return sum(rows(fileValues(m)), "sizeBytes") } +func unionFiles(sources []Object) Object { + u := Object{} + for _, source := range sources { + for _, path := range sortedKeys(source) { + f := source[path] + check(u[path] == nil || eq(u[path], f), "Baseline parent content conflict") + u[path] = f + } + } + return u +} +func activityModel(inherited, final, packaging Object, source string) Object { + outputs := []any{} + removed := []any{} + for _, p := range sortedKeys(final) { + f := final[p] + if !eq(inherited[p], f) && !eq(packaging[p], f) { + outputs = append(outputs, f) + } + } + for _, p := range sortedKeys(inherited) { + if final[p] == nil { + removed = append(removed, p) + } + } + inputs := fileValues(packaging) + return Object{"contract": "whole-workspace-v1", "source": source, "inputs": inputs, "inputsByResourceType": Object{"cloud_vm": inputs, "hpc_machine": []any{}}, "outputs": outputs, "removedPaths": removed} +} +func measuredVolumes(audit, raw, request Object, d string) Object { + validateBaseline(audit, d, nil) + check(obj(get(raw, "run"))["id"] == audit["run_id"] && obj(raw["run"])["status"] == "completed", "Completed reference identity mismatch") + snapshots := Object{} + for _, h := range rows(get(raw, "handles")) { + check(h["status"] == "completed" && h["exitCode"] != nil && num(h["exitCode"]) == 0, "Unsuccessful reference handle") + id := str(get(h, "activityId")) + check(snapshots[id] == nil, "Duplicate reference handle") + snapshots[id] = files(get(obj(get(h, "artifacts")), "finalSnapshot")) + } + check(len(snapshots) == 58, "Reference snapshots do not match activities") + for _, a := range rows(audit["activities"]) { + check(snapshots[str(a["activity_id"])] != nil, "Reference snapshots do not match activities") + } + inputs := Object{} + parents := Object{} + for _, e := range rows(get(obj(get(request, "workflow")), "dependencies")) { + id := str(e["activityId"]) + parents[id] = append(arr(parents[id]), get(e, "dependsOnActivityId")) + } + bytes := Object{} + for _, id := range sortedKeys(snapshots) { + sources := []Object{} + total := 0.0 + for _, p := range arr(parents[id]) { + m := obj(get(snapshots, str(p))) + sources = append(sources, m) + total += fileBytes(m) + } + u := unionFiles(sources) + inputs[id] = Object{"inputTotalBytes": total, "inputUniqueBytes": fileBytes(u), "inputFiles": len(u)} + bytes[id] = fileBytes(obj(snapshots[id])) + } + durations := Object{} + for _, a := range rows(get(obj(request["workflow"]), "activities")) { + durations[str(a["id"])] = get(obj(get(a, "simulation")), "durationSeconds") + } + return Object{"baselineRunId": audit["run_id"], "auditedEvidenceSHA256": d, "activityReferenceDurations": durations, "observedProducerWorkspaceBytes": bytes, "observedConsumerInputVolumes": inputs} +} +func projection(audit, raw, request Object, d string) Object { + result := measuredVolumes(audit, raw, request, d) + workflow := obj(request["workflow"]) + ids := []string{} + for _, a := range rows(workflow["activities"]) { + ids = append(ids, str(a["id"])) + } + check(len(ids) == 58, "Baseline handle mismatch") + snapshots := Object{} + for _, h := range rows(raw["handles"]) { + snapshots[str(h["activityId"])] = files(obj(h["artifacts"])["finalSnapshot"]) + } + preds := Object{} + for _, e := range rows(workflow["dependencies"]) { + id := str(e["activityId"]) + preds[id] = append(arr(preds[id]), e["dependsOnActivityId"]) + } + resources := index(request["resources"], "id") + assignments := Object{} + for _, a := range rows(obj(request["plan"])["assignments"]) { + assignments[str(a["activityId"])] = get(obj(get(resources, str(a["resourceId"]))), "type") + } + cloudRoots, hpcRoots := []string{}, []string{} + for _, id := range ids { + if len(arr(preds[id])) == 0 { + if assignments[id] == "cloud_vm" { + cloudRoots = append(cloudRoots, id) + } + if assignments[id] == "hpc_machine" { + hpcRoots = append(hpcRoots, id) + } + } + } + check(len(cloudRoots) >= 2, "Need multiple OCI root observations") + common := clone(obj(snapshots[cloudRoots[0]])) + for _, id := range cloudRoots[1:] { + for _, p := range sortedKeys(common) { + if !eq(common[p], obj(snapshots[id])[p]) { + delete(common, p) + } + } + } + check(len(common) > 0, "Packaging evidence missing") + check(len(hpcRoots) > 0, "HPC packaging evidence missing") + for _, id := range hpcRoots { + for _, p := range sortedKeys(common) { + check(obj(snapshots[id])[p] == nil, "OCI packaging unexpectedly present on HPC root") + } + } + models := Object{} + for _, id := range ids { + sources := []Object{} + for _, parent := range arr(preds[id]) { + sources = append(sources, obj(get(snapshots, str(parent)))) + } + models[id] = activityModel(unionFiles(sources), obj(get(snapshots, id)), common, "baseline:"+str(audit["run_id"])) + } + result["schemaVersion"] = 1 + result["contract"] = "whole-workspace-v1" + result["runtimePackagingFiles"] = len(common) + result["activityModels"] = models + result["dependencies"] = workflow["dependencies"] + result["baselineAssignedResourceTypes"] = assignments + result["limitations"] = list( + "File sizes and identities are previous-run observations for the same immutable image and dataset, not future observations.", + "Runtime packaging variant is supported only for evidenced cloud_vm and hpc_machine types.", + "No transfer deduplication, snapshot reuse or shared storage is inferred by this manifest model.", + ) + return result +} diff --git a/internal/planning/algorithms/capacity_test.go b/internal/planning/algorithms/capacity_test.go new file mode 100644 index 00000000..c91e05f1 --- /dev/null +++ b/internal/planning/algorithms/capacity_test.go @@ -0,0 +1,13 @@ +package algorithms + +import ( + "github.com/UFFeScience/akoflow/internal/domain" + "testing" +) + +func TestPlanCoresRespectExplicitSingleMicroLane(t *testing.T) { + got := cores(domain.Resource{ID: "micro", CPUCores: 2, CPUCapacity: 1}) + if len(got) != 1 || got[0] != "micro-core-1" { + t.Fatalf("lanes=%v", got) + } +} diff --git a/internal/planning/algorithms/cloud_lifecycle.go b/internal/planning/algorithms/cloud_lifecycle.go index bc610de9..de863d0b 100644 --- a/internal/planning/algorithms/cloud_lifecycle.go +++ b/internal/planning/algorithms/cloud_lifecycle.go @@ -22,6 +22,7 @@ func addCloudLifecycle(plan *domain.SchedulePlan, request domain.PlanningRequest for resourceID, assignments := range byResource { resource := resources[resourceID] first, last, runtimeCost := math.MaxFloat64, 0.0, 0.0 + activeStart := math.MaxFloat64 activityIDs := make([]string, 0, len(assignments)) for _, assignment := range assignments { activityIDs = append(activityIDs, assignment.ID) @@ -32,6 +33,8 @@ func addCloudLifecycle(plan *domain.SchedulePlan, request domain.PlanningRequest last = assignment.PredictedFinishAt } runtimeCost += assignment.PredictedRuntimeSeconds * resource.PricePerSecond + preparation := metadataNumber(assignment.Metadata, "workspacePreparationSeconds") + metadataNumber(assignment.Metadata, "bootOverheadSeconds") + metadataNumber(assignment.Metadata, "containerOverheadSeconds") + activeStart = math.Min(activeStart, math.Max(0, assignment.PredictedStartAt-preparation)) } provision := metadataSeconds(resource.Metadata, "terraformSeconds", 40) sshWait := metadataSeconds(resource.Metadata, "sshWaitSeconds", 20) @@ -51,7 +54,14 @@ func addCloudLifecycle(plan *domain.SchedulePlan, request domain.PlanningRequest vmCost := activeSeconds * resource.PricePerSecond diskGiB := float64(resource.StorageBytes) / float64(1<<30) diskCost := diskGiB * metadataNumber(resource.Metadata, "diskPricePerGiBMonth") * activeSeconds / (730 * 3600) - plan.Predicted.Cost += math.Max(0, vmCost-runtimeCost) + diskCost + // PRISM already bills idle intervals inside the active resource window. + // Replace that window, rather than adding idle a second time. Legacy + // schedulers still charge task runtimes and keep their original basis. + baseCost := runtimeCost + if plan.Metadata["costModel"] == "resource-active-window" { + baseCost = math.Max(0, last-activeStart) * resource.PricePerSecond + } + plan.Predicted.Cost += vmCost - baseCost + diskCost if plan.Metadata == nil { plan.Metadata = map[string]any{} } diff --git a/internal/planning/algorithms/cloud_lifecycle_test.go b/internal/planning/algorithms/cloud_lifecycle_test.go index dc837ce0..837e8d49 100644 --- a/internal/planning/algorithms/cloud_lifecycle_test.go +++ b/internal/planning/algorithms/cloud_lifecycle_test.go @@ -1,18 +1,37 @@ package algorithms import ( + "math" "testing" "github.com/UFFeScience/akoflow/internal/domain" ) +func TestLifecycleReplacesAlreadyBilledIdleWindow(t *testing.T) { + resource := domain.Resource{ID: "cloud", Type: domain.ResourceCloudVM, PricePerSecond: 0.01} + plan := domain.SchedulePlan{ + ID: "plan", + Predicted: domain.PredictedMetrics{Cost: 0.8}, + Metadata: map[string]any{"costModel": "resource-active-window"}, + Assignments: []domain.PlanAssignment{{ResourceID: "cloud", PredictedStartAt: 20, PredictedFinishAt: 30, PredictedRuntimeSeconds: 10}, {ResourceID: "cloud", PredictedStartAt: 80, PredictedFinishAt: 100, PredictedRuntimeSeconds: 20}}, + } + addCloudLifecycle(&plan, domain.PlanningRequest{Resources: []domain.Resource{resource}}) + if math.Abs(plan.Predicted.Cost-1.2) > 1e-12 { + t.Fatalf("cost=%g; window 0..120 should be billed once", plan.Predicted.Cost) + } +} + func TestAddCloudLifecycleAccountsForPreparationIdleAndDisk(t *testing.T) { resource := planningResource("cloud", 2, 1, 0.001) resource.Type = domain.ResourceCloudVM resource.StorageBytes = 100 << 30 resource.Metadata = map[string]any{"terraformSeconds": 10.0, "sshWaitSeconds": 5.0, "configurationSeconds": 10.0, "validationSeconds": 5.0, "destroySeconds": 5.0, "diskPricePerGiBMonth": 0.1} request := domain.PlanningRequest{Resources: []domain.Resource{resource}} - plan := domain.SchedulePlan{ID: "plan", Predicted: domain.PredictedMetrics{Cost: 0.01, Feasible: true}, Assignments: []domain.PlanAssignment{{ID: "activity", ResourceID: resource.ID, PredictedStartAt: 30, PredictedFinishAt: 40, PredictedRuntimeSeconds: 10}}} + plan := domain.SchedulePlan{ + ID: "plan", + Predicted: domain.PredictedMetrics{Cost: 0.01, Feasible: true}, + Assignments: []domain.PlanAssignment{{ID: "activity", ResourceID: resource.ID, PredictedStartAt: 30, PredictedFinishAt: 40, PredictedRuntimeSeconds: 10}}, + } addCloudLifecycle(&plan, request) if len(plan.LifecycleActions) != 4 { t.Fatalf("actions = %#v", plan.LifecycleActions) diff --git a/internal/planning/algorithms/fixed_plan_evaluation.go b/internal/planning/algorithms/fixed_plan_evaluation.go new file mode 100644 index 00000000..a9d15350 --- /dev/null +++ b/internal/planning/algorithms/fixed_plan_evaluation.go @@ -0,0 +1,72 @@ +package algorithms + +import ( + "fmt" + "github.com/UFFeScience/akoflow/internal/domain" +) + +// EvaluateFixedPlan uses the production PRISM stage evaluator without changing +// resource/core placement or order. It prevents stale predicted metrics after +// updating calibration parameters for an offline validation. +func EvaluateFixedPlan(request domain.PlanningRequest, plan domain.SchedulePlan) (domain.SchedulePlan, error) { + search, err := newCompactPRISMContext(request, map[string]any{}) + if err != nil { + return plan, err + } + if len(plan.Assignments) != len(search.activities) { + return plan, fmt.Errorf("fixed evaluation requires complete placement") + } + seen := map[string]bool{} + for _, a := range plan.Assignments { + ordinal, ok := search.activityOrdinal[a.ActivityID] + if !ok || seen[a.ActivityID] { + return plan, fmt.Errorf("unknown or duplicate fixed activity") + } + seen[a.ActivityID] = true + r := compactPRISMResourceOrdinal(search, a.ResourceID) + if r < 0 || r >= len(search.resources) || !search.feasible[ordinal][r] { + return plan, fmt.Errorf("ineligible fixed resource for %s", a.ActivityID) + } + coreOK := false + for _, core := range search.resources[r].cores { + if core == a.CoreID { + coreOK = true + } + } + if !coreOK { + return plan, fmt.Errorf("invalid fixed core for %s", a.ActivityID) + } + } + state := initialCompactPRISMState(search) + state.evaluatedAssignments = append([]domain.PlanAssignment(nil), plan.Assignments...) + state, err = evaluateCompleteCompactPRISMState(search, state) + if err != nil { + return plan, err + } + plan.Assignments = state.evaluatedAssignments + plan.Predicted.MakespanSeconds = state.makespan + plan.Predicted.Cost = state.cost + plan.Predicted.Feasible = true + if request.DeadlineSeconds > 0 && state.makespan > request.DeadlineSeconds { + plan.Predicted.Feasible = false + } + if request.Budget > 0 && state.cost > request.Budget { + plan.Predicted.Feasible = false + } + metadata := clonePRISMEvaluationMetadata(plan.Metadata) + if metadata == nil { + metadata = map[string]any{} + } + metadata["evaluationMode"] = "fixed-placement-production-stage-evaluator" + stages := map[string]float64{"computeSeconds": 0, "preparationSeconds": 0, "externalAdmissionSeconds": 0, "runtimeOverheadSeconds": 0, "publicationSeconds": 0} + for _, a := range plan.Assignments { + stages["computeSeconds"] += a.PredictedRuntimeSeconds + stages["preparationSeconds"] += prismMetadataNumber(a.Metadata, "workspacePreparationSeconds") + stages["externalAdmissionSeconds"] += prismMetadataNumber(a.Metadata, "externalAdmissionSeconds") + stages["runtimeOverheadSeconds"] += prismMetadataNumber(a.Metadata, "containerOverheadSeconds") + prismMetadataNumber(a.Metadata, "bootOverheadSeconds") + stages["publicationSeconds"] += prismMetadataNumber(a.Metadata, "workspacePublicationSeconds") + } + metadata["stageWorkTotalsNotElapsedWall"] = stages + plan.Metadata = metadata + return plan, nil +} diff --git a/internal/planning/algorithms/fixed_plan_evaluation_test.go b/internal/planning/algorithms/fixed_plan_evaluation_test.go new file mode 100644 index 00000000..7b4ae594 --- /dev/null +++ b/internal/planning/algorithms/fixed_plan_evaluation_test.go @@ -0,0 +1,31 @@ +package algorithms + +import ( + "context" + "github.com/stretchr/testify/require" + "testing" +) + +func TestFixedEvaluationUpdatesForecastWithoutChangingPlacement(t *testing.T) { + r := planningRequestFixture() + sink := &testSink{} + require.NoError(t, (HEFT{}).Schedule(context.Background(), r, nil, nil, sink)) + before := sink.plans[0] + for i := range r.Resources { + r.Resources[i].ComputeSpeedup *= 2 + } + after, err := EvaluateFixedPlan(r, before) + require.NoError(t, err) + require.Len(t, after.Assignments, len(before.Assignments)) + for i, a := range after.Assignments { + require.Equal(t, before.Assignments[i].ActivityID, a.ActivityID) + require.Equal(t, before.Assignments[i].ResourceID, a.ResourceID) + require.Equal(t, before.Assignments[i].CoreID, a.CoreID) + require.Equal(t, before.Assignments[i].OrderOnResource, a.OrderOnResource) + } + require.NotEqual(t, before.Predicted.MakespanSeconds, after.Predicted.MakespanSeconds) + require.Equal(t, "fixed-placement-production-stage-evaluator", after.Metadata["evaluationMode"]) + before.Assignments = before.Assignments[:1] + _, err = EvaluateFixedPlan(r, before) + require.Error(t, err) +} diff --git a/internal/planning/algorithms/model.go b/internal/planning/algorithms/model.go index 7804f3df..6de0a64e 100644 --- a/internal/planning/algorithms/model.go +++ b/internal/planning/algorithms/model.go @@ -139,7 +139,7 @@ func dataBytes(workflow domain.WorkflowVersion) map[string]map[string]int64 { func duration(activity domain.Activity, resource domain.Resource, profiles []domain.ActivityResourceProfile) float64 { for _, profile := range profiles { if profile.ActivityTypeID == activity.ActivityTypeID && profile.ResourceID == resource.ID && profile.RuntimeSeconds > 0 { - return profile.RuntimeSeconds + return profile.RuntimeFor(activity) } } base := 1.0 @@ -173,7 +173,7 @@ func opaqueBatchTarget(resource domain.Resource) bool { } func cores(resource domain.Resource) []string { - count := resource.CPUCores + count := resource.SchedulingCPUCores() if count < 1 { count = 1 } diff --git a/internal/planning/algorithms/normalized_profile_test.go b/internal/planning/algorithms/normalized_profile_test.go new file mode 100644 index 00000000..9104298b --- /dev/null +++ b/internal/planning/algorithms/normalized_profile_test.go @@ -0,0 +1,15 @@ +package algorithms + +import ( + "github.com/UFFeScience/akoflow/internal/domain" + "testing" +) + +func TestPlannerUsesSameNormalizedMeasuredWorkAsSimulation(t *testing.T) { + activity := domain.Activity{ActivityTypeID: "family", Simulation: &domain.ActivitySimulation{DurationSeconds: 10}} + resource := domain.Resource{ID: "micro", ComputeSpeedup: .25} + profiles := []domain.ActivityResourceProfile{{ActivityTypeID: "family", ResourceID: "micro", RuntimeSeconds: 2, ModelVersion: "reference-work-v1"}} + if got := duration(activity, resource, profiles); got != 20 { + t.Fatalf("expected normalized work 20, got %g", got) + } +} diff --git a/internal/planning/algorithms/prism_context.go b/internal/planning/algorithms/prism_context.go index 8ea32212..7b20f277 100644 --- a/internal/planning/algorithms/prism_context.go +++ b/internal/planning/algorithms/prism_context.go @@ -5,6 +5,8 @@ import ( "sort" "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/materializationmodel" + domainworkflow "github.com/UFFeScience/akoflow/internal/domain/workflow" ) const prismPendingChunkSize = 64 @@ -21,6 +23,8 @@ type compactPRISMResource struct { } type compactPRISMContext struct { + materializationModel *materializationmodel.Model + requireMeasuredProfiles bool request domain.PlanningRequest activities []domain.Activity activityOrdinal map[string]int @@ -47,6 +51,11 @@ func newCompactPRISMContext( request domain.PlanningRequest, configuration map[string]any, ) (compactPRISMContext, error) { + projected, projectionErr := domainworkflow.ProjectWorkspaceDependencies(request.Workflow) + if projectionErr != nil { + return compactPRISMContext{}, projectionErr + } + request.Workflow = projected topological, err := topologicalOrder(request.Workflow) if err != nil { return compactPRISMContext{}, err @@ -55,7 +64,20 @@ func newCompactPRISMContext( if len(resources) == 0 { return compactPRISMContext{}, fmt.Errorf("execution scope has no schedulable resources") } - router := newCompactPRISMRouter(request.NetworkTopology, resources) + materialization, err := materializationmodel.Decode(request.NetworkTopology.Metadata) + if err != nil { + return compactPRISMContext{}, err + } + routingResources := resources + if materialization != nil { + for _, resource := range request.Resources { + if resource.ID == materialization.RelayResourceID && !resource.Schedulable { + routingResources = append(append([]domain.Resource(nil), resources...), resource) + break + } + } + } + router := newCompactPRISMRouter(request.NetworkTopology, routingResources) rankByID, err := prismCommunicationRanksWithRouter(request, topological, resources, router) if err != nil { return compactPRISMContext{}, err @@ -69,23 +91,25 @@ func newCompactPRISMContext( return ranked[i].ID < ranked[j].ID }) search := compactPRISMContext{ - request: request, - activities: ranked, - activityOrdinal: make(map[string]int, len(ranked)), - predecessors: make([][]compactPRISMDependency, len(ranked)), - successors: make([][]int, len(ranked)), - durations: make([][]float64, len(ranked)), - feasible: make([][]bool, len(ranked)), - minimumCosts: make([]float64, len(ranked)), - averageDurations: make([]float64, len(ranked)), - ranks: make([]float64, len(ranked)), - interference: make([]map[int]float64, len(ranked)), - interferenceModel: "pairwise-cpu-priority", - interferenceRules: make(map[string]map[string]float64), - router: router, - readyBranchLimit: intOption(configuration, "readyBranchLimit", 3, 1, 16), - beamWidth: intOption(configuration, "beamWidth", 120, 1, 10000), + materializationModel: materialization, + request: request, + activities: ranked, + activityOrdinal: make(map[string]int, len(ranked)), + predecessors: make([][]compactPRISMDependency, len(ranked)), + successors: make([][]int, len(ranked)), + durations: make([][]float64, len(ranked)), + feasible: make([][]bool, len(ranked)), + minimumCosts: make([]float64, len(ranked)), + averageDurations: make([]float64, len(ranked)), + ranks: make([]float64, len(ranked)), + interference: make([]map[int]float64, len(ranked)), + interferenceModel: "pairwise-cpu-priority", + interferenceRules: make(map[string]map[string]float64), + router: router, + readyBranchLimit: intOption(configuration, "readyBranchLimit", 3, 1, 16), + beamWidth: intOption(configuration, "beamWidth", 120, 1, 10000), } + search.requireMeasuredProfiles, _ = configuration["requireMeasuredProfiles"].(bool) for ordinal, activity := range ranked { search.activityOrdinal[activity.ID] = ordinal search.ranks[ordinal] = rankByID[activity.ID] @@ -186,6 +210,16 @@ func (search *compactPRISMContext) buildExecutionMatrices() { activity, resource.resource, ) + if search.requireMeasuredProfiles { + measured := false + for _, profile := range search.request.ActivityProfiles { + if profile.ActivityTypeID == activity.ActivityTypeID && profile.ResourceID == resource.resource.ID && profile.UsesWorkUnits() && profile.RuntimeFor(activity) > 0 { + measured = true + break + } + } + search.feasible[activityOrdinal][resourceOrdinal] = search.feasible[activityOrdinal][resourceOrdinal] && measured + } search.durations[activityOrdinal][resourceOrdinal] = duration( activity, resource.resource, diff --git a/internal/planning/algorithms/prism_evaluator.go b/internal/planning/algorithms/prism_evaluator.go index df2cc32e..e3fb327b 100644 --- a/internal/planning/algorithms/prism_evaluator.go +++ b/internal/planning/algorithms/prism_evaluator.go @@ -3,6 +3,8 @@ package algorithms import ( "container/heap" "context" + "encoding/json" + "errors" "fmt" "math" "runtime" @@ -10,6 +12,9 @@ import ( "sync" "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/materializationmodel" + "github.com/UFFeScience/akoflow/internal/domain/storagebudget" + domainworkflow "github.com/UFFeScience/akoflow/internal/domain/workflow" ) type prismEvaluationDependency struct { @@ -18,25 +23,32 @@ type prismEvaluationDependency struct { } type prismEvaluationTask struct { - assignment domain.PlanAssignment - resource domain.Resource - remainingInputs int - dataReadyAt float64 - transferSeconds float64 - laneReady bool - started bool - completed bool - baseRuntime float64 - slowdown float64 - remainingRuntime float64 - rate float64 - rateUpdatedAt float64 + assignment domain.PlanAssignment + resource domain.Resource + remainingInputs int + dataReadyAt float64 + transferSeconds float64 + laneReady bool + started bool + completed bool + baseRuntime float64 + preparationSeconds float64 + publicationSeconds float64 + admissionSeconds float64 + publishing bool + preparationGroup string + preparationHeld bool + slowdown float64 + remainingRuntime float64 + rate float64 + rateUpdatedAt float64 } type prismEvaluationTaskEvent struct { - activity int - finishAt float64 - startsTask bool + activity int + finishAt float64 + startsTask bool + releasesPreparation bool } type prismEvaluationTaskQueue []prismEvaluationTaskEvent @@ -72,17 +84,25 @@ func (queue *prismEvaluationReadyQueue) Pop() any { } type prismEvaluationFlow struct { - consumer int - source string - target string - bytes int64 - remaining float64 - startedAt float64 - payloadAt float64 - route compactPRISMRoute - active bool + consumer int + source string + target string + bytes int64 + remaining float64 + startedAt float64 + payloadAt float64 + route compactPRISMRoute + active bool + stages []materializationmodel.Stage + stageIndex int + postStage bool + rateLimit float64 + stageInitialized bool + serviceHeld bool } +type prismServicePool struct{ slots, used int } + func reevaluateCompleteCompactPRISMStates( search compactPRISMContext, states []compactPRISMState, @@ -99,6 +119,8 @@ func reevaluateCompleteCompactPRISMStates( var workers sync.WaitGroup var firstErr error var errOnce sync.Once + valid := make([]bool, len(states)) + quotaErrors := make([]error, len(states)) workers.Add(workerCount) for worker := 0; worker < workerCount; worker++ { go func() { @@ -109,6 +131,11 @@ func reevaluateCompleteCompactPRISMStates( } evaluated, err := evaluateCompleteCompactPRISMState(search, states[job.index]) if err != nil { + var quota *storagebudget.InsufficientQuota + if errors.As(err, "a) { + quotaErrors[job.index] = err + continue + } errOnce.Do(func() { firstErr = err cancel() @@ -116,6 +143,7 @@ func reevaluateCompleteCompactPRISMStates( continue } result[job.index] = evaluated + valid[job.index] = true } }() } @@ -130,7 +158,20 @@ func reevaluateCompleteCompactPRISMStates( if firstErr != nil { return nil, firstErr } - return result, nil + filtered := result[:0] + for i := range result { + if valid[i] { + filtered = append(filtered, result[i]) + } + } + if len(filtered) == 0 { + for _, err := range quotaErrors { + if err != nil { + return nil, err + } + } + } + return filtered, nil } func evaluateCompleteCompactPRISMState( @@ -141,27 +182,31 @@ func evaluateCompleteCompactPRISMState( if len(assignments) != len(search.activities) { return state, nil } - resourceByID := make(map[string]domain.Resource, len(search.resources)) - for _, resource := range search.resources { - resourceByID[resource.resource.ID] = resource.resource - } - tasks := make([]prismEvaluationTask, len(search.activities)) - assignmentOrdinal := make(map[string]int, len(assignments)) - for _, assignment := range assignments { - assignment.Metadata = clonePRISMEvaluationMetadata(assignment.Metadata) - ordinal, exists := search.activityOrdinal[assignment.ActivityID] - if !exists { - return state, fmt.Errorf("evaluate PRISM candidate: unknown activity %q", assignment.ActivityID) - } - resource, exists := resourceByID[assignment.ResourceID] - if !exists { - return state, fmt.Errorf("evaluate PRISM candidate: unknown resource %q", assignment.ResourceID) - } - tasks[ordinal] = prismEvaluationTask{assignment: assignment, resource: resource, baseRuntime: search.durations[ordinal][compactPRISMResourceOrdinal(search, resource.ID)], slowdown: 1} - assignmentOrdinal[assignment.ActivityID] = ordinal + inputs, err := buildPRISMCandidateInputs(search, assignments) + if err != nil { + return state, err } + resourceByID, storageResources := inputs.resourceByID, inputs.storageResources + tasks, assignmentOrdinal := inputs.tasks, inputs.assignmentOrdinal outgoing := make([][]prismEvaluationDependency, len(tasks)) - data := dataBytes(search.request.Workflow) + projectedWorkflow, err := projectPRISMCandidateWorkspaces(search, assignments, resourceByID) + if err != nil { + return state, err + } + data := dataBytes(projectedWorkflow) + preparationModel, err := materializationmodel.Decode(search.request.NetworkTopology.Metadata) + if err != nil { + return state, err + } + servicePools, err := configurePRISMMaterialization(tasks, assignmentOrdinal, projectedWorkflow, preparationModel) + if err != nil { + return state, err + } + type incomingDependency struct { + producer int + bytes int64 + } + incoming := make([][]incomingDependency, len(tasks)) for _, dependency := range search.request.Workflow.Dependencies { producer := assignmentOrdinal[dependency.DependsOnActivityID] consumer := assignmentOrdinal[dependency.ActivityID] @@ -170,27 +215,13 @@ func evaluateCompleteCompactPRISMState( bytes: data[dependency.ActivityID][dependency.DependsOnActivityID], }) tasks[consumer].remainingInputs++ + incoming[consumer] = append(incoming[consumer], incomingDependency{producer, data[dependency.ActivityID][dependency.DependsOnActivityID]}) } - laneSuccessor := make([]int, len(tasks)) - for index := range laneSuccessor { - laneSuccessor[index] = -1 - tasks[index].laneReady = true - } - byLane := make(map[string][]int) - for ordinal, task := range tasks { - lane := task.assignment.ResourceID + "\x00" + task.assignment.CoreID + "\x00" + task.assignment.SlotID - byLane[lane] = append(byLane[lane], ordinal) - } - for _, lane := range byLane { - sort.Slice(lane, func(left, right int) bool { - return tasks[lane[left]].assignment.OrderOnResource < - tasks[lane[right]].assignment.OrderOnResource - }) - for index := 1; index < len(lane); index++ { - tasks[lane[index]].laneReady = false - laneSuccessor[lane[index-1]] = lane[index] - } + remainingProducers := make([]int, len(tasks)) + for index := range tasks { + remainingProducers[index] = tasks[index].remainingInputs } + laneSuccessor := initializePRISMLanes(tasks) clock := 0.0 completed := 0 @@ -211,7 +242,12 @@ func evaluateCompleteCompactPRISMState( transferCost := 0.0 for completed < len(tasks) { - started := prismStartReadyTasks(search, tasks, readyTasks, taskEvents, clock, resourceStart, resourceFinish, resourceUsed, activeByResource) + stageCost, err := initializePRISMFlowStages(flows, servicePools, search.router, clock) + if err != nil { + return state, err + } + transferCost += stageCost + started := prismStartReadyTasks(search, tasks, readyTasks, taskEvents, clock, resourceStart, resourceFinish, resourceUsed, activeByResource, servicePools) if completed == len(tasks) { break } @@ -226,24 +262,13 @@ func evaluateCompleteCompactPRISMState( } return state, fmt.Errorf("evaluate PRISM candidate: execution cannot progress") } - delta := math.Max(0, nextAt-clock) - for index := range flows { - if flows[index].active && flows[index].payloadAt <= clock { - flows[index].remaining = math.Max(0, flows[index].remaining-rates[index]*delta) - // At large simulated timestamps, a sub-nanosecond completion - // increment can round back to the current clock. Snap that final - // fraction to zero so the event loop always makes progress. - if rates[index] > 0 && flows[index].remaining/rates[index] <= 1e-9 { - flows[index].remaining = 0 - } - } - } + advancePRISMFlowPayloads(flows, rates, clock, nextAt) clock = nextAt for taskEvents.Len() > 0 && (*taskEvents)[0].finishAt <= clock+1e-9 { event := heap.Pop(taskEvents).(prismEvaluationTaskEvent) if prismHandleEvaluationStartEvent( - search, tasks, event, taskEvents, clock, resourceFinish, activeByResource, + search, tasks, event, taskEvents, clock, resourceFinish, activeByResource, servicePools, ) { continue } @@ -256,6 +281,18 @@ func evaluateCompleteCompactPRISMState( prismRecomputePriorityRates(search, tasks, activeByResource[resourceID], taskEvents, clock) continue } + if !tasks[event.activity].publishing && tasks[event.activity].publicationSeconds > 0 { + task := &tasks[event.activity] + task.publishing = true + delete(activeByResource[resourceID], event.activity) + prismRecomputePriorityRates(search, tasks, activeByResource[resourceID], taskEvents, clock) + task.assignment.Metadata["nativeComputeFinishedAt"] = clock + task.assignment.Metadata["workspacePublicationSeconds"] = task.publicationSeconds + task.assignment.PredictedFinishAt = clock + task.publicationSeconds + resourceFinish[resourceID] = math.Max(resourceFinish[resourceID], task.assignment.PredictedFinishAt) + heap.Push(taskEvents, prismEvaluationTaskEvent{activity: event.activity, finishAt: task.assignment.PredictedFinishAt}) + continue + } tasks[event.activity].completed = true delete(activeByResource[resourceID], event.activity) prismRecomputePriorityRates(search, tasks, activeByResource[resourceID], taskEvents, clock) @@ -265,6 +302,27 @@ func evaluateCompleteCompactPRISMState( prismQueueEvaluationTaskIfReady(tasks, readyTasks, successor) } for _, dependency := range outgoing[event.activity] { + if preparationModel != nil { + remainingProducers[dependency.consumer]-- + if remainingProducers[dependency.consumer] > 0 { + continue + } + consumer := &tasks[dependency.consumer] + for _, input := range incoming[dependency.consumer] { + route, routeErr := preparationModel.Route(tasks[input.producer].resource.ID, consumer.resource.ID, input.bytes) + if routeErr != nil { + return state, routeErr + } + if len(route.Stages) == 0 { + prismSatisfyEvaluationInput(consumer, clock, 0) + continue + } + flow := prismEvaluationFlow{consumer: dependency.consumer, startedAt: clock, stages: route.Stages, active: true} + flows = append(flows, flow) + } + prismQueueEvaluationTaskIfReady(tasks, readyTasks, dependency.consumer) + continue + } producer := tasks[event.activity] consumer := &tasks[dependency.consumer] if dependency.bytes <= 0 || producer.assignment.ResourceID == consumer.assignment.ResourceID { @@ -298,36 +356,30 @@ func evaluateCompleteCompactPRISMState( }) } } - for index := range flows { - if !flows[index].active || flows[index].payloadAt > clock+1e-9 || flows[index].remaining > 1e-6 { - continue - } - flows[index].active = false - prismSatisfyEvaluationInput( - &tasks[flows[index].consumer], - clock, - clock-flows[index].startedAt, - ) - prismQueueEvaluationTaskIfReady(tasks, readyTasks, flows[index].consumer) - } - // Only active transfers affect future bandwidth allocation. Keeping every - // completed flow made each later event rescan the entire dependency - // history, turning large DAG evaluation into quadratic work. - activeFlows := flows[:0] - for _, flow := range flows { - if flow.active { - activeFlows = append(activeFlows, flow) - } - } - flows = activeFlows + flows = completePRISMFlowStages(flows, tasks, readyTasks, servicePools, clock) } + if err := checkPRISMCandidateStorage(search.request.Workflow, storageResources, tasks); err != nil { + return state, err + } return compactPRISMFinishEvaluation( state, assignments, assignmentOrdinal, tasks, resourceByID, resourceStart, resourceFinish, clock, transferCost, ), nil } +func checkPRISMCandidateStorage(workflow domainworkflow.WorkflowVersion, resources []domain.Resource, tasks []prismEvaluationTask) error { + timings := make([]storagebudget.Timing, 0, len(tasks)) + for _, task := range tasks { + a := task.assignment + timings = append(timings, storagebudget.Timing{ActivityID: a.ActivityID, ResourceID: a.ResourceID, Start: a.PredictedStartAt, Finish: a.PredictedFinishAt}) + } + if _, _, err := storagebudget.CheckScheduled(workflow, resources, timings); err != nil { + return fmt.Errorf("evaluate PRISM storage budget: %w", err) + } + return nil +} + func compactPRISMFinishEvaluation( state compactPRISMState, assignments []domain.PlanAssignment, @@ -358,7 +410,16 @@ func prismHandleEvaluationStartEvent( clock float64, resourceFinish map[string]float64, activeByResource map[string]map[int]bool, + servicePools map[string]*prismServicePool, ) bool { + if event.releasesPreparation { + task := &tasks[event.activity] + if task.preparationHeld { + servicePools[task.preparationGroup].used-- + task.preparationHeld = false + } + return true + } if !event.startsTask { return false } @@ -393,20 +454,31 @@ func prismStartReadyTasks( resourceFinish map[string]float64, resourceUsed map[string]bool, activeByResource map[string]map[int]bool, + servicePools map[string]*prismServicePool, ) bool { started := false + blocked := []int{} for ready.Len() > 0 { index := heap.Pop(ready).(int) task := &tasks[index] if task.started || task.remainingInputs > 0 || !task.laneReady { continue } + if pool := servicePools[task.preparationGroup]; pool != nil && task.preparationSeconds > 0 { + if pool.used >= pool.slots { + blocked = append(blocked, index) + continue + } + pool.used++ + task.preparationHeld = true + heap.Push(events, prismEvaluationTaskEvent{activity: index, finishAt: clock + task.preparationSeconds, releasesPreparation: true}) + } task.started = true started = true overhead := prismMetadataNumber(task.assignment.Metadata, "bootOverheadSeconds") + prismMetadataNumber(task.assignment.Metadata, "containerOverheadSeconds") task.assignment.PredictedReadyAt = task.dataReadyAt - task.assignment.PredictedStartAt = clock + overhead + task.assignment.PredictedStartAt = clock + overhead + task.preparationSeconds + task.admissionSeconds resourceID := task.assignment.ResourceID task.remainingRuntime = task.baseRuntime task.rate = 1 @@ -418,6 +490,10 @@ func prismStartReadyTasks( task.assignment.Metadata = map[string]any{} } task.assignment.Metadata["queueSeconds"] = math.Max(0, clock-task.dataReadyAt) + task.assignment.Metadata["externalAdmissionSeconds"] = task.admissionSeconds + if task.preparationSeconds > 0 { + task.assignment.Metadata["workspacePreparationSeconds"] = task.preparationSeconds + } task.assignment.Metadata["networkContentionModel"] = "simgrid-shared-link-events" task.assignment.Metadata["cpuPriorityWeight"] = 1.0 if !resourceUsed[resourceID] { @@ -441,6 +517,9 @@ func prismStartReadyTasks( activeByResource, ) } + for _, index := range blocked { + heap.Push(ready, index) + } return started } @@ -533,7 +612,7 @@ func prismRecomputePriorityRates(search compactPRISMContext, tasks []prismEvalua } } } - capacity := math.Max(1, float64(task.resource.CPUCores)) + capacity := float64(task.resource.SchedulingCPUCores()) baseRate := math.Min(1, capacity/float64(len(active))) task.rate = baseRate / slowdown task.slowdown = 1 / math.Max(task.rate, 1e-12) @@ -569,7 +648,7 @@ func prismRecomputePriorityRates(search compactPRISMContext, tasks []prismEvalua } for index := range active { task := &tasks[index] - capacity := math.Max(1, float64(task.resource.CPUCores)) + capacity := float64(task.resource.SchedulingCPUCores()) task.rate = math.Min(1, capacity*weights[index]/math.Max(total, 1e-12)) task.slowdown = 1 / math.Max(task.rate, 1e-12) task.rateUpdatedAt = clock @@ -603,13 +682,17 @@ func prismEvaluationFlowRates(flows []prismEvaluationFlow, clock float64) ([]flo pairUsers := make(map[string]int) nextAt := math.Inf(1) for index, flow := range flows { - if !flow.active { + if !flow.active || (len(flow.stages) > 0 && !flow.stageInitialized) { continue } if flow.payloadAt > clock { nextAt = math.Min(nextAt, flow.payloadAt) continue } + if flow.postStage || flow.remaining == 0 { + nextAt = math.Min(nextAt, clock) + continue + } for _, hop := range flow.route.hops { linkUsers[hop.key]++ } @@ -619,10 +702,13 @@ func prismEvaluationFlowRates(flows []prismEvaluationFlow, clock float64) ([]flo _ = index } for index, flow := range flows { - if !flow.active || flow.payloadAt > clock { + if !flow.active || (len(flow.stages) > 0 && !flow.stageInitialized) || flow.payloadAt > clock || flow.postStage || flow.remaining == 0 { continue } rate := math.Inf(1) + if flow.rateLimit > 0 { + rate = flow.rateLimit + } // Distinct routes can still contend at their sending or receiving // resource. Count the flow itself once, even when source and target are // shared by the same set of active flows. @@ -675,3 +761,201 @@ func compactPRISMStateFromEvaluation( state.projectedCost = cost return state } + +func configurePRISMMaterialization(tasks []prismEvaluationTask, assignmentOrdinal map[string]int, workflow domainworkflow.WorkflowVersion, preparationModel *materializationmodel.Model) (map[string]*prismServicePool, error) { + var err error + servicePools := map[string]*prismServicePool{} + if preparationModel != nil { + for _, r := range preparationModel.Resources { + if r.BulkServiceGroupID != "" { + servicePools[r.BulkServiceGroupID] = &prismServicePool{slots: r.BulkServiceSlots} + } + } + for _, activity := range workflow.Activities { + ordinal := assignmentOrdinal[activity.ID] + var volume domainworkflow.WorkspaceProjectionVolumes + if activity.Metadata["workspaceProjectionVolumes"] == nil { + return nil, fmt.Errorf("materialized candidate requires workspace volumes for %s", activity.ID) + } + payload, marshalErr := json.Marshal(activity.Metadata["workspaceProjectionVolumes"]) + if marshalErr != nil { + return nil, marshalErr + } + if err = json.Unmarshal(payload, &volume); err != nil { + return nil, err + } + tasks[ordinal].preparationSeconds, err = preparationModel.PreparationSeconds(tasks[ordinal].resource.ID, volume.InputTotalBytes, volume.InputUniqueBytes) + if err != nil { + return nil, err + } + tasks[ordinal].publicationSeconds, err = preparationModel.PublicationSeconds(tasks[ordinal].resource.ID, volume.TerminalObservationBytes, volume.PublicationCleanupBytes) + tasks[ordinal].preparationGroup = preparationModel.Resources[tasks[ordinal].resource.ID].BulkServiceGroupID + if err != nil { + return nil, err + } + tasks[ordinal].admissionSeconds, err = preparationModel.AdmissionSeconds(tasks[ordinal].resource.ID) + if err != nil { + return nil, err + } + if tasks[ordinal].assignment.Metadata == nil { + tasks[ordinal].assignment.Metadata = map[string]any{} + } + tasks[ordinal].assignment.Metadata["workspaceMaterializationModelSource"] = preparationModel.Source + tasks[ordinal].assignment.Metadata["workspaceMaterializationModelVersion"] = preparationModel.SchemaVersion + tasks[ordinal].assignment.Metadata["workspacePublicationSeconds"] = tasks[ordinal].publicationSeconds + } + } + return servicePools, nil +} + +func initializePRISMLanes(tasks []prismEvaluationTask) []int { + laneSuccessor := make([]int, len(tasks)) + for index := range laneSuccessor { + laneSuccessor[index] = -1 + tasks[index].laneReady = true + } + byLane := make(map[string][]int) + for ordinal, task := range tasks { + lane := task.assignment.ResourceID + "\x00" + task.assignment.CoreID + "\x00" + task.assignment.SlotID + byLane[lane] = append(byLane[lane], ordinal) + } + for _, lane := range byLane { + sort.Slice(lane, func(left, right int) bool { + return tasks[lane[left]].assignment.OrderOnResource < + tasks[lane[right]].assignment.OrderOnResource + }) + for index := 1; index < len(lane); index++ { + tasks[lane[index]].laneReady = false + laneSuccessor[lane[index-1]] = lane[index] + } + } + return laneSuccessor +} + +func initializePRISMFlowStages(flows []prismEvaluationFlow, servicePools map[string]*prismServicePool, router compactPRISMRouter, clock float64) (float64, error) { + totalCost := 0.0 + for i := range flows { + flow := &flows[i] + if !flow.active || len(flow.stages) == 0 || flow.stageInitialized { + continue + } + stage := flow.stages[flow.stageIndex] + if pool := servicePools[stage.ServiceGroupID]; pool != nil { + if pool.used >= pool.slots { + continue + } + pool.used++ + flow.serviceHeld = true + } + cost, err := prismInitializeMaterializationStage(flow, router, clock) + if err != nil { + return 0, err + } + totalCost += cost + flow.stageInitialized = true + } + return totalCost, nil +} + +func advancePRISMFlowPayloads(flows []prismEvaluationFlow, rates []float64, clock, nextAt float64) { + delta := math.Max(0, nextAt-clock) + for index := range flows { + if flows[index].active && flows[index].payloadAt <= clock { + flows[index].remaining = math.Max(0, flows[index].remaining-rates[index]*delta) + // At large simulated timestamps, a sub-nanosecond completion + // increment can round back to the current clock. Snap that final + // fraction to zero so the event loop always makes progress. + if rates[index] > 0 && flows[index].remaining/rates[index] <= 1e-9 { + flows[index].remaining = 0 + } + } + } +} + +func completePRISMFlowStages(flows []prismEvaluationFlow, tasks []prismEvaluationTask, readyTasks *prismEvaluationReadyQueue, servicePools map[string]*prismServicePool, clock float64) []prismEvaluationFlow { + for index := range flows { + if !flows[index].active || (len(flows[index].stages) > 0 && !flows[index].stageInitialized) || flows[index].payloadAt > clock+1e-9 || flows[index].remaining > 1e-6 { + continue + } + if len(flows[index].stages) > 0 { + flow := &flows[index] + if !flow.postStage { + flow.postStage = true + flow.payloadAt = clock + flow.stages[flow.stageIndex].AfterSeconds + if flow.payloadAt > clock+1e-9 { + continue + } + } + if flow.serviceHeld { + servicePools[flow.stages[flow.stageIndex].ServiceGroupID].used-- + flow.serviceHeld = false + } + flow.stageIndex++ + flow.stageInitialized = false + if flow.stageIndex < len(flow.stages) { + continue + } + } + flows[index].active = false + prismSatisfyEvaluationInput( + &tasks[flows[index].consumer], + clock, + clock-flows[index].startedAt, + ) + prismQueueEvaluationTaskIfReady(tasks, readyTasks, flows[index].consumer) + } + // Only active transfers affect future bandwidth allocation. Keeping every + // completed flow made each later event rescan the entire dependency + // history, turning large DAG evaluation into quadratic work. + activeFlows := flows[:0] + for _, flow := range flows { + if flow.active { + activeFlows = append(activeFlows, flow) + } + } + return activeFlows +} + +type prismCandidateInputs struct { + resourceByID map[string]domain.Resource + storageResources []domain.Resource + tasks []prismEvaluationTask + assignmentOrdinal map[string]int +} + +func buildPRISMCandidateInputs(search compactPRISMContext, assignments []domain.PlanAssignment) (prismCandidateInputs, error) { + resourceByID := make(map[string]domain.Resource, len(search.resources)) + storageResources := make([]domain.Resource, 0, len(search.resources)) + for _, resource := range search.resources { + resourceByID[resource.resource.ID] = resource.resource + storageResources = append(storageResources, resource.resource) + } + tasks := make([]prismEvaluationTask, len(search.activities)) + assignmentOrdinal := make(map[string]int, len(assignments)) + for _, assignment := range assignments { + assignment.Metadata = clonePRISMEvaluationMetadata(assignment.Metadata) + ordinal, exists := search.activityOrdinal[assignment.ActivityID] + if !exists { + return prismCandidateInputs{}, fmt.Errorf("evaluate PRISM candidate: unknown activity %q", assignment.ActivityID) + } + resource, exists := resourceByID[assignment.ResourceID] + if !exists { + return prismCandidateInputs{}, fmt.Errorf("evaluate PRISM candidate: unknown resource %q", assignment.ResourceID) + } + tasks[ordinal] = prismEvaluationTask{assignment: assignment, resource: resource, baseRuntime: search.durations[ordinal][compactPRISMResourceOrdinal(search, resource.ID)], slowdown: 1} + assignmentOrdinal[assignment.ActivityID] = ordinal + } + return prismCandidateInputs{resourceByID: resourceByID, storageResources: storageResources, tasks: tasks, assignmentOrdinal: assignmentOrdinal}, nil +} + +func projectPRISMCandidateWorkspaces(search compactPRISMContext, assignments []domain.PlanAssignment, resourceByID map[string]domain.Resource) (domainworkflow.WorkflowVersion, error) { + assignedTypes := map[string]string{} + for _, assignment := range assignments { + assignedTypes[assignment.ActivityID] = string(resourceByID[assignment.ResourceID].Type) + } + projectedWorkflow, err := domainworkflow.ProjectWorkspaceDependenciesForResourceTypes(search.request.Workflow, assignedTypes) + if err != nil { + return domainworkflow.WorkflowVersion{}, fmt.Errorf("evaluate PRISM workspace projection: %w", err) + } + return projectedWorkflow, nil +} diff --git a/internal/planning/algorithms/prism_materialization.go b/internal/planning/algorithms/prism_materialization.go new file mode 100644 index 00000000..f114437d --- /dev/null +++ b/internal/planning/algorithms/prism_materialization.go @@ -0,0 +1,29 @@ +package algorithms + +import "fmt" + +// Each relay leg is a separate network demand. Preparation and receiver work +// are causal delays around that leg, not an extra end-to-end payload. +func prismInitializeMaterializationStage(flow *prismEvaluationFlow, router compactPRISMRouter, clock float64) (float64, error) { + stage := flow.stages[flow.stageIndex] + flow.source, flow.target = stage.SourceResourceID, stage.TargetResourceID + flow.bytes, flow.remaining = stage.Bytes, float64(stage.Bytes) + flow.rateLimit = stage.EffectiveBytesPerSecond + flow.postStage = false + flow.payloadAt = clock + stage.BeforeSeconds + flow.route = compactPRISMRoute{} + if stage.Bytes == 0 { + return 0, nil + } + route, exists := router.route(flow.source, flow.target) + if !exists { + return 0, fmt.Errorf("materialization stage %s has no route from %s to %s", stage.Name, flow.source, flow.target) + } + flow.route = route + cost := 0.0 + for _, hop := range route.hops { + flow.payloadAt += hop.link.LatencySeconds + cost += float64(stage.Bytes) * hop.link.PricePerByte + } + return cost, nil +} diff --git a/internal/planning/algorithms/prism_materialization_search.go b/internal/planning/algorithms/prism_materialization_search.go new file mode 100644 index 00000000..39ec5a5b --- /dev/null +++ b/internal/planning/algorithms/prism_materialization_search.go @@ -0,0 +1,77 @@ +package algorithms + +import ( + "encoding/json" + "math" + + domainworkflow "github.com/UFFeScience/akoflow/internal/domain/workflow" +) + +// The search frontier uses conservative projected manifests; complete +// candidates are reevaluated with actual chosen resource-type variants. Both +// use the same causal stages and disallow routes without measured profiles. +func compactPRISMResolveMaterializedFanIn(search compactPRISMContext, state compactPRISMState, target string, contention bool, prepared compactPRISMFanIn) compactPRISMTransferResult { + result := compactPRISMTransferResult{state: state} + invalid := func() compactPRISMTransferResult { result.readyAt = math.Inf(1); return result } + barrier := 0.0 + for _, dependency := range prepared.dependencies { + barrier = math.Max(barrier, dependency.assignment.finishAt) + } + result.readyAt = barrier + var batch *compactPRISMNetworkBatch + if contention { + batch = newCompactPRISMNetworkBatch(state) + } + for _, dependency := range prepared.dependencies { + source := search.resources[dependency.assignment.resourceOrdinal].resource.ID + materialized, err := search.materializationModel.Route(source, target, dependency.bytes) + if err != nil { + return invalid() + } + clock := barrier + for _, stage := range materialized.Stages { + clock += stage.BeforeSeconds + if stage.Bytes > 0 { + route, exists := search.router.route(stage.SourceResourceID, stage.TargetResourceID) + if !exists { + return invalid() + } + latency := 0.0 + for _, hop := range route.hops { + latency += hop.link.LatencySeconds + result.cost += float64(stage.Bytes) * hop.link.PricePerByte + } + demand := compactPRISMTransferSecondsOnRoute(compactPRISMState{}, stage.SourceResourceID, stage.TargetResourceID, stage.Bytes, clock, route) + if contention { + demand = compactPRISMTransferSecondsOnRouteBatch(batch, stage.SourceResourceID, stage.TargetResourceID, stage.Bytes, clock, route) + } + if stage.EffectiveBytesPerSecond > 0 { + demand = math.Max(demand, latency+float64(stage.Bytes)/stage.EffectiveBytesPerSecond) + } + if contention { + batch.add(stage.SourceResourceID, stage.TargetResourceID, clock, clock+demand, route.hops) + } + clock += demand + } + clock += stage.AfterSeconds + } + result.seconds += clock - barrier + result.readyAt = math.Max(result.readyAt, clock) + } + if len(prepared.dependencies) > 0 { + var volume domainworkflow.WorkspaceProjectionVolumes + encoded, err := json.Marshal(search.activities[prepared.activityOrdinal].Metadata["workspaceProjectionVolumes"]) + if err != nil || json.Unmarshal(encoded, &volume) != nil { + return invalid() + } + merge, err := search.materializationModel.PreparationSeconds(target, volume.InputTotalBytes, volume.InputUniqueBytes) + if err != nil { + return invalid() + } + result.readyAt += merge + } + if contention { + result.state = batch.commit() + } + return result +} diff --git a/internal/planning/algorithms/prism_materialization_search_test.go b/internal/planning/algorithms/prism_materialization_search_test.go new file mode 100644 index 00000000..7113322b --- /dev/null +++ b/internal/planning/algorithms/prism_materialization_search_test.go @@ -0,0 +1,53 @@ +package algorithms + +import ( + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/materializationmodel" + domainworkflow "github.com/UFFeScience/akoflow/internal/domain/workflow" + "github.com/stretchr/testify/require" + "math" + "testing" +) + +func TestSearchModelsRelayBeforeRankingAndRoutesNonSchedulableController(t *testing.T) { + model := materializationmodel.Model{SchemaVersion: 1, Source: "fixture", RelayResourceID: "relay", Resources: map[string]materializationmodel.Resource{ + "source": {Class: "hpc", Domain: "hpc"}, "target": {Class: "micro", Domain: "cloud"}, "bad": {Class: "unmeasured", Domain: "cloud"}, "relay": {Class: "relay", Domain: "relay"}, + }, Profiles: []materializationmodel.Profile{ + {Strategy: "gateway", SourceClass: "hpc", TargetClass: "micro", Source: "fixture", Samples: 1, ArchiveToWorkspaceRatio: 1, IngressIncludesSourcePreparation: true, IngressArchiveBytesPerSecond: 100, UploadArchiveBytesPerSecond: 50}, + }, MergeProfiles: []materializationmodel.MergeProfile{ + {TargetClass: "micro", Source: "fixture", Samples: 1, CheckSecondsPerInputByte: .01}, + }} + a, b := planningActivity("a", 1), planningActivity("b", 1) + a.Metadata = map[string]any{"workspaceProjection": domainworkflow.WorkspaceProjection{Contract: "whole-workspace-v1", Source: "fixture", Outputs: []domainworkflow.ProjectedFile{{Path: "file", SizeBytes: 100, ContentID: "file"}}}} + b.Metadata = map[string]any{"workspaceProjection": domainworkflow.WorkspaceProjection{Contract: "whole-workspace-v1", Source: "fixture"}} + relay := planningResource("relay", 1, 1, 0) + relay.Schedulable = false + request := domain.PlanningRequest{Workflow: domain.WorkflowVersion{ + Activities: []domain.Activity{a, b}, + Dependencies: []domain.ActivityDependency{{ActivityID: "b", DependsOnActivityID: "a"}}, + }, ExecutionScope: domain.ExecutionScope{ + EnvironmentVersionIDs: []string{"environment"}, + }, Resources: []domain.Resource{ + planningResource("source", 1, 1, 0), + planningResource("target", 1, 1, 0), + planningResource("bad", 1, 1, 0), + relay, + }, NetworkTopology: domain.NetworkTopology{Metadata: map[string]any{ + "workspaceMaterializationModel": model, + }, Links: []domain.NetworkLink{ + {ID: "in", SourceResourceID: "source", TargetResourceID: "relay", BandwidthBitsPerSecond: 8e9, SharingPolicy: "SHARED"}, + {ID: "out", SourceResourceID: "relay", TargetResourceID: "target", BandwidthBitsPerSecond: 8e9, SharingPolicy: "SHARED"}, + }}} + search, err := newCompactPRISMContext(request, nil) + require.NoError(t, err) + state := compactPRISMPlace(search, initialCompactPRISMState(search), search.activityOrdinal["a"], compactPRISMResourceOrdinal(search, "source"), 0, true) + transfer := compactPRISMResolveTransfers(search, state, search.activityOrdinal["b"], "target", true) + require.InDelta(t, 5, transfer.readyAt, 1e-8) + state = compactPRISMPlace(search, state, search.activityOrdinal["b"], compactPRISMResourceOrdinal(search, "target"), 1, true) + require.InDelta(t, 6, state.makespan, 1e-8) + evaluated, err := evaluateCompleteCompactPRISMState(search, state) + require.NoError(t, err) + require.InDelta(t, state.makespan, evaluated.makespan, 1e-8) + unsupported := compactPRISMResolveTransfers(search, state, search.activityOrdinal["b"], "bad", true) + require.True(t, math.IsInf(unsupported.readyAt, 1), "unmeasured route must not win search with a guessed direct link") +} diff --git a/internal/planning/algorithms/prism_materialization_test.go b/internal/planning/algorithms/prism_materialization_test.go new file mode 100644 index 00000000..9a8cf4e2 --- /dev/null +++ b/internal/planning/algorithms/prism_materialization_test.go @@ -0,0 +1,237 @@ +package algorithms + +import ( + "math" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/materializationmodel" + domainworkflow "github.com/UFFeScience/akoflow/internal/domain/workflow" +) + +// Same causal fixture as the native mixed-origin test: roots finish at 1/5, +// peer ends at 6, relay legs at 7/10, merge ends at 12, consumer at 13. +func TestPRISMMaterializationMatchesNativeMixedOriginBarrier(t *testing.T) { + model, request := mixedOriginPRISMFixture() + search, err := newCompactPRISMContext(request, nil) + if err != nil { + t.Fatal(err) + } + assignments := []domain.PlanAssignment{{ActivityID: "a", ResourceID: "peer", CoreID: "peer-core-1"}, {ActivityID: "c", ResourceID: "hpc", CoreID: "hpc-core-1"}, {ActivityID: "b", ResourceID: "target", CoreID: "target-core-1"}} + evaluated, err := evaluateCompleteCompactPRISMState(search, compactPRISMStateFromEvaluation(compactPRISMState{}, assignments, 0, 0)) + if err != nil { + t.Fatal(err) + } + if math.Abs(evaluated.makespan-13) > 1e-8 { + t.Fatalf("native parity expected 13 seconds, got %g", evaluated.makespan) + } + actual := compactPRISMAssignments(search, evaluated) + for _, assignment := range actual { + if assignment.ActivityID == "b" && (math.Abs(assignment.PredictedStartAt-12) > 1e-8 || assignment.Metadata["workspacePreparationSeconds"] != float64(2)) { + t.Fatalf("incorrect consumer barrier: %+v", assignment) + } + } + model, request = assertPRISMPublicationAdmissionObservation(t, model, request, assignments) + // Independently measured 2s source preparation is charged before ingress, + // not as extra network bytes or a second observed queue. + model.Profiles[1].IngressIncludesSourcePreparation = false + model.Profiles[1].SeparatedIngressServiceSource = "queue-free-fixture" + model.Profiles[1].SnapshotSecondsPerByte = .02 + request.NetworkTopology.Metadata["workspaceMaterializationModel"] = model + separated, err := newCompactPRISMContext(request, nil) + if err != nil { + t.Fatal(err) + } + withPreparation, err := evaluateCompleteCompactPRISMState(separated, compactPRISMStateFromEvaluation(compactPRISMState{}, assignments, 0, 0)) + if err != nil { + t.Fatal(err) + } + if math.Abs(withPreparation.makespan-15) > 1e-8 { + t.Fatalf("separated native parity expected 15, got %g", withPreparation.makespan) + } + model.Profiles[1].IngressIncludesSourcePreparation = true + model.Profiles[1].SeparatedIngressServiceSource = "" + model.Profiles[1].SnapshotSecondsPerByte = 0 + // Same native fixture with a single shared transport slot: peer (1s), + // relay ingress (2s), egress (3s), then one merge (2s). Roots end at 5. + for id, resource := range model.Resources { + if resource.Domain == "relay" { + continue + } + resource.BulkServiceGroupID = "control-transport" + resource.BulkServiceSlots = 1 + resource.BulkServiceSource = "fixture-provider-config" + model.Resources[id] = resource + } + request.NetworkTopology.Metadata["workspaceMaterializationModel"] = model + search, err = newCompactPRISMContext(request, nil) + if err != nil { + t.Fatal(err) + } + evaluated, err = evaluateCompleteCompactPRISMState(search, compactPRISMStateFromEvaluation(compactPRISMState{}, assignments, 0, 0)) + if err != nil { + t.Fatal(err) + } + if math.Abs(evaluated.makespan-14) > 1e-8 { + t.Fatalf("shared service pool native parity expected 14 seconds, got %g", evaluated.makespan) + } +} + +func TestMaterializationRateLimitAndReceiverAreSeparateEvents(t *testing.T) { + flow := prismEvaluationFlow{active: true, remaining: 100, rateLimit: 25} + rates, at := prismEvaluationFlowRates([]prismEvaluationFlow{flow}, 0) + if rates[0] != 25 || at != 4 { + t.Fatalf("effective cap ignored: %v %g", rates, at) + } + flow.postStage, flow.payloadAt, flow.remaining = true, 7, 0 + rates, at = prismEvaluationFlowRates([]prismEvaluationFlow{flow}, 4) + if rates[0] != 0 || at != 7 { + t.Fatalf("receiver work consumed bandwidth: %v %g", rates, at) + } +} + +func TestPRISMSixSourceFanInUsesTwoBulkServiceWaves(t *testing.T) { + model := materializationmodel.Model{SchemaVersion: 1, Source: "service-fixture", RelayResourceID: "relay", + Resources: map[string]materializationmodel.Resource{ + "hpc": {Class: "hpc", Domain: "hpc", BulkServiceGroupID: "hpc-session", BulkServiceSlots: 3, BulkServiceSource: "fixture"}, + "target": {Class: "micro", Domain: "cloud", BulkServiceGroupID: "cloud-session", BulkServiceSlots: 3, BulkServiceSource: "fixture"}, + "relay": {Class: "relay", Domain: "relay"}, + }, + Profiles: []materializationmodel.Profile{{Strategy: "gateway", SourceClass: "hpc", TargetClass: "micro", Source: "fixture", Samples: 1, + ArchiveToWorkspaceRatio: 1, IngressArchiveBytesPerSecond: 10, UploadArchiveBytesPerSecond: 100, IngressIncludesSourcePreparation: true}}, + MergeProfiles: []materializationmodel.MergeProfile{{TargetClass: "micro", Source: "fixture", Samples: 1, CheckSecondsPerInputByte: 1.0 / 300}}, + } + request := domain.PlanningRequest{ExecutionScope: domain.ExecutionScope{EnvironmentVersionIDs: []string{"environment"}}, + Resources: []domain.Resource{planningResource("hpc", 1, 1, 0), planningResource("target", 1, 1, 0), planningResource("relay", 1, 1, 0)}, + NetworkTopology: domain.NetworkTopology{Metadata: map[string]any{"workspaceMaterializationModel": model}, Links: []domain.NetworkLink{ + {ID: "ingress", SourceResourceID: "hpc", TargetResourceID: "relay", BandwidthBitsPerSecond: 8e9, SharingPolicy: "SHARED"}, + {ID: "egress", SourceResourceID: "relay", TargetResourceID: "target", BandwidthBitsPerSecond: 8e9, SharingPolicy: "SHARED"}, + }}, + } + assignments := []domain.PlanAssignment{} + for i := 0; i < 6; i++ { + id := string(rune('a' + i)) + activity := planningActivity(id, 1) + activity.Metadata = map[string]any{"workspaceProjection": domainworkflow.WorkspaceProjection{Contract: "whole-workspace-v1", Source: "fixture", + Outputs: []domainworkflow.ProjectedFile{{Path: id, SizeBytes: 100, ContentID: id}}}} + request.Workflow.Activities = append(request.Workflow.Activities, activity) + request.Workflow.Dependencies = append(request.Workflow.Dependencies, domain.ActivityDependency{ActivityID: "consumer", DependsOnActivityID: id}) + assignments = append(assignments, domain.PlanAssignment{ActivityID: id, ResourceID: "hpc", CoreID: "hpc-core-1"}) + } + consumer := planningActivity("consumer", 1) + consumer.Metadata = map[string]any{"workspaceProjection": domainworkflow.WorkspaceProjection{Contract: "whole-workspace-v1", Source: "fixture"}} + request.Workflow.Activities = append(request.Workflow.Activities, consumer) + assignments = append(assignments, domain.PlanAssignment{ActivityID: "consumer", ResourceID: "target", CoreID: "target-core-1"}) + search, err := newCompactPRISMContext(request, nil) + if err != nil { + t.Fatal(err) + } + evaluated, err := evaluateCompleteCompactPRISMState(search, compactPRISMStateFromEvaluation(compactPRISMState{}, assignments, 0, 0)) + if err != nil { + t.Fatal(err) + } + if math.Abs(evaluated.makespan-30) > 1e-8 { + t.Fatalf("expected six serial roots plus two waves, merge and compute: 30s; got %g", evaluated.makespan) + } +} + +func mixedOriginPRISMFixture() (materializationmodel.Model, domain.PlanningRequest) { + model := materializationmodel.Model{SchemaVersion: 1, Source: "measured-fixture", RelayResourceID: "relay", + Resources: map[string]materializationmodel.Resource{ + "peer": {Class: "micro", Domain: "cloud"}, "target": {Class: "micro", Domain: "cloud"}, + "hpc": {Class: "hpc", Domain: "hpc"}, "relay": {Class: "relay", Domain: "relay"}, + }, + Profiles: []materializationmodel.Profile{ + {Strategy: "direct-runtime", SourceClass: "micro", TargetClass: "micro", Source: "fixture", Samples: 1, ArchiveToWorkspaceRatio: 1, UploadArchiveBytesPerSecond: 100}, + { + Strategy: "gateway", + SourceClass: "hpc", + TargetClass: "micro", + Source: "fixture", + Samples: 1, + ArchiveToWorkspaceRatio: 1, + IngressArchiveBytesPerSecond: 100, + UploadArchiveBytesPerSecond: 100, + IngressIncludesSourcePreparation: true, + ValidationSecondsPerByte: .01, + ReceiverSecondsPerByte: .02, + }, + }, MergeProfiles: []materializationmodel.MergeProfile{{TargetClass: "micro", Source: "fixture", Samples: 1, CheckSecondsPerInputByte: .01}}, + } + activities := []domain.Activity{planningActivity("a", 1), planningActivity("c", 5), planningActivity("b", 1)} + for index := range activities { + projection := domainworkflow.WorkspaceProjection{Contract: "whole-workspace-v1", Source: "fixture"} + if index < 2 { + projection.Outputs = []domainworkflow.ProjectedFile{{Path: activities[index].ID, SizeBytes: 100, ContentID: activities[index].ID}} + } + activities[index].Metadata = map[string]any{"workspaceProjection": projection} + } + request := domain.PlanningRequest{ + Workflow: domain.WorkflowVersion{Activities: activities, Dependencies: []domain.ActivityDependency{{ActivityID: "b", DependsOnActivityID: "a"}, {ActivityID: "b", DependsOnActivityID: "c"}}}, + ExecutionScope: domain.ExecutionScope{EnvironmentVersionIDs: []string{"environment"}}, + Resources: []domain.Resource{planningResource("peer", 1, 1, 0), planningResource("hpc", 1, 1, 0), planningResource("target", 1, 1, 0), planningResource("relay", 1, 1, 0)}, + NetworkTopology: domain.NetworkTopology{Metadata: map[string]any{"workspaceMaterializationModel": model}}, + } + for index, pair := range [][2]string{{"peer", "target"}, {"hpc", "relay"}, {"relay", "target"}} { + request.NetworkTopology.Links = append(request.NetworkTopology.Links, domain.NetworkLink{ + ID: string(rune('a' + index)), + SourceResourceID: pair[0], + TargetResourceID: pair[1], + BandwidthBitsPerSecond: 8e9, + SharingPolicy: "SHARED", + PricePerByte: .001, + }) + } + return model, request +} + +func assertPRISMPublicationAdmissionObservation(t *testing.T, model materializationmodel.Model, request domain.PlanningRequest, assignments []domain.PlanAssignment) (materializationmodel.Model, domain.PlanningRequest) { + t.Helper() + model.PublicationProfiles = []materializationmodel.PublicationProfile{{TargetClass: "micro", Source: "queue-free-fixture", Samples: 1, ScanSecondsPerWorkspaceByte: .01}} + request.NetworkTopology.Metadata["workspaceMaterializationModel"] = model + publicationSearch, err := newCompactPRISMContext(request, nil) + if err != nil { + t.Fatal(err) + } + published, err := evaluateCompleteCompactPRISMState(publicationSearch, compactPRISMStateFromEvaluation(compactPRISMState{}, assignments, 0, 0)) + if err != nil { + t.Fatal(err) + } + if math.Abs(published.makespan-15) > 1e-8 { + t.Fatalf("publication barrier expected 15, got %g", published.makespan) + } + targetResource := model.Resources["target"] + targetResource.ExecutionRuntimeID = "fixture-runtime" + model.Resources["target"] = targetResource + model.AdmissionProfiles = []materializationmodel.AdmissionProfile{{TargetClass: "micro", RuntimeID: "fixture-runtime", Source: "native-submit-fixture", Samples: 1, FixedSeconds: 7}} + model.SchemaVersion = 2 + request.NetworkTopology.Metadata["workspaceMaterializationModel"] = model + admissionSearch, err := newCompactPRISMContext(request, nil) + if err != nil { + t.Fatal(err) + } + admitted, err := evaluateCompleteCompactPRISMState(admissionSearch, compactPRISMStateFromEvaluation(compactPRISMState{}, assignments, 0, 0)) + if err != nil { + t.Fatal(err) + } + if math.Abs(admitted.makespan-22) > 1e-8 { + t.Fatalf("admission+publication parity expected22, got%g", admitted.makespan) + } + model.AdmissionProfiles = nil + model.PublicationProfiles = nil + model.InputObservationProfiles = []materializationmodel.InputObservationProfile{{TargetClass: "micro", Source: "queue-free-fixture", Samples: 1, SecondsPerUniqueByte: .1}} + request.NetworkTopology.Metadata["workspaceMaterializationModel"] = model + observedSearch, err := newCompactPRISMContext(request, nil) + if err != nil { + t.Fatal(err) + } + observed, err := evaluateCompleteCompactPRISMState(observedSearch, compactPRISMStateFromEvaluation(compactPRISMState{}, assignments, 0, 0)) + if err != nil { + t.Fatal(err) + } + if math.Abs(observed.makespan-33) > 1e-8 { + t.Fatalf("input union observation must add 20 seconds once, got %g", observed.makespan) + } + model.InputObservationProfiles = nil + return model, request +} diff --git a/internal/planning/algorithms/prism_state.go b/internal/planning/algorithms/prism_state.go index a60b13a5..ae230f49 100644 --- a/internal/planning/algorithms/prism_state.go +++ b/internal/planning/algorithms/prism_state.go @@ -1,10 +1,12 @@ package algorithms import ( + "encoding/json" "math" "math/bits" "github.com/UFFeScience/akoflow/internal/domain" + domainworkflow "github.com/UFFeScience/akoflow/internal/domain/workflow" ) type compactPRISMState struct { @@ -46,7 +48,8 @@ type compactPRISMFanInDependency struct { } type compactPRISMFanIn struct { - dependencies []compactPRISMFanInDependency + activityOrdinal int + dependencies []compactPRISMFanInDependency } func initialCompactPRISMState(search compactPRISMContext) compactPRISMState { @@ -194,16 +197,17 @@ func compactPRISMPlacePreparedAlternatives( contention, prepared, ) - start := math.Max(transfer.readyAt, coreRoot.bestValue) - order := compactPRISMIntLookup(state.coreOrder, coreOrdinal) - boot := 0.0 - if order == 0 { - boot = resource.resource.BootOverheadSeconds - start += boot + if search.materializationModel != nil && math.IsInf(transfer.readyAt, 1) { + return nil } + start, order, boot := compactPRISMPlacementStart(state, resource, coreOrdinal, transfer.readyAt, coreRoot.bestValue) container := resource.resource.ContainerOverhead start += container baseRuntime := search.durations[activityOrdinal][resourceOrdinal] + publication, valid := compactPRISMPublicationSeconds(search, activityOrdinal, resource.resource.ID) + if !valid { + return nil + } factor, waitUntil := compactPRISMInterferenceDecision( search, state, activityOrdinal, resourceOrdinal, start, baseRuntime, ) @@ -220,6 +224,7 @@ func compactPRISMPlacePreparedAlternatives( boot, container, ) + immediate.finishAt += publication children := []compactPRISMState{compactPRISMCommitPlacement( search, state, @@ -242,6 +247,7 @@ func compactPRISMPlacePreparedAlternatives( search.activities[activityOrdinal].Priority, transfer, delayedStart, baseRuntime*delayedFactor, boot, container, ) + delayed.finishAt += publication children = append(children, compactPRISMCommitPlacement( search, state, transfer.state, activityOrdinal, resourceOrdinal, coreOrdinal, sequence, delayed, @@ -249,6 +255,30 @@ func compactPRISMPlacePreparedAlternatives( return children } +func compactPRISMPlacementStart(state compactPRISMState, resource compactPRISMResource, coreOrdinal int, readyAt, availableAt float64) (float64, int, float64) { + start := math.Max(readyAt, availableAt) + order := compactPRISMIntLookup(state.coreOrder, coreOrdinal) + boot := 0.0 + if order == 0 { + boot = resource.resource.BootOverheadSeconds + start += boot + } + return start, order, boot +} + +func compactPRISMPublicationSeconds(search compactPRISMContext, activityOrdinal int, resourceID string) (float64, bool) { + if search.materializationModel == nil { + return 0, true + } + var volume domainworkflow.WorkspaceProjectionVolumes + encoded, err := json.Marshal(search.activities[activityOrdinal].Metadata["workspaceProjectionVolumes"]) + if err != nil || json.Unmarshal(encoded, &volume) != nil { + return 0, false + } + publication, err := search.materializationModel.PublicationSeconds(resourceID, volume.TerminalObservationBytes, volume.PublicationCleanupBytes) + return publication, err == nil +} + func compactPRISMInterferenceDecision( search compactPRISMContext, state compactPRISMState, @@ -323,7 +353,7 @@ func compactPRISMInterferenceDecision( priority = 1 } totalPriority += priority - cores := math.Max(1, float64(search.resources[resourceOrdinal].resource.CPUCores)) + cores := float64(search.resources[resourceOrdinal].resource.SchedulingCPUCores()) return math.Max(1, totalPriority/(cores*priority)), start } @@ -334,7 +364,8 @@ func compactPRISMPrepareFanIn( ) compactPRISMFanIn { predecessors := search.predecessors[activityOrdinal] prepared := compactPRISMFanIn{ - dependencies: make([]compactPRISMFanInDependency, 0, len(predecessors)), + activityOrdinal: activityOrdinal, + dependencies: make([]compactPRISMFanInDependency, 0, len(predecessors)), } for _, dependency := range predecessors { assignment, exists := compactPRISMAssignmentChunkLookup( @@ -375,6 +406,9 @@ func compactPRISMResolvePreparedTransfers( contention bool, prepared compactPRISMFanIn, ) compactPRISMTransferResult { + if search.materializationModel != nil { + return compactPRISMResolveMaterializedFanIn(search, state, resourceID, contention, prepared) + } result := compactPRISMTransferResult{state: state} var networkBatch *compactPRISMNetworkBatch if contention { diff --git a/internal/planning/algorithms/storage_budget_test.go b/internal/planning/algorithms/storage_budget_test.go new file mode 100644 index 00000000..4a0a8fd5 --- /dev/null +++ b/internal/planning/algorithms/storage_budget_test.go @@ -0,0 +1,48 @@ +package algorithms + +import ( + "testing" + "time" + + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/storagebudget" + workflow "github.com/UFFeScience/akoflow/internal/domain/workflow" + "github.com/stretchr/testify/require" +) + +func TestCompletePRISMCandidateRejectsExhaustedSharedQuota(t *testing.T) { + a := planningActivity("a", 1) + a.Metadata = map[string]any{"workspaceProjection": workflow.WorkspaceProjection{Contract: "whole-workspace-v1", Source: "measured", Outputs: []workflow.ProjectedFile{{Path: "out", ContentID: "out", SizeBytes: 100}}}} + r := planningResource("node", 1, 1, 0) + r.Metadata = map[string]any{"workspaceStorageBudget": storagebudget.Snapshot{StorageID: "user-home", AvailableBytes: 0, Source: "quota-api", ObservedAt: time.Now()}} + request := domain.PlanningRequest{Workflow: domain.WorkflowVersion{Activities: []domain.Activity{a}}, Resources: []domain.Resource{r}, ExecutionScope: domain.ExecutionScope{EnvironmentVersionIDs: []string{"environment"}}} + search, err := newCompactPRISMContext(request, nil) + require.NoError(t, err) + assignments := []domain.PlanAssignment{{ActivityID: "a", ResourceID: "node", CoreID: "node-core-1"}} + _, err = evaluateCompleteCompactPRISMState(search, compactPRISMStateFromEvaluation(compactPRISMState{}, assignments, 0, 0)) + require.ErrorContains(t, err, "storage quota") +} + +func TestQuotaInfeasibleCandidateDoesNotAbortOtherCandidates(t *testing.T) { + a := planningActivity("a", 1) + a.Metadata = map[string]any{"workspaceProjection": workflow.WorkspaceProjection{Contract: "whole-workspace-v1", Source: "measured", Outputs: []workflow.ProjectedFile{{Path: "out", ContentID: "out", SizeBytes: 100}}}} + r := planningResource("full", 1, 1, 0) + r.Metadata = map[string]any{"workspaceStorageBudget": storagebudget.Snapshot{StorageID: "full", AvailableBytes: 0, Source: "quota-api", ObservedAt: time.Now()}} + r2 := planningResource("available", 1, 1, 0) + r2.Metadata = map[string]any{"workspaceStorageBudget": storagebudget.Snapshot{StorageID: "available", AvailableBytes: 1000, Source: "quota-api", ObservedAt: time.Now()}} + search, err := newCompactPRISMContext(domain.PlanningRequest{ + Workflow: domain.WorkflowVersion{Activities: []domain.Activity{a}}, + Resources: []domain.Resource{r, r2}, + ExecutionScope: domain.ExecutionScope{EnvironmentVersionIDs: []string{"environment"}}, + }, nil) + require.NoError(t, err) + states := []compactPRISMState{} + for _, id := range []string{"full", "available"} { + states = append(states, compactPRISMStateFromEvaluation(compactPRISMState{}, []domain.PlanAssignment{{ActivityID: "a", ResourceID: id, CoreID: id + "-core-1"}}, 0, 0)) + } + got, err := reevaluateCompleteCompactPRISMStates(search, states) + require.NoError(t, err) + require.Len(t, got, 1) + _, err = reevaluateCompleteCompactPRISMStates(search, states[:1]) + require.ErrorContains(t, err, "storage quota") +} diff --git a/internal/provider/cloud/ansible/runner.go b/internal/provider/cloud/ansible/runner.go index 4d9e6f2f..262adea9 100644 --- a/internal/provider/cloud/ansible/runner.go +++ b/internal/provider/cloud/ansible/runner.go @@ -5,9 +5,11 @@ import ( "context" "encoding/json" "fmt" + "io" "os" "os/exec" "path/filepath" + "strconv" "strings" "time" @@ -80,15 +82,15 @@ func (r Runner) runPlaybook(ctx context.Context, workspace string, spec ports.Ma binary = "ansible-playbook" } arguments := []string{"-i", spec.Address + ",", "-u", spec.SSHUser, "--private-key", keyPath, "--extra-vars", "@" + variablesPath, playbookPath} + if spec.KnownHostsFile != "" && spec.HostKeyAlias != "" { + arguments = append(arguments, "--ssh-common-args", "-o StrictHostKeyChecking=yes -o HostKeyAlgorithms=ssh-ed25519 -o UserKnownHostsFile="+strconv.Quote(spec.KnownHostsFile)+" -o HostKeyAlias="+strconv.Quote(spec.HostKeyAlias)) + } playbookStartedAt := time.Now().UTC() writeProvisionLog(logFile, "applying %s", filepath.Base(playbookPath)) _ = logFile.Sync() - output, err := r.execute(ctx, workspace, append([]string{binary}, arguments...)...) - if len(output) > 0 { - _, _ = logFile.Write(output) - if output[len(output)-1] != '\n' { - _, _ = logFile.WriteString("\n") - } + output, err := r.execute(ctx, workspace, logFile, append([]string{binary}, arguments...)...) + if len(output) > 0 && output[len(output)-1] != '\n' { + _, _ = logFile.WriteString("\n") } if err != nil { writeProvisionLog(logFile, "failed after %.3fs: %v", time.Since(playbookStartedAt).Seconds(), err) @@ -100,16 +102,24 @@ func (r Runner) runPlaybook(ctx context.Context, workspace string, spec ports.Ma return nil } -func (r Runner) execute(ctx context.Context, workspace string, commandAndArguments ...string) ([]byte, error) { +func (r Runner) execute(ctx context.Context, workspace string, logFile io.Writer, commandAndArguments ...string) ([]byte, error) { + environment := append(os.Environ(), "ANSIBLE_HOST_KEY_CHECKING=False", "PYTHONUNBUFFERED=1") if r.Execute != nil { - return r.Execute(ctx, workspace, append(os.Environ(), "ANSIBLE_HOST_KEY_CHECKING=False"), commandAndArguments...) + output, err := r.Execute(ctx, workspace, environment, commandAndArguments...) + if _, writeErr := logFile.Write(output); err == nil { + err = writeErr + } + return output, err } command := exec.CommandContext(ctx, commandAndArguments[0], commandAndArguments[1:]...) command.Dir = workspace - command.Env = append(os.Environ(), "ANSIBLE_HOST_KEY_CHECKING=False") + command.Env = environment var output bytes.Buffer - command.Stdout = &output - command.Stderr = &output + // Both streams share one writer, so os/exec serializes writes. Keep the + // existing failure diagnostics while publishing progress before process exit. + writer := io.MultiWriter(logFile, &output) + command.Stdout = writer + command.Stderr = writer err := command.Run() return output.Bytes(), err } @@ -194,6 +204,26 @@ func runCheck(ctx context.Context, spec ports.MachineConfigurationSpec, keyPath, "-o", "UserKnownHostsFile=/dev/null", "-o", "ConnectTimeout=10", "-i", keyPath, spec.SSHUser + "@" + spec.Address, command, } + if spec.KnownHostsFile != "" && spec.HostKeyAlias != "" { + arguments = []string{ + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=yes", + "-o", + "HostKeyAlgorithms=ssh-ed25519", + "-o", + "UserKnownHostsFile=" + spec.KnownHostsFile, + "-o", + "HostKeyAlias=" + spec.HostKeyAlias, + "-o", + "ConnectTimeout=10", + "-i", + keyPath, + spec.SSHUser + "@" + spec.Address, + command, + } + } output, err := exec.CommandContext(ctx, "ssh", arguments...).CombinedOutput() if err != nil { return fmt.Errorf("ssh: %w: %s", err, strings.TrimSpace(string(output))) diff --git a/internal/provider/cloud/ansible/runner_test.go b/internal/provider/cloud/ansible/runner_test.go index beb105c0..47ef9e93 100644 --- a/internal/provider/cloud/ansible/runner_test.go +++ b/internal/provider/cloud/ansible/runner_test.go @@ -109,6 +109,54 @@ func TestRunPlaybookReturnsCommandOutputOnFailure(t *testing.T) { } } +func TestExecutePublishesOutputBeforeProcessExit(t *testing.T) { + root := t.TempDir() + logPath := filepath.Join(root, "provision.log") + logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + t.Fatal(err) + } + defer logFile.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + gate := filepath.Join(root, "finish") + result := make(chan error, 1) + go func() { + output, err := (Runner{}).execute(ctx, root, logFile, "sh", "-c", `printf 'TASK progress\n'; while [ ! -f "$1" ]; do sleep 0.01; done; printf 'failure detail\n' >&2; exit 2`, "sh", gate) + if err == nil || !strings.Contains(string(output), "failure detail") { + result <- errors.New("missing process failure diagnostics") + return + } + result <- nil + }() + for { + content, err := os.ReadFile(logPath) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(content), "TASK progress") { + break + } + select { + case <-ctx.Done(): + t.Fatal("progress was not published before process exit") + case err := <-result: + t.Fatalf("process exited before gate release: %v", err) + case <-time.After(10 * time.Millisecond): + } + } + if err := os.WriteFile(gate, nil, 0600); err != nil { + t.Fatal(err) + } + if err := <-result; err != nil { + t.Fatal(err) + } + content, err := os.ReadFile(logPath) + if err != nil || strings.Count(string(content), "TASK progress") != 1 || strings.Count(string(content), "failure detail") != 1 { + t.Fatalf("expected both streams exactly once, got %q (%v)", content, err) + } +} + func TestPrivateKeyPathRejectsExternalSchemes(t *testing.T) { if _, err := privateKeyPath("vault:key"); err == nil { t.Fatal("expected unsupported credential reference") diff --git a/internal/provider/cloud/gcp/catalog.go b/internal/provider/cloud/gcp/catalog.go index 76ab7646..f93fdc42 100644 --- a/internal/provider/cloud/gcp/catalog.go +++ b/internal/provider/cloud/gcp/catalog.go @@ -27,6 +27,14 @@ const ( billingEndpoint = "https://cloudbilling.googleapis.com/v1/services/6F81-5844-456A/skus" ) +type APIError struct { + StatusCode int + Status string + Body string +} + +func (e *APIError) Error() string { return fmt.Sprintf("GCP API returned %s: %s", e.Status, e.Body) } + type Catalog struct { client *http.Client computeEndpoint string @@ -202,7 +210,18 @@ func applyPrices(skus []billingSKU, result *domain.CloudCatalog) { if price == nil || (price.core == 0 && price.memory == 0) { continue } - machine.PricePerHour = price.core*float64(machine.VCPU) + price.memory*(float64(machine.MemoryMiB)/1024) + billableCPU := float64(machine.VCPU) + // Shared-core E2 exposes two guest CPUs but bills its fractional + // sustained allocation, not both guest CPUs as dedicated cores. + switch machine.ProviderTypeID { + case "e2-micro": + billableCPU = 0.25 + case "e2-small": + billableCPU = 0.5 + case "e2-medium": + billableCPU = 1 + } + machine.PricePerHour = price.core*billableCPU + price.memory*(float64(machine.MemoryMiB)/1024) machine.PricePerMinute = machine.PricePerHour / 60 machine.PriceCurrency = "USD" machine.PriceSource = "gcp-public-catalog" @@ -248,13 +267,23 @@ func skuUnitPrice(sku billingSKU) (float64, string) { if len(rates) == 0 { return 0, "" } - price := rates[0].UnitPrice - units, _ := strconv.ParseFloat(price.Units, 64) - return units + float64(price.Nanos)/1e9, price.CurrencyCode + // Standard PD can start with a free-tier allowance. A zero first tier + // is not an unlimited free disk: estimate the first paid marginal tier. + for _, rate := range rates { + price := rate.UnitPrice + units, _ := strconv.ParseFloat(price.Units, 64) + value := units + float64(price.Nanos)/1e9 + if value > 0 { + return value, price.CurrencyCode + } + } + return 0, rates[0].UnitPrice.CurrencyCode } func diskSKUDescription(diskType, description string) bool { - if !strings.Contains(description, "capacity") || strings.Contains(description, "snapshot") { + // Capacity targets use zonal PD boot disks. Regional replicas, Extreme + // PD and Hyperdisk have distinct rates despite matching broad keywords. + if !strings.Contains(description, "capacity") || strings.Contains(description, "snapshot") || strings.Contains(description, "regional") || strings.Contains(description, "extreme") || strings.Contains(description, "hyperdisk") { return false } switch diskType { @@ -336,7 +365,7 @@ func (c *Catalog) get(ctx context.Context, token, endpoint string, output any) e defer response.Body.Close() data, _ := io.ReadAll(io.LimitReader(response.Body, 64<<20)) if response.StatusCode/100 != 2 { - return fmt.Errorf("GCP API returned %s: %s", response.Status, strings.TrimSpace(string(data))) + return &APIError{StatusCode: response.StatusCode, Status: response.Status, Body: strings.TrimSpace(string(data))} } if err := json.Unmarshal(data, output); err != nil { return fmt.Errorf("decode GCP response: %w", err) diff --git a/internal/provider/cloud/gcp/disk_price_match_test.go b/internal/provider/cloud/gcp/disk_price_match_test.go new file mode 100644 index 00000000..253333c5 --- /dev/null +++ b/internal/provider/cloud/gcp/disk_price_match_test.go @@ -0,0 +1,42 @@ +package gcp + +import ( + "encoding/json" + "testing" +) + +func TestZonalBootDiskPricesRejectOtherStorageProducts(t *testing.T) { + for _, pair := range [][2]string{ + {"pd-standard", "extreme pd capacity"}, + {"pd-standard", "regional standard pd capacity"}, + {"pd-balanced", "regional balanced pd capacity"}, + {"pd-balanced", "hyperdisk balanced capacity"}, + } { + if diskSKUDescription(pair[0], pair[1]) { + t.Fatalf("matched other disk product: %v", pair) + } + } + for _, pair := range [][2]string{{"pd-standard", "storage pd capacity"}, {"pd-balanced", "balanced pd capacity"}, {"pd-ssd", "ssd backed pd capacity"}} { + if !diskSKUDescription(pair[0], pair[1]) { + t.Fatalf("rejected zonal disk: %v", pair) + } + } +} + +func TestDiskFreeAllowanceDoesNotHidePaidMarginalRate(t *testing.T) { + var sku billingSKU + if err := json.Unmarshal( + []byte( + ("{\"pricingInfo\":[{\"pricingExpression\":{\"usageUnit\":\"GiBy.mo\",\"tieredRates\":[{\"startUsageAmount\":0,\"un" + + "itPrice\":{\"currencyCode\":\"USD\",\"units\":\"0\",\"nanos\":0}},{\"startUsageAmount\":30,\"unitPrice\":{\"currency" + + "Code\":\"USD\",\"units\":\"0\",\"nanos\":40000000}}]}}]}"), + ), + &sku, + ); err != nil { + t.Fatal(err) + } + price, currency := skuUnitPrice(sku) + if price != 0.04 || currency != "USD" { + t.Fatalf("free allowance treated as unlimited: %v %s", price, currency) + } +} diff --git a/internal/provider/cloud/gcp/host_identity.go b/internal/provider/cloud/gcp/host_identity.go new file mode 100644 index 00000000..51764001 --- /dev/null +++ b/internal/provider/cloud/gcp/host_identity.go @@ -0,0 +1,88 @@ +package gcp + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + + "golang.org/x/crypto/ssh" +) + +var errHostKeyPublicationPending = errors.New("provider-backed host key publication pending") + +// HostIdentity contains only public evidence obtained through authenticated +// Compute API reads. It does not enroll a key or relax existing SSH trust. +type HostIdentity struct { + Generation string + PublicKey string + Fingerprint string +} + +func (c *Catalog) VerifyHostIdentity(ctx context.Context, credential []byte, project, zone, name, expectedGeneration string) (HostIdentity, error) { + if project == "" || zone == "" || name == "" || expectedGeneration == "" { + return HostIdentity{}, fmt.Errorf("cloud host identity requires project, zone, name and generation") + } + var account serviceAccount + if err := json.Unmarshal(credential, &account); err != nil { + return HostIdentity{}, fmt.Errorf("decode GCP credential: %w", err) + } + token, err := c.accessToken(ctx, account) + if err != nil { + return HostIdentity{}, err + } + endpoint := fmt.Sprintf("%s/projects/%s/zones/%s/instances/%s", c.computeEndpoint, url.PathEscape(project), url.PathEscape(zone), url.PathEscape(name)) + var before struct { + ID string `json:"id"` + Name string `json:"name"` + } + if err := c.get(ctx, token, endpoint, &before); err != nil { + return HostIdentity{}, err + } + if before.ID != expectedGeneration || before.Name != name { + return HostIdentity{}, fmt.Errorf("cloud instance generation does not match expected identity") + } + var attributes struct { + QueryValue struct { + Items []struct{ Namespace, Key, Value string } `json:"items"` + } `json:"queryValue"` + } + if err := c.get(ctx, token, endpoint+"/getGuestAttributes?queryPath=hostkeys/", &attributes); err != nil { + var apiError *APIError + if errors.As(err, &apiError) && apiError.StatusCode == 404 { + return HostIdentity{}, errHostKeyPublicationPending + } + return HostIdentity{}, fmt.Errorf("read provider-backed host keys: %w", err) + } + var publicKey ssh.PublicKey + for _, item := range attributes.QueryValue.Items { + if item.Namespace != "hostkeys" || item.Key != "ssh-ed25519" { + continue + } + if publicKey != nil { + return HostIdentity{}, fmt.Errorf("ambiguous provider-backed ED25519 keys") + } + key, _, options, rest, err := ssh.ParseAuthorizedKey([]byte(item.Key + " " + strings.TrimSpace(item.Value))) + if err != nil || len(options) != 0 || len(strings.TrimSpace(string(rest))) != 0 || key.Type() != ssh.KeyAlgoED25519 { + return HostIdentity{}, fmt.Errorf("invalid provider-backed ED25519 key") + } + publicKey = key + } + if publicKey == nil { + return HostIdentity{}, errHostKeyPublicationPending + } + // Fence replacement between the identity and guest-attribute reads. + var after struct { + ID string `json:"id"` + Name string `json:"name"` + } + if err := c.get(ctx, token, endpoint, &after); err != nil { + return HostIdentity{}, err + } + if after.ID != before.ID || after.Name != before.Name { + return HostIdentity{}, fmt.Errorf("cloud instance generation changed during host verification") + } + return HostIdentity{Generation: before.ID, PublicKey: strings.TrimSpace(string(ssh.MarshalAuthorizedKey(publicKey))), Fingerprint: ssh.FingerprintSHA256(publicKey)}, nil +} diff --git a/internal/provider/cloud/gcp/host_identity_test.go b/internal/provider/cloud/gcp/host_identity_test.go new file mode 100644 index 00000000..160cbcee --- /dev/null +++ b/internal/provider/cloud/gcp/host_identity_test.go @@ -0,0 +1,65 @@ +package gcp + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "golang.org/x/crypto/ssh" +) + +func TestVerifyHostIdentityFencesGenerationAndRequiresProviderKey(t *testing.T) { + pub, _, _ := ed25519.GenerateKey(rand.Reader) + key, _ := ssh.NewPublicKey(pub) + encoded := strings.Fields(string(ssh.MarshalAuthorizedKey(key)))[1] + for _, scenario := range []string{"valid", "wrong-generation", "replaced-during-read", "missing-key", "invalid-key", "duplicate-key"} { + t.Run(scenario, func(t *testing.T) { + reads := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/token" { + _, _ = w.Write([]byte(`{"access_token":"token"}`)) + return + } + if r.Header.Get("Authorization") != "Bearer token" { + t.Error("unauthenticated provider read") + } + if strings.HasSuffix(r.URL.Path, "getGuestAttributes") { + items := []map[string]string{{"namespace": "hostkeys", "key": "ssh-ed25519", "value": encoded}} + if scenario == "missing-key" { + items = nil + } + if scenario == "invalid-key" { + items[0]["value"] = "invalid" + } + if scenario == "duplicate-key" { + items = append(items, items[0]) + } + _ = json.NewEncoder(w).Encode(map[string]any{"queryValue": map[string]any{"items": items}}) + return + } + reads++ + id := "123" + if scenario == "wrong-generation" || (scenario == "replaced-during-read" && reads > 1) { + id = "456" + } + _ = json.NewEncoder(w).Encode(map[string]string{"id": id, "name": "worker"}) + })) + defer server.Close() + catalog := New(server.Client()) + catalog.computeEndpoint = server.URL + identity, err := catalog.VerifyHostIdentity(context.Background(), testCredential(t, server.URL+"/token"), "project", "zone", "worker", "123") + if scenario == "valid" { + if err != nil || identity.Generation != "123" || identity.Fingerprint != ssh.FingerprintSHA256(key) { + t.Fatalf("identity=%+v err=%v", identity, err) + } + } else if err == nil { + t.Fatalf("accepted unsafe identity for %s", scenario) + } + }) + } +} diff --git a/internal/provider/cloud/gcp/host_trust.go b/internal/provider/cloud/gcp/host_trust.go new file mode 100644 index 00000000..326635b3 --- /dev/null +++ b/internal/provider/cloud/gcp/host_trust.go @@ -0,0 +1,54 @@ +package gcp + +import ( + "bytes" + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "strings" + + "golang.org/x/crypto/ssh" +) + +// Enroll preserves previous generations and refuses to overwrite a key for an +// existing generation. Callers must obtain identity through VerifyHostIdentity. +func (identity HostIdentity) Enroll(root, logicalID string) (alias, path string, err error) { + if identity.Generation == "" || logicalID == "" { + return "", "", fmt.Errorf("host trust requires logical and provider generation identities") + } + key, _, options, rest, err := ssh.ParseAuthorizedKey([]byte(identity.PublicKey)) + if err != nil || len(options) != 0 || len(strings.TrimSpace(string(rest))) != 0 || key.Type() != ssh.KeyAlgoED25519 || ssh.FingerprintSHA256(key) != identity.Fingerprint { + return "", "", fmt.Errorf("invalid verified host identity") + } + alias = fmt.Sprintf("akoflow-generation-%x", sha256.Sum256([]byte(logicalID+"\x00"+identity.Generation))) + path = filepath.Join(root, alias+".known_hosts") + if err := os.MkdirAll(root, 0700); err != nil { + return "", "", err + } + content := append([]byte(alias+" "), ssh.MarshalAuthorizedKey(key)...) + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if os.IsExist(err) { + info, statErr := os.Lstat(path) + if statErr != nil || !info.Mode().IsRegular() { + return "", "", fmt.Errorf("unsafe host trust record") + } + previous, readErr := os.ReadFile(path) + if readErr != nil || !bytes.Equal(previous, content) { + return "", "", fmt.Errorf("host key changed for verified provider generation") + } + return alias, path, nil + } + if err != nil { + return "", "", err + } + _, writeErr := file.Write(content) + closeErr := file.Close() + if writeErr != nil { + return "", "", writeErr + } + if closeErr != nil { + return "", "", closeErr + } + return alias, path, nil +} diff --git a/internal/provider/cloud/gcp/host_trust_test.go b/internal/provider/cloud/gcp/host_trust_test.go new file mode 100644 index 00000000..bab72d53 --- /dev/null +++ b/internal/provider/cloud/gcp/host_trust_test.go @@ -0,0 +1,37 @@ +package gcp + +import ( + "crypto/ed25519" + "crypto/rand" + "os" + "testing" + + "golang.org/x/crypto/ssh" +) + +func TestEnrollmentPreservesGenerationsAndRejectsChangedKeys(t *testing.T) { + makeIdentity := func(generation string) HostIdentity { + pub, _, _ := ed25519.GenerateKey(rand.Reader) + key, _ := ssh.NewPublicKey(pub) + return HostIdentity{Generation: generation, PublicKey: string(ssh.MarshalAuthorizedKey(key)), Fingerprint: ssh.FingerprintSHA256(key)} + } + root := t.TempDir() + first := makeIdentity("123") + alias, path, err := first.Enroll(root, "worker") + if err != nil { + t.Fatal(err) + } + if _, _, err := first.Enroll(root, "worker"); err != nil { + t.Fatal(err) + } + if _, _, err := makeIdentity("123").Enroll(root, "worker"); err == nil { + t.Fatal("overwrote generation trust") + } + nextAlias, nextPath, err := makeIdentity("456").Enroll(root, "worker") + if err != nil || nextAlias == alias || nextPath == path { + t.Fatalf("replacement enrollment: %s %s %v", nextAlias, nextPath, err) + } + if _, err := os.Stat(path); err != nil { + t.Fatal("previous trust record lost") + } +} diff --git a/internal/provider/cloud/gcp/host_verifier.go b/internal/provider/cloud/gcp/host_verifier.go new file mode 100644 index 00000000..5c555e30 --- /dev/null +++ b/internal/provider/cloud/gcp/host_verifier.go @@ -0,0 +1,49 @@ +package gcp + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" +) + +type HostVerifier struct { + Catalog *Catalog + Root string +} + +func (v HostVerifier) Verify(ctx context.Context, connection domain.EnvironmentConnection, credential []byte, instance domain.CloudProvisionedInstance) (ports.CloudHostTrust, error) { + if instance.Provider != "gcp" || v.Catalog == nil { + return ports.CloudHostTrust{}, fmt.Errorf("provider-backed host verification unavailable") + } + zone, _ := instance.TerraformOutput["zone"].(string) + waitCtx, cancel := context.WithTimeout(ctx, 90*time.Second) + defer cancel() + identity, err := waitForHostKey(waitCtx, 5*time.Second, func(ctx context.Context) (HostIdentity, error) { + return v.Catalog.VerifyHostIdentity(ctx, credential, configString(connection.Configuration, "projectId"), zone, instance.Name, instance.ProviderID) + }) + if err != nil { + return ports.CloudHostTrust{}, err + } + alias, path, err := identity.Enroll(v.Root, instance.ID) + return ports.CloudHostTrust{Alias: alias, KnownHostsFile: path}, err +} + +func waitForHostKey(ctx context.Context, interval time.Duration, read func(context.Context) (HostIdentity, error)) (HostIdentity, error) { + for { + identity, err := read(ctx) + if !errors.Is(err, errHostKeyPublicationPending) { + return identity, err + } + timer := time.NewTimer(interval) + select { + case <-ctx.Done(): + timer.Stop() + return HostIdentity{}, fmt.Errorf("host key publication wait expired: %w", ctx.Err()) + case <-timer.C: + } + } +} diff --git a/internal/provider/cloud/gcp/host_wait_test.go b/internal/provider/cloud/gcp/host_wait_test.go new file mode 100644 index 00000000..077dfef0 --- /dev/null +++ b/internal/provider/cloud/gcp/host_wait_test.go @@ -0,0 +1,36 @@ +package gcp + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestHostKeyWaitRetriesOnlyPublicationPending(t *testing.T) { + calls := 0 + result, err := waitForHostKey(context.Background(), time.Millisecond, func(context.Context) (HostIdentity, error) { + calls++ + if calls == 1 { + return HostIdentity{}, errHostKeyPublicationPending + } + return HostIdentity{Generation: "123"}, nil + }) + if err != nil || calls != 2 || result.Generation != "123" { + t.Fatalf("calls=%d result=%+v err=%v", calls, result, err) + } + calls = 0 + _, err = waitForHostKey(context.Background(), time.Millisecond, func(context.Context) (HostIdentity, error) { + calls++ + return HostIdentity{}, errors.New("generation mismatch") + }) + if err == nil || calls != 1 { + t.Fatal("retried identity mismatch") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = waitForHostKey(ctx, time.Hour, func(context.Context) (HostIdentity, error) { return HostIdentity{}, errHostKeyPublicationPending }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancel: %v", err) + } +} diff --git a/internal/provider/cloud/gcp/shared_core_price_test.go b/internal/provider/cloud/gcp/shared_core_price_test.go new file mode 100644 index 00000000..fc7d63e0 --- /dev/null +++ b/internal/provider/cloud/gcp/shared_core_price_test.go @@ -0,0 +1,23 @@ +package gcp + +import ( + "math" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestE2SharedCorePricesUseFractionalAllocationNotGuestCPUCount(t *testing.T) { + catalog := domain.CloudCatalog{Region: "us-central1", Machines: []domain.CloudMachineOffering{ + {ProviderTypeID: "e2-micro", Family: "e2", VCPU: 2, MemoryMiB: 1024}, + {ProviderTypeID: "e2-small", Family: "e2", VCPU: 2, MemoryMiB: 2048}, + {ProviderTypeID: "e2-medium", Family: "e2", VCPU: 2, MemoryMiB: 4096}, + {ProviderTypeID: "e2-standard-2", Family: "e2", VCPU: 2, MemoryMiB: 8192}, + }} + applyPrices([]billingSKU{priceSKU("E2 Instance Core running in Americas", "CPU", 0.02), priceSKU("E2 Instance Ram running in Americas", "RAM", 0.003)}, &catalog) + for i, want := range []float64{0.008, 0.016, 0.032, 0.064} { + if got := catalog.Machines[i].PricePerHour; math.Abs(got-want) > 1e-12 { + t.Fatalf("%s: price=%f want=%f", catalog.Machines[i].ProviderTypeID, got, want) + } + } +} diff --git a/internal/provider/cloud/runtime/factory.go b/internal/provider/cloud/runtime/factory.go index 5be56f41..42dfe00e 100644 --- a/internal/provider/cloud/runtime/factory.go +++ b/internal/provider/cloud/runtime/factory.go @@ -131,13 +131,17 @@ func (a *Adapter) delegateForHandle( } func (a *Adapter) remoteAdapter(instance domain.CloudProvisionedInstance) (ports.RuntimeAdapter, error) { + alias, knownHosts, err := instance.VerifiedSSHTrust() + if err != nil { + return nil, err + } connection := domain.EnvironmentConnection{ ID: a.environmentID + "-" + instance.ID, EnvironmentID: a.environmentID, Name: instance.Name, Type: domain.ConnectionSSH, Endpoint: instance.PublicAddress, Username: instance.SSHUsername, CredentialRef: instance.SSHCredentialRef, Configuration: map[string]any{ "adapter": "ssh-docker", "port": 22, "dynamicHost": true, - "hostKeyAlias": instance.ID, + "hostKeyAlias": alias, "knownHostsFile": knownHosts, "strictHostKeyChecking": true, }, } return (remote.Factory{Executor: a.executor}).Build(domain.EnvironmentRuntime{}, connection) diff --git a/internal/provider/cloud/runtime/factory_test.go b/internal/provider/cloud/runtime/factory_test.go index 6a803559..50b5a35b 100644 --- a/internal/provider/cloud/runtime/factory_test.go +++ b/internal/provider/cloud/runtime/factory_test.go @@ -91,6 +91,7 @@ func (s *executorStub) Run(context.Context, string, []string, []byte) ([]byte, e func TestCloudRuntimeUsesAllocatedInstanceAndStartsRemoteDocker(t *testing.T) { provisioner := &cloudProvisionerStub{created: domain.CloudProvisionedInstance{ ID: "instance", CapacityTargetID: "target", EnvironmentID: "environment", Status: "ready", + ProviderID: "123", TerraformOutput: map[string]any{"verifiedHostGeneration": "123", "verifiedHostKeyAlias": "verified-generation", "verifiedKnownHostsFile": t.TempDir() + "/known_hosts"}, PublicAddress: "203.0.113.10", SSHUsername: "akoflow", SSHCredentialRef: "file:/key", }} store := &cloudStoreStub{instances: []domain.CloudProvisionedInstance{provisioner.created}} @@ -111,6 +112,18 @@ func TestCloudRuntimeUsesAllocatedInstanceAndStartsRemoteDocker(t *testing.T) { } } +func TestCloudRuntimeRejectsUnverifiedOrStaleGeneration(t *testing.T) { + adapter := &Adapter{executor: &executorStub{}} + for _, instance := range []domain.CloudProvisionedInstance{ + {ID: "worker", ProviderID: "123"}, + {ID: "worker", ProviderID: "456", TerraformOutput: map[string]any{"verifiedHostGeneration": "123", "verifiedHostKeyAlias": "old", "verifiedKnownHostsFile": "old"}}, + } { + if _, err := adapter.remoteAdapter(instance); err == nil { + t.Fatal("accepted unverified cloud generation") + } + } +} + func TestCloudRuntimeRejectsMissingConcreteAllocation(t *testing.T) { adapter := &Adapter{environmentID: "environment", store: &cloudStoreStub{}, provisioner: &cloudProvisionerStub{}} if _, err := adapter.Start(context.Background(), domain.ActivityExecutionContext{Resource: domain.Resource{ID: "target"}}); err == nil { diff --git a/internal/provider/cloud/terraform/runner.go b/internal/provider/cloud/terraform/runner.go index a9ebfa0d..22f6e18d 100644 --- a/internal/provider/cloud/terraform/runner.go +++ b/internal/provider/cloud/terraform/runner.go @@ -115,10 +115,27 @@ func (r Runner) setDesiredStatus(ctx context.Context, instanceID, status string) if err := os.WriteFile(variablesPath, encoded, 0600); err != nil { return ports.TerraformResult{}, err } - if _, err = r.run( - ctx, workspace, "apply", "-auto-approve", "-no-color", "-input=false", - "-var-file=terraform.tfvars.json", - ); err != nil { + plan, err := os.CreateTemp(workspace, "lifecycle-*.tfplan") + if err != nil { + return ports.TerraformResult{}, err + } + planPath := plan.Name() + if err = plan.Close(); err != nil { + os.Remove(planPath) + return ports.TerraformResult{}, err + } + defer os.Remove(planPath) + if _, err = r.run(ctx, workspace, "plan", "-no-color", "-input=false", "-out="+planPath, "-var-file=terraform.tfvars.json"); err != nil { + return ports.TerraformResult{}, err + } + planJSON, err := r.run(ctx, workspace, "show", "-json", planPath) + if err != nil { + return ports.TerraformResult{}, err + } + if err = validateLifecyclePlan(planJSON); err != nil { + return ports.TerraformResult{}, err + } + if _, err = r.run(ctx, workspace, "apply", "-auto-approve", "-no-color", "-input=false", planPath); err != nil { return ports.TerraformResult{}, err } output, err := r.run(ctx, workspace, "output", "-json") @@ -128,6 +145,39 @@ func (r Runner) setDesiredStatus(ctx context.Context, instanceID, status string) return parseOutput(output) } +// Lifecycle operations must never silently replace a configured worker or its +// disk. Apply the inspected saved plan, rather than planning again at apply time. +func validateLifecyclePlan(encoded []byte) error { + var plan struct { + ResourceChanges []struct { + Address string `json:"address"` + Mode string `json:"mode"` + Change struct { + Actions []string `json:"actions"` + } `json:"change"` + } `json:"resource_changes"` + } + if err := json.Unmarshal(encoded, &plan); err != nil { + return fmt.Errorf("decode cloud lifecycle plan: %w", err) + } + worker := false + for _, resource := range plan.ResourceChanges { + if resource.Mode == "data" { + continue + } + if resource.Address == "google_compute_instance.worker" { + worker = true + } + if len(resource.Change.Actions) != 1 || (resource.Change.Actions[0] != "no-op" && !(resource.Address == "google_compute_instance.worker" && resource.Change.Actions[0] == "update")) { + return fmt.Errorf("unsafe cloud lifecycle plan for %s: actions %v; explicit provision/configuration required", resource.Address, resource.Change.Actions) + } + } + if !worker { + return fmt.Errorf("cloud lifecycle plan has no existing worker; explicit provision/configuration required") + } + return nil +} + func (r Runner) prepare(spec ports.TerraformProvisionSpec) (string, error) { workspace, err := r.workspace(spec.InstanceID) if err != nil { @@ -218,7 +268,8 @@ func (r Runner) run(ctx context.Context, workspace string, arguments ...string) writeProvisionLog(logFile, "Terraform", "terraform %s", strings.Join(arguments, " ")) _ = logFile.Sync() output, err := r.execute(ctx, workspace, append([]string{binary}, arguments...)...) - if len(output) > 0 { + privatePlan := len(arguments) > 1 && arguments[0] == "show" && arguments[1] == "-json" + if len(output) > 0 && !privatePlan { _, _ = logFile.Write(output) if output[len(output)-1] != '\n' { _, _ = logFile.WriteString("\n") @@ -227,6 +278,9 @@ func (r Runner) run(ctx context.Context, workspace string, arguments ...string) if err != nil { writeProvisionLog(logFile, "Terraform", "failed after %.3fs: %v", time.Since(startedAt).Seconds(), err) _ = logFile.Sync() + if privatePlan { + return nil, fmt.Errorf("terraform show plan: %w", err) + } return nil, fmt.Errorf("terraform %s: %w: %s", arguments[0], err, strings.TrimSpace(string(output))) } writeProvisionLog(logFile, "Terraform", "completed in %.3fs", time.Since(startedAt).Seconds()) @@ -411,7 +465,30 @@ resource "google_compute_instance" "worker" { subnetwork = var.subnetwork == "" ? null : var.subnetwork access_config {} } - metadata = { ssh-keys = "${var.ssh_user}:${var.ssh_public_key}" } + metadata = { + ssh-keys = "${var.ssh_user}:${var.ssh_public_key}" + enable-guest-attributes = "TRUE" + # Existing images might not publish hostkeys automatically when guest + # attributes were enabled after their first boot. Publish only the public + # ED25519 key through the instance metadata channel on startup. + startup-script = <<-SCRIPT + #!/bin/sh + set -eu + key_file=/etc/ssh/ssh_host_ed25519_key.pub + attempt=0 + while [ ! -s "$key_file" ] && [ "$attempt" -lt 30 ]; do + attempt=$((attempt + 1)) + sleep 1 + done + test -s "$key_file" + key=$(awk '$1 == "ssh-ed25519" {print $2}' "$key_file") + test -n "$key" + curl --fail --silent --show-error --connect-timeout 5 --max-time 15 \ + --retry 5 --retry-delay 2 -X PUT -H 'Metadata-Flavor: Google' \ + --data "$key" \ + http://metadata.google.internal/computeMetadata/v1/instance/guest-attributes/hostkeys/ssh-ed25519 + SCRIPT + } scheduling { provisioning_model = var.spot ? "SPOT" : "STANDARD" preemptible = var.spot diff --git a/internal/provider/cloud/terraform/runner_test.go b/internal/provider/cloud/terraform/runner_test.go index 8c7a41cc..26e218b0 100644 --- a/internal/provider/cloud/terraform/runner_test.go +++ b/internal/provider/cloud/terraform/runner_test.go @@ -30,6 +30,67 @@ func TestParseOutput(t *testing.T) { } } +func TestLifecycleRejectsCreationReplacementDeletionAndMissingWorker(t *testing.T) { + for _, actions := range []string{`["create"]`, `["delete","create"]`, `["delete"]`, `[]`} { + encoded := []byte(`{"resource_changes":[{"address":"google_compute_instance.worker","mode":"managed","change":{"actions":` + actions + `}}]}`) + if err := validateLifecyclePlan(encoded); err == nil { + t.Fatalf("accepted unsafe actions %s", actions) + } + } + if err := validateLifecyclePlan([]byte(`{}`)); err == nil { + t.Fatal("accepted missing worker") + } +} + +func TestLifecycleUsesValidatedSavedPlanAndHidesPlanPayload(t *testing.T) { + for _, actions := range []string{`["update"]`, `["create"]`} { + t.Run(actions, func(t *testing.T) { + root := t.TempDir() + workspace := filepath.Join(root, "cloud-instance-worker") + if err := os.Mkdir(workspace, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(workspace, "terraform.tfvars.json"), []byte(`{}`), 0600); err != nil { + t.Fatal(err) + } + var applied bool + runner := Runner{Root: root, Execute: func(_ context.Context, _ string, _ []string, args ...string) ([]byte, error) { + switch args[1] { + case "show": + return []byte(`{"variables":{"secret":"must-not-be-logged"},"resource_changes":[{"address":"google_compute_instance.worker","mode":"managed","change":{"actions":` + actions + `}}]}`), nil + case "apply": + applied = true + if !strings.HasSuffix(args[len(args)-1], ".tfplan") { + t.Fatal("apply must use saved plan") + } + case "output": + return []byte(`{"instance_id":{"value":"original"},"public_ip":{"value":"203.0.113.1"}}`), nil + } + return nil, nil + }} + _, err := runner.Start(context.Background(), "cloud-instance-worker") + if actions == `["create"]` { + if err == nil || applied { + t.Fatal("unsafe plan reached apply") + } + } else if err != nil || !applied { + t.Fatalf("safe update rejected: %v", err) + } + log, err := os.ReadFile(filepath.Join(workspace, "provision.log")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(log), "must-not-be-logged") { + t.Fatal("plan payload leaked") + } + files, _ := filepath.Glob(filepath.Join(workspace, "lifecycle-*.tfplan")) + if len(files) != 0 { + t.Fatal("saved plan retained") + } + }) + } +} + func TestGeneratedGCPModuleIsFormatted(t *testing.T) { if _, err := exec.LookPath("terraform"); err != nil { t.Skip("terraform is not installed") diff --git a/internal/provider/command.go b/internal/provider/command.go index 3db303e2..4ab30caf 100644 --- a/internal/provider/command.go +++ b/internal/provider/command.go @@ -18,6 +18,9 @@ type OSCommandExecutor struct{} func (OSCommandExecutor) Run(ctx context.Context, name string, args []string, input []byte) ([]byte, error) { command := exec.CommandContext(ctx, name, args...) + // ProxyCommand grandchildren can retain stdout/stderr after ssh exits or + // is cancelled. Bound pipe draining without limiting a live command's work. + command.WaitDelay = 2 * time.Second command.Stdin = bytes.NewReader(input) output, err := command.CombinedOutput() if err != nil { diff --git a/internal/provider/command_wait_test.go b/internal/provider/command_wait_test.go new file mode 100644 index 00000000..b220dfe6 --- /dev/null +++ b/internal/provider/command_wait_test.go @@ -0,0 +1,36 @@ +package provider + +import ( + "context" + "errors" + "os" + "os/exec" + "runtime" + "strconv" + "strings" + "testing" + "time" +) + +func TestCommandBoundsOrphanedOutputPipeAfterParentExit(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX child process fixture") + } + started := time.Now() + output, err := (OSCommandExecutor{}).Run(context.Background(), "sh", []string{"-c", "sleep 30 & printf '%s\\n' \"$!\""}, nil) + // Kill only the exact child created by this fixture, never other processes. + pid, parseErr := strconv.Atoi(strings.TrimSpace(string(output))) + if parseErr != nil { + t.Fatalf("missing fixture child: %q %v", output, parseErr) + } + process, findErr := os.FindProcess(pid) + if findErr == nil { + defer process.Kill() + } + if !errors.Is(err, exec.ErrWaitDelay) { + t.Fatalf("expected pipe deadline, got %v", err) + } + if time.Since(started) > 5*time.Second { + t.Fatal("orphan pipe held executor") + } +} diff --git a/internal/provider/local/adapter.go b/internal/provider/local/adapter.go index 49a27e71..30cce791 100644 --- a/internal/provider/local/adapter.go +++ b/internal/provider/local/adapter.go @@ -85,11 +85,12 @@ func (a *Adapter) Start(ctx context.Context, execution domain.ActivityExecutionC return domain.ActivityHandle{}, fmt.Errorf("start local activity: %w", err) } startedAt := runtimecommon.UnixSeconds(time.Now()) - handle := domain.ActivityHandle{ID: execution.Run.ID + ":" + activity.ID, RunID: execution.Run.ID, + handle := domain.ActivityHandle{ID: execution.HandleID(), RunID: execution.Run.ID, ActivityID: activity.ID, ResourceID: execution.Resource.ID, RuntimeID: execution.RuntimeID, ExternalID: strconv.Itoa(command.Process.Pid), Status: domain.HandleRunning, StartedAt: startedAt, Endpoints: localEndpoints(activity), Metadata: map[string]any{ + "executionAttempt": execution.Attempt, domain.TimingSubmittedAt: startedAt, "artifactObservationDriver": "filesystem-diff", "artifactObservationRoot": root, @@ -121,6 +122,9 @@ func (a *Adapter) Start(ctx context.Context, execution domain.ActivityExecutionC func (a *Adapter) startContainer(ctx context.Context, execution domain.ActivityExecutionContext, root string, before runtimecommon.ArtifactSnapshot, image string) (domain.ActivityHandle, error) { activity := execution.Activity name := "akoflow-" + strings.NewReplacer(":", "-", "/", "-").Replace(execution.Run.ID+"-"+activity.ID) + if execution.Attempt > 1 { + name += fmt.Sprintf("-attempt-%d", execution.Attempt) + } args := []string{"run", "--detach", "--name", name, "--label", "akoflow.run=" + execution.Run.ID} if volume := os.Getenv("AKOFLOW_LOCAL_WORKSPACE_VOLUME"); volume != "" { mountRoot := os.Getenv("AKOFLOW_LOCAL_WORKSPACE_ROOT") @@ -151,7 +155,7 @@ func (a *Adapter) startContainer(ctx context.Context, execution domain.ActivityE return domain.ActivityHandle{}, fmt.Errorf("start local activity: missing container ID") } startedAt := runtimecommon.UnixSeconds(time.Now()) - handle := domain.ActivityHandle{ID: execution.Run.ID + ":" + activity.ID, RunID: execution.Run.ID, + handle := domain.ActivityHandle{ID: execution.HandleID(), RunID: execution.Run.ID, ActivityID: activity.ID, ResourceID: execution.Resource.ID, RuntimeID: execution.RuntimeID, ExternalID: containerID, Status: domain.HandleRunning, StartedAt: startedAt, Endpoints: localEndpoints(activity), Metadata: map[string]any{ diff --git a/internal/provider/remote/adapter.go b/internal/provider/remote/adapter.go index 2f9fd78f..8c55a677 100644 --- a/internal/provider/remote/adapter.go +++ b/internal/provider/remote/adapter.go @@ -6,6 +6,7 @@ import ( "context" "encoding/json" "fmt" + "math" "sort" "strconv" "strings" @@ -56,37 +57,27 @@ func (a *Adapter) Start(ctx context.Context, execution domain.ActivityExecutionC return domain.ActivityHandle{}, fmt.Errorf("activity image is required for remote Docker execution") } name := "akoflow-" + safeName(execution.Run.ID) + "-" + safeName(activity.ID) + if execution.Attempt > 1 { + name += "-attempt-" + strconv.Itoa(execution.Attempt) + } workingDirectory := strings.TrimSpace(activity.Command.WorkingDirectory) if workingDirectory == "" { workingDirectory = "/akoflow/workspace/runs/" + safeName(execution.Run.ID) + "/" + safeName(activity.ID) } + workspaceStart := time.Now() if _, err := a.executor.Run(ctx, "mkdir", []string{"-p", workingDirectory}, nil); err != nil { return domain.ActivityHandle{}, fmt.Errorf("prepare remote workspace: %w", err) } - before, err := a.snapshot(ctx, workingDirectory) + workspaceSeconds := time.Since(workspaceStart).Seconds() + snapshotStart := time.Now() + before, snapshotServiceSeconds, err := a.measuredSnapshot(ctx, workingDirectory) if err != nil { return domain.ActivityHandle{}, fmt.Errorf("snapshot remote workspace: %w", err) } beforeJSON, _ := json.Marshal(before) - args := []string{"run", "--detach", "--name", name, "--label", "akoflow.run=" + execution.Run.ID} - for key, value := range activity.Command.Environment { - args = append(args, "--env", key+"="+value) - } - if activity.Resources.CPU > 0 { - args = append(args, "--cpus", strconv.FormatFloat(activity.Resources.CPU, 'f', -1, 64)) - } - if activity.Resources.MemoryBytes > 0 { - args = append(args, "--memory", strconv.FormatInt(activity.Resources.MemoryBytes, 10)) - } - args = append(args, "--volume", workingDirectory+":"+workingDirectory, "--workdir", workingDirectory) - args = append(args, activity.Command.Image) - if seed, _ := activity.Metadata["workspaceSeedPath"].(string); seed != "" { - args = append(args, "sh", "-c", `cp -an "$1"/. "$2"/ && cd "$2" && shift 2 && exec "$@"`, "akoflow-seed", seed, workingDirectory) - } - if activity.Command.Entrypoint != "" { - args = append(args, activity.Command.Entrypoint) - } - args = append(args, activity.Command.Arguments...) + snapshotSeconds := time.Since(snapshotStart).Seconds() + args := remoteDockerArguments(execution, name, workingDirectory) + containerLaunchStart := time.Now() output, err := a.executor.Run(ctx, "docker", args, nil) if err != nil { return domain.ActivityHandle{}, fmt.Errorf("start remote Docker container: %w", err) @@ -100,18 +91,51 @@ func (a *Adapter) Start(ctx context.Context, execution domain.ActivityExecutionC return domain.ActivityHandle{}, fmt.Errorf("start remote Docker container: missing container ID") } now := runtimecommon.UnixSeconds(time.Now()) + containerLaunchSeconds := time.Since(containerLaunchStart).Seconds() handle := domain.ActivityHandle{ID: runtimecommon.NewID("activity"), RunID: execution.Run.ID, ActivityID: activity.ID, ResourceID: execution.Resource.ID, RuntimeID: execution.RuntimeID, ExternalID: containerID, Status: domain.HandleStarting, StartedAt: now, Metadata: map[string]any{ - "containerName": name, "executionTarget": "remote-docker", "artifactObservationRoot": workingDirectory, - "artifactObservationBefore": string(beforeJSON), + "executionAttempt": execution.Attempt, + "containerName": name, "executionTarget": "remote-docker", "artifactObservationRoot": workingDirectory, + "artifactObservationBefore": string(beforeJSON), + "remoteWorkspacePrepareSeconds": workspaceSeconds, + "remoteWorkspaceSnapshotSeconds": snapshotSeconds, + "remoteContainerLaunchSeconds": containerLaunchSeconds, }} + if snapshotServiceSeconds > 0 { + handle.Metadata["remoteWorkspaceSnapshotServiceSeconds"] = snapshotServiceSeconds + } + telemetryStart := time.Now() telemetry.ObserveDocker(ctx, &handle, containerID, a.executor.Run) + handle.Metadata["remoteInitialTelemetrySeconds"] = time.Since(telemetryStart).Seconds() return handle, nil } +func remoteDockerArguments(execution domain.ActivityExecutionContext, name, workingDirectory string) []string { + activity := execution.Activity + args := []string{"run", "--detach", "--name", name, "--label", "akoflow.run=" + execution.Run.ID} + for key, value := range activity.Command.Environment { + args = append(args, "--env", key+"="+value) + } + if activity.Resources.CPU > 0 { + args = append(args, "--cpus", strconv.FormatFloat(activity.Resources.CPU, 'f', -1, 64)) + } + if activity.Resources.MemoryBytes > 0 { + args = append(args, "--memory", strconv.FormatInt(activity.Resources.MemoryBytes, 10)) + } + args = append(args, "--volume", workingDirectory+":"+workingDirectory, "--workdir", workingDirectory) + args = append(args, activity.Command.Image) + if seed, _ := activity.Metadata["workspaceSeedPath"].(string); seed != "" { + args = append(args, "sh", "-c", `cp -an "$1"/. "$2"/ && cd "$2" && shift 2 && exec "$@"`, "akoflow-seed", seed, workingDirectory) + } + if activity.Command.Entrypoint != "" { + args = append(args, activity.Command.Entrypoint) + } + return append(args, activity.Command.Arguments...) +} + func (a *Adapter) Inspect(ctx context.Context, handle domain.ActivityHandle) (domain.ActivityHandle, error) { - output, err := a.executor.Run(ctx, "docker", []string{"inspect", "--format", "{{.State.Status}}|{{.State.ExitCode}}", handle.ExternalID}, nil) + output, err := a.executor.Run(ctx, "docker", []string{"inspect", "--format", "{{.State.Status}}|{{.State.ExitCode}}|{{.State.FinishedAt}}", handle.ExternalID}, nil) if err != nil { return handle, err } @@ -124,6 +148,18 @@ func (a *Adapter) Inspect(ctx context.Context, handle domain.ActivityHandle) (do code, _ := strconv.Atoi(parts[1]) handle.ExitCode = &code handle.FinishedAt = runtimecommon.UnixSeconds(time.Now()) + if handle.Metadata == nil { + handle.Metadata = map[string]any{} + } + handle.Metadata["terminalObservedAt"] = handle.FinishedAt + if len(parts) > 2 { + if finished, parseErr := time.Parse(time.RFC3339Nano, parts[2]); parseErr == nil && !finished.IsZero() { + native := runtimecommon.UnixSeconds(finished) + if native >= handle.StartedAt { + handle.FinishedAt = native + } + } + } if code == 0 { handle.Status = domain.HandleCompleted } else { @@ -132,9 +168,15 @@ func (a *Adapter) Inspect(ctx context.Context, handle domain.ActivityHandle) (do } root, _ := handle.Metadata["artifactObservationRoot"].(string) if root != "" { + observationStart := time.Now() + handle.Metadata["remoteArtifactObservationStartedAt"] = runtimecommon.UnixSeconds(observationStart) artifacts, observeErr := a.collectArtifacts(ctx, handle, code) + observationFinish := time.Now() + handle.Metadata["remoteArtifactObservationFinishedAt"] = runtimecommon.UnixSeconds(observationFinish) + handle.Metadata["remoteArtifactObservationSeconds"] = observationFinish.Sub(observationStart).Seconds() if observeErr == nil { handle.Artifacts = artifacts + handle.Metadata["remoteArtifactsReadyAt"] = runtimecommon.UnixSeconds(observationFinish) } else { if handle.Metadata == nil { handle.Metadata = map[string]any{} @@ -157,7 +199,15 @@ func (a *Adapter) Inspect(ctx context.Context, handle domain.ActivityHandle) (do } func (a *Adapter) snapshot(ctx context.Context, root string) ([]remoteArtifactFile, error) { - const script = `import hashlib,json,os,sys + files, _, err := a.measuredSnapshot(ctx, root) + return files, err +} + +// Service time is measured on the remote host, excluding SSH admission and +// connection establishment. Outer wall time remains a separate diagnostic. +func (a *Adapter) measuredSnapshot(ctx context.Context, root string) ([]remoteArtifactFile, float64, error) { + const script = `import hashlib,json,os,sys,time +started=time.monotonic() root=sys.argv[1] result=[] for base,dirs,files in os.walk(root): @@ -170,16 +220,31 @@ for base,dirs,files in os.walk(root): for chunk in iter(lambda: stream.read(1024*1024),b''): h.update(chunk) stat=os.stat(full) result.append({'path':os.path.relpath(full,root).replace(os.sep,'/'),'size':stat.st_size,'checksum':h.hexdigest(),'modified':stat.st_mtime_ns}) -print(json.dumps(result,separators=(',',':')))` +print(json.dumps({'files':result,'serviceSeconds':time.monotonic()-started},separators=(',',':')))` output, err := a.executor.Run(ctx, "python3", []string{"-c", script, root}, nil) if err != nil { - return nil, err + return nil, 0, err } var files []remoteArtifactFile - if err := json.Unmarshal(output, &files); err != nil { - return nil, fmt.Errorf("decode remote artifact snapshot: %w", err) + // Accept legacy array responses for existing executors and fixtures; only + // the envelope carries independently measured service time. + if strings.HasPrefix(strings.TrimSpace(string(output)), "[") { + if err := json.Unmarshal(output, &files); err != nil { + return nil, 0, fmt.Errorf("decode remote artifact snapshot: %w", err) + } + return files, 0, nil + } + var measured struct { + Files []remoteArtifactFile `json:"files"` + ServiceSeconds *float64 `json:"serviceSeconds"` } - return files, nil + if err := json.Unmarshal(output, &measured); err != nil { + return nil, 0, fmt.Errorf("decode remote artifact snapshot: %w", err) + } + if measured.Files == nil || measured.ServiceSeconds == nil || *measured.ServiceSeconds < 0 || math.IsNaN(*measured.ServiceSeconds) || math.IsInf(*measured.ServiceSeconds, 0) { + return nil, 0, fmt.Errorf("invalid remote artifact snapshot measurement") + } + return measured.Files, *measured.ServiceSeconds, nil } func (a *Adapter) collectArtifacts(ctx context.Context, handle domain.ActivityHandle, exitCode int) (*domain.ArtifactManifest, error) { @@ -194,10 +259,13 @@ func (a *Adapter) collectArtifacts(ctx context.Context, handle domain.ActivityHa return nil, fmt.Errorf("decode initial remote snapshot: %w", err) } } - after, err := a.snapshot(ctx, root) + after, serviceSeconds, err := a.measuredSnapshot(ctx, root) if err != nil { return nil, err } + if handle.Metadata != nil && serviceSeconds > 0 { + handle.Metadata["remoteArtifactObservationServiceSeconds"] = serviceSeconds + } return remoteArtifactManifest(handle, root, exitCode, before, after), nil } diff --git a/internal/provider/remote/adapter_test.go b/internal/provider/remote/adapter_test.go index 59a55f6f..f72f5ff1 100644 --- a/internal/provider/remote/adapter_test.go +++ b/internal/provider/remote/adapter_test.go @@ -5,15 +5,91 @@ import ( "fmt" "strings" "testing" + "time" "github.com/UFFeScience/akoflow/internal/domain" provider "github.com/UFFeScience/akoflow/internal/provider" ) +func TestMeasuredSnapshotSeparatesRemoteServiceFromTransportWall(t *testing.T) { + executor := &commandExecutorStub{responses: [][]byte{[]byte(`{"files":[],"serviceSeconds":0.001}`)}, delays: []time.Duration{20 * time.Millisecond}} + adapter := &Adapter{executor: executor} + started := time.Now() + files, service, err := adapter.measuredSnapshot(context.Background(), "/workspace") + if err != nil || len(files) != 0 || service != .001 || time.Since(started).Seconds() <= service { + t.Fatalf("service and transport wall conflated: %v %g %v", files, service, err) + } +} + +func TestMeasuredSnapshotRejectsIncompleteOrNegativeService(t *testing.T) { + for _, response := range []string{`{}`, `{"files":[]}`, `{"files":[],"serviceSeconds":-1}`} { + adapter := &Adapter{executor: &commandExecutorStub{responses: [][]byte{[]byte(response)}}} + if _, _, err := adapter.measuredSnapshot(context.Background(), "/workspace"); err == nil { + t.Fatalf("accepted invalid measurement %s", response) + } + } +} + +func TestTerminalArtifactObservationPublishesQueueFreeService(t *testing.T) { + finished := time.Now().UTC().Add(-time.Minute) + executor := &commandExecutorStub{responses: [][]byte{ + []byte("exited|0|" + finished.Format(time.RFC3339Nano)), + []byte(`{"files":[],"serviceSeconds":0.001}`), nil, + }, delays: []time.Duration{0, 20 * time.Millisecond}} + adapter := &Adapter{executor: executor} + handle, err := adapter.Inspect(context.Background(), domain.ActivityHandle{ + ExternalID: "container", + StartedAt: float64(finished.UnixNano())/1e9 - 10, + Metadata: map[string]any{"artifactObservationRoot": "/workspace", "artifactObservationBefore": "[]"}, + }) + if err != nil || handle.Metadata["remoteArtifactObservationServiceSeconds"] != .001 || handle.Metadata["remoteArtifactObservationSeconds"].(float64) <= .001 { + t.Fatalf("terminal service/wall evidence: %+v %v", handle, err) + } +} + +func TestRemoteCompletionUsesNativeTimeNotInspectionDelay(t *testing.T) { + finished := time.Now().UTC().Add(-time.Minute) + executor := &commandExecutorStub{responses: [][]byte{[]byte("exited|0|" + finished.Format(time.RFC3339Nano))}} + adapter := &Adapter{executor: executor} + handle, err := adapter.Inspect(context.Background(), domain.ActivityHandle{ExternalID: "container", StartedAt: float64(finished.UnixNano())/1e9 - 10}) + if err != nil { + t.Fatal(err) + } + if handle.FinishedAt != float64(finished.UnixNano())/1e9 || handle.Metadata["terminalObservedAt"].(float64) <= handle.FinishedAt { + t.Fatalf("native/observed completion conflated: %+v", handle) + } +} + +func TestRemoteObservationTimingDoesNotExtendNativeCompute(t *testing.T) { + finished := time.Now().UTC().Add(-time.Minute) + executor := &commandExecutorStub{ + responses: [][]byte{[]byte("exited|0|" + finished.Format(time.RFC3339Nano)), []byte("[]"), nil}, + delays: []time.Duration{0, 20 * time.Millisecond}, + } + adapter := &Adapter{executor: executor} + handle, err := adapter.Inspect(context.Background(), domain.ActivityHandle{ + ExternalID: "container", StartedAt: float64(finished.UnixNano())/1e9 - 10, + Metadata: map[string]any{"artifactObservationRoot": "/workspace", "artifactObservationBefore": "[]"}, + }) + if err != nil || handle.Status != domain.HandleCompleted { + t.Fatalf("inspect: %+v %v", handle, err) + } + if handle.FinishedAt != float64(finished.UnixNano())/1e9 { + t.Fatal("observation inflated command runtime") + } + if handle.Metadata["remoteArtifactObservationSeconds"].(float64) < .02 { + t.Fatal("missing observation service duration") + } + if handle.Metadata["remoteArtifactsReadyAt"].(float64) <= handle.FinishedAt { + t.Fatal("publication must remain separate from native completion") + } +} + type commandExecutorStub struct { calls []commandCall responses [][]byte errors []error + delays []time.Duration } type commandCall struct { name string @@ -23,6 +99,9 @@ type commandCall struct { func (s *commandExecutorStub) Run(_ context.Context, name string, args []string, _ []byte) ([]byte, error) { s.calls = append(s.calls, commandCall{name: name, args: append([]string(nil), args...)}) index := len(s.calls) - 1 + if index < len(s.delays) { + time.Sleep(s.delays[index]) + } var response []byte var err error if index < len(s.responses) { @@ -49,6 +128,33 @@ func executionFixture() domain.ActivityExecutionContext { } } +func TestRecoveryUsesAttemptScopedContainerName(t *testing.T) { + executor := &commandExecutorStub{responses: [][]byte{nil, []byte("[]"), []byte("container-id\n")}} + execution := executionFixture() + execution.Attempt = 2 + handle, err := (&Adapter{executor: executor}).Start(context.Background(), execution) + if err != nil || handle.Metadata["containerName"] != "akoflow-run-one-activity-one-attempt-2" { + t.Fatalf("handle=%+v err=%v", handle, err) + } +} + +func TestStartRecordsIndependentRemoteLaunchStages(t *testing.T) { + executor := &commandExecutorStub{ + responses: [][]byte{nil, []byte("[]"), []byte("container-id\n")}, + delays: []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 15 * time.Millisecond, 20 * time.Millisecond}, + } + handle, err := (&Adapter{executor: executor}).Start(context.Background(), executionFixture()) + if err != nil { + t.Fatal(err) + } + for index, key := range []string{"remoteWorkspacePrepareSeconds", "remoteWorkspaceSnapshotSeconds", "remoteContainerLaunchSeconds", "remoteInitialTelemetrySeconds"} { + seconds, ok := handle.Metadata[key].(float64) + if !ok || seconds < executor.delays[index].Seconds() { + t.Fatalf("stage %s did not account for its own executor delay: %v", key, handle.Metadata[key]) + } + } +} + func TestFactoryValidatesDirectDockerConnection(t *testing.T) { factory := Factory{} if factory.Driver() != domain.RuntimeDriverSSH { diff --git a/internal/provider/simgrid/lifecycle_cost.go b/internal/provider/simgrid/lifecycle_cost.go new file mode 100644 index 00000000..98e670f9 --- /dev/null +++ b/internal/provider/simgrid/lifecycle_cost.go @@ -0,0 +1,65 @@ +package simgrid + +import ( + "math" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +// Lifecycle billing is separate from simulated CPU work. Charge each cloud +// window once, including provisioning/idle/stop and provisioned disk; never +// overwrite an observed trace with the plan's predicted total cost. +func applyCloudLifecycleCosts(tasks []domain.TaskExecution, resources []domain.Resource, actions []domain.PlannedLifecycleAction) { + for _, resource := range resources { + if resource.Type != domain.ResourceCloudVM { + continue + } + start, stopDuration, found := math.MaxFloat64, 0.0, false + for _, action := range actions { + if action.CapacityTargetID != resource.ID { + continue + } + if action.Action == "provision" { + start = math.Min(start, action.EarliestStart) + found = true + } + if action.Action == "stop" { + stopDuration = math.Max(stopDuration, action.ExpectedDuration) + } + } + if !found { + continue + } + indexes := []int{} + finish, runtime := 0.0, 0.0 + for i, task := range tasks { + if task.AllocatedResourceID != resource.ID { + continue + } + indexes = append(indexes, i) + finish = math.Max(finish, task.FinishedAt) + start = math.Min(start, task.StartedAt) + runtime += task.RuntimeSeconds + } + if len(indexes) == 0 { + continue + } + seconds := math.Max(0, finish+stopDuration-start) + diskPrice, _ := resource.Metadata["diskPricePerGiBMonth"].(float64) + disk := float64(resource.StorageBytes) / float64(1<<30) * diskPrice * seconds / (730 * 3600) + total := seconds*resource.PricePerSecond + disk + for _, i := range indexes { + share := 1.0 / float64(len(indexes)) + if runtime > 0 { + share = tasks[i].RuntimeSeconds / runtime + } + tasks[i].Cost = total * share + if tasks[i].Metadata == nil { + tasks[i].Metadata = map[string]any{} + } + tasks[i].Metadata["costBasis"] = "cloud-lifecycle-compute-and-disk" + tasks[i].Metadata["cloudBillingWindowSeconds"] = seconds + tasks[i].Metadata["cloudDiskCostShare"] = disk * share + } + } +} diff --git a/internal/provider/simgrid/lifecycle_cost_test.go b/internal/provider/simgrid/lifecycle_cost_test.go new file mode 100644 index 00000000..adf2577c --- /dev/null +++ b/internal/provider/simgrid/lifecycle_cost_test.go @@ -0,0 +1,23 @@ +package simgrid + +import ( + "math" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +func TestCloudLifecycleCostsUsesSimulatedFinishAndBillsIdleOnce(t *testing.T) { + resource := domain.Resource{ID: "cloud", Type: domain.ResourceCloudVM, PricePerSecond: 0.01, StorageBytes: 30 << 30, Metadata: map[string]any{"diskPricePerGiBMonth": 0.1}} + tasks := []domain.TaskExecution{{AllocatedResourceID: "cloud", StartedAt: 20, FinishedAt: 30, RuntimeSeconds: 10}, {AllocatedResourceID: "cloud", StartedAt: 80, FinishedAt: 100, RuntimeSeconds: 20}} + actions := []domain.PlannedLifecycleAction{{CapacityTargetID: "cloud", Action: "provision", EarliestStart: 5}, {CapacityTargetID: "cloud", Action: "stop", EarliestStart: 9999, ExpectedDuration: 5}} + applyCloudLifecycleCosts(tasks, []domain.Resource{resource}, actions) + expected := 1.0 + 30*0.1*100/(730*3600) + if math.Abs(tasks[0].Cost+tasks[1].Cost-expected) > 1e-12 { + t.Fatalf("cost=%g want=%g", tasks[0].Cost+tasks[1].Cost, expected) + } + applyCloudLifecycleCosts(tasks, []domain.Resource{resource}, actions) + if math.Abs(tasks[0].Cost+tasks[1].Cost-expected) > 1e-12 { + t.Fatal("billing must be idempotent") + } +} diff --git a/internal/provider/simgrid/normalized_profile_test.go b/internal/provider/simgrid/normalized_profile_test.go new file mode 100644 index 00000000..88b1a1df --- /dev/null +++ b/internal/provider/simgrid/normalized_profile_test.go @@ -0,0 +1,15 @@ +package simgrid + +import ( + "github.com/UFFeScience/akoflow/internal/domain" + "testing" +) + +func TestNormalizedMeasuredProfileOverridesOldReferenceAndPlanRuntime(t *testing.T) { + activity := domain.Activity{ActivityTypeID: "family", Metadata: map[string]any{"baseRuntimeSeconds": 1}, Simulation: &domain.ActivitySimulation{DurationSeconds: 10}} + resource := domain.Resource{ID: "micro", ComputeSpeedup: .25} + profiles := map[string]domain.ActivityResourceProfile{"family\x00micro": {ActivityTypeID: "family", ResourceID: "micro", RuntimeSeconds: 2, ModelVersion: "reference-work-v1"}} + if got := resolveRuntime(activity, domain.PlanAssignment{PredictedRuntimeSeconds: 999}, resource, profiles); got != 20 { + t.Fatalf("expected normalized work 20 rather than old speed/reference/prediction, got %g", got) + } +} diff --git a/internal/provider/simgrid/offline_case.go b/internal/provider/simgrid/offline_case.go new file mode 100644 index 00000000..f402cd6c --- /dev/null +++ b/internal/provider/simgrid/offline_case.go @@ -0,0 +1,27 @@ +package simgrid + +import ( + "fmt" + "math" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" +) + +// BuildOfflineCase uses the exact production contract without submitting a run, +// contacting infrastructure, or reading the execution database. Assignments are +// retained, while transfer stages use the shared materialization model. +func BuildOfflineCase(request ports.ExecutionRequest, referenceFLOPS float64) (platform, input []byte, err error) { + if request.Run.Mode != domain.ExecutionModeSimulation { + return nil, nil, fmt.Errorf("offline case requires simulation mode") + } + if referenceFLOPS <= 0 || math.IsNaN(referenceFLOPS) || math.IsInf(referenceFLOPS, 0) { + return nil, nil, fmt.Errorf("positive finite reference FLOPS required") + } + platform, err = buildPlatformXML(request.Resources, request.NetworkTopology, referenceFLOPS) + if err != nil { + return nil, nil, err + } + input, err = buildRunnerInput(request, referenceFLOPS) + return platform, input, err +} diff --git a/internal/provider/simgrid/offline_case_test.go b/internal/provider/simgrid/offline_case_test.go new file mode 100644 index 00000000..52a6f8e2 --- /dev/null +++ b/internal/provider/simgrid/offline_case_test.go @@ -0,0 +1,34 @@ +package simgrid + +import ( + "math" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/stretchr/testify/require" +) + +func TestOfflineCaseMatchesProductionContract(t *testing.T) { + request := materializedRequestFixture() + request.Run.Mode = domain.ExecutionModeSimulation + platform, input, err := BuildOfflineCase(request, 1e9) + require.NoError(t, err) + wantPlatform, err := buildPlatformXML(request.Resources, request.NetworkTopology, 1e9) + require.NoError(t, err) + wantInput, err := buildRunnerInput(request, 1e9) + require.NoError(t, err) + require.Equal(t, wantPlatform, platform) + require.JSONEq(t, string(wantInput), string(input)) +} + +func TestOfflineCaseRejectsRealAndInvalidReference(t *testing.T) { + request := processRequestFixture() + request.Run.Mode = domain.ExecutionModeReal + _, _, err := BuildOfflineCase(request, 1e9) + require.ErrorContains(t, err, "simulation mode") + request.Run.Mode = domain.ExecutionModeSimulation + for _, invalid := range []float64{0, -1, math.NaN(), math.Inf(1)} { + _, _, err = BuildOfflineCase(request, invalid) + require.ErrorContains(t, err, "reference FLOPS") + } +} diff --git a/internal/provider/simgrid/platform.go b/internal/provider/simgrid/platform.go index 15555cdc..2d96e5c0 100644 --- a/internal/provider/simgrid/platform.go +++ b/internal/provider/simgrid/platform.go @@ -29,7 +29,7 @@ func buildPlatformXML(resources []domain.Resource, topology domain.NetworkTopolo if resource.ID == "" { return nil, fmt.Errorf("SimGrid resource id is required") } - cores := resource.CPUCores + cores := resource.SchedulingCPUCores() if cores < 1 { cores = 1 } diff --git a/internal/provider/simgrid/platform_capacity_test.go b/internal/provider/simgrid/platform_capacity_test.go new file mode 100644 index 00000000..7de6712f --- /dev/null +++ b/internal/provider/simgrid/platform_capacity_test.go @@ -0,0 +1,17 @@ +package simgrid + +import ( + "github.com/UFFeScience/akoflow/internal/domain" + "strings" + "testing" +) + +func TestPlatformRespectsExplicitSingleMicroLane(t *testing.T) { + xml, err := buildPlatformXML([]domain.Resource{{ID: "micro", CPUCores: 2, CPUCapacity: 1, ComputeSpeedup: 0.25}}, domain.NetworkTopology{}, 1e9) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(xml), `core="1"`) || strings.Contains(string(xml), `core="2"`) { + t.Fatalf("platform=%s", xml) + } +} diff --git a/internal/provider/simgrid/process_executor.go b/internal/provider/simgrid/process_executor.go index 4f9e652a..acf91476 100644 --- a/internal/provider/simgrid/process_executor.go +++ b/internal/provider/simgrid/process_executor.go @@ -12,6 +12,7 @@ import ( "github.com/UFFeScience/akoflow/internal/application/ports" "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/materializationmodel" "github.com/UFFeScience/akoflow/internal/provider" ) @@ -62,7 +63,49 @@ func (e *ProcessExecutor) Execute(ctx context.Context, request ports.ExecutionRe return domain.ExecutionTrace{}, err } defer e.release() + preparationModel, err := materializationmodel.Decode(request.NetworkTopology.Metadata) + if err != nil { + return domain.ExecutionTrace{}, err + } + if err := e.checkPreparationCapabilities(ctx, request, preparationModel); err != nil { + return domain.ExecutionTrace{}, err + } + return e.executeBundle(ctx, request, preparationModel) +} +func (e *ProcessExecutor) checkPreparationCapabilities(ctx context.Context, request ports.ExecutionRequest, preparationModel *materializationmodel.Model) error { + if preparationModel != nil { + probeCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + payload, probeErr := e.commands.Run(probeCtx, e.config.BinaryPath, []string{"--capabilities"}, nil) + cancel() + var capabilities struct { + Version int `json:"workspaceMaterializationModelVersion"` + MaxVersion int `json:"workspaceMaterializationModelMaxVersion"` + BulkServiceVersion int `json:"bulkServicePoolModelVersion"` + AdmissionVersion int `json:"runtimeAdmissionModelVersion"` + } + if probeErr != nil || json.Unmarshal(payload, &capabilities) != nil || (capabilities.Version != preparationModel.SchemaVersion && capabilities.MaxVersion < preparationModel.SchemaVersion) { + return fmt.Errorf("SimGrid runner does not confirm workspace materialization model version %d", preparationModel.SchemaVersion) + } + for _, resource := range preparationModel.Resources { + if resource.BulkServiceGroupID != "" && capabilities.BulkServiceVersion != 1 { + return fmt.Errorf("SimGrid runner does not confirm bulk service pool model version 1") + } + } + for _, a := range request.Plan.Assignments { + delay, err := preparationModel.AdmissionSeconds(a.ResourceID) + if err != nil { + return err + } + if delay > 0 && capabilities.AdmissionVersion != 1 { + return fmt.Errorf("SimGrid runner does not confirm runtime admission model version 1") + } + } + } + return nil +} + +func (e *ProcessExecutor) executeBundle(ctx context.Context, request ports.ExecutionRequest, preparationModel *materializationmodel.Model) (domain.ExecutionTrace, error) { bundle, err := e.createBundle(request) if err != nil { return domain.ExecutionTrace{}, err @@ -87,7 +130,20 @@ func (e *ProcessExecutor) Execute(ctx context.Context, request ports.ExecutionRe if trace.RunID != request.Run.ID || trace.PlanID != request.Plan.ID { return domain.ExecutionTrace{}, fmt.Errorf("SimGrid returned trace for run %q and plan %q", trace.RunID, trace.PlanID) } + if err := validateAdmissionTrace(request, preparationModel, trace); err != nil { + return domain.ExecutionTrace{}, err + } trace.Mode, trace.Predicted = request.Run.Mode, request.Plan.Predicted + normalizeWorkspaceQueue(&trace) + applyCloudLifecycleCosts(trace.Tasks, request.Resources, request.Plan.LifecycleActions) + trace.Executed.Cost = 0 + for _, task := range trace.Tasks { + trace.Executed.Cost += task.Cost + } + for _, transfer := range trace.Transfers { + trace.Executed.Cost += transfer.Cost + } + trace.Executed.Feasible = (request.Plan.DeadlineSeconds <= 0 || trace.Executed.MakespanSeconds <= request.Plan.DeadlineSeconds) && (request.Plan.Budget <= 0 || trace.Executed.Cost <= request.Plan.Budget) return trace, nil } diff --git a/internal/provider/simgrid/process_executor_test.go b/internal/provider/simgrid/process_executor_test.go index 6ca4fcf8..325f193e 100644 --- a/internal/provider/simgrid/process_executor_test.go +++ b/internal/provider/simgrid/process_executor_test.go @@ -9,17 +9,22 @@ import ( "time" "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/materializationmodel" "github.com/stretchr/testify/require" ) type runnerCommandFake struct { - name string - args []string - trace domain.ExecutionTrace + name string + args []string + trace domain.ExecutionTrace + capabilities []byte } func (f *runnerCommandFake) Run(_ context.Context, name string, args []string, _ []byte) ([]byte, error) { f.name, f.args = name, append([]string(nil), args...) + if len(args) == 1 && args[0] == "--capabilities" && f.capabilities != nil { + return f.capabilities, nil + } output := argumentValue(args, "--output") payload, err := json.Marshal(f.trace) if err != nil { @@ -56,6 +61,31 @@ func TestProcessExecutorBuildsBundleWaitsAndReadsTrace(t *testing.T) { require.Contains(t, string(platformPayload), ``) } +func TestProcessExecutorRejectsUnconfirmedNativeMaterializationCapability(t *testing.T) { + request := materializedRequestFixture() + command := &runnerCommandFake{} + executor, err := NewProcessExecutor(command, ProcessConfig{BinaryPath: "legacy-runner", Workspace: t.TempDir()}) + require.NoError(t, err) + _, err = executor.Execute(context.Background(), request) + require.ErrorContains(t, err, "does not confirm workspace materialization model") + require.Equal(t, []string{"--capabilities"}, command.args) +} + +func TestProcessExecutorRejectsRunnerThatIgnoresBulkServicePools(t *testing.T) { + request := materializedRequestFixture() + model := request.NetworkTopology.Metadata["workspaceMaterializationModel"].(materializationmodel.Model) + r := model.Resources["edge"] + r.BulkServiceGroupID, r.BulkServiceSlots, r.BulkServiceSource = "hpc-session", 3, "provider-config" + model.Resources["edge"] = r + request.NetworkTopology.Metadata["workspaceMaterializationModel"] = model + command := &runnerCommandFake{capabilities: []byte(`{"workspaceMaterializationModelVersion":1}`)} + executor, err := NewProcessExecutor(command, ProcessConfig{BinaryPath: "old-materialization-runner", Workspace: t.TempDir()}) + require.NoError(t, err) + _, err = executor.Execute(context.Background(), request) + require.ErrorContains(t, err, "does not confirm bulk service pool") + require.Equal(t, []string{"--capabilities"}, command.args) +} + func TestPlatformBuildsMultiHopRoutes(t *testing.T) { resources := []domain.Resource{ {ID: "edge", ComputeSpeedup: 1}, {ID: "gateway", ComputeSpeedup: 2}, {ID: "cloud", ComputeSpeedup: 4}, diff --git a/internal/provider/simgrid/runner_contract.go b/internal/provider/simgrid/runner_contract.go index e6f4c9ae..14748a3b 100644 --- a/internal/provider/simgrid/runner_contract.go +++ b/internal/provider/simgrid/runner_contract.go @@ -6,34 +6,49 @@ import ( "github.com/UFFeScience/akoflow/internal/application/ports" "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/materializationmodel" + domainworkflow "github.com/UFFeScience/akoflow/internal/domain/workflow" ) type runnerInput struct { - RunID string `json:"runId"` - PlanID string `json:"planId"` - DeadlineSeconds float64 `json:"deadlineSeconds"` - Budget float64 `json:"budget"` - Tasks []runnerTask `json:"tasks"` - Dependencies []runnerDependency `json:"dependencies"` - ResourceLaneOrders []runnerLaneOrder `json:"resourceLaneOrders"` - Interference *domain.InterferenceMatrix `json:"interference,omitempty"` + MaterializationModelVersion int `json:"materializationModelVersion,omitempty"` + MaterializationRelayResourceID string `json:"materializationRelayResourceId,omitempty"` + MaterializationModelSource string `json:"materializationModelSource,omitempty"` + RunID string `json:"runId"` + PlanID string `json:"planId"` + DeadlineSeconds float64 `json:"deadlineSeconds"` + Budget float64 `json:"budget"` + Tasks []runnerTask `json:"tasks"` + Dependencies []runnerDependency `json:"dependencies"` + ResourceLaneOrders []runnerLaneOrder `json:"resourceLaneOrders"` + Interference *domain.InterferenceMatrix `json:"interference,omitempty"` } type runnerTask struct { - ID string `json:"id"` - ActivityTypeID string `json:"activityTypeId"` - AssignmentID string `json:"assignmentId"` - ResourceID string `json:"resourceId"` - FLOPs float64 `json:"flops"` - OverheadSeconds float64 `json:"overheadSeconds"` - PricePerSecond float64 `json:"pricePerSecond"` + ID string `json:"id"` + ActivityTypeID string `json:"activityTypeId"` + AssignmentID string `json:"assignmentId"` + ResourceID string `json:"resourceId"` + FLOPs float64 `json:"flops"` + OverheadSeconds float64 `json:"overheadSeconds"` + PricePerSecond float64 `json:"pricePerSecond"` + PreparationSeconds float64 `json:"preparationSeconds,omitempty"` + PublicationSeconds float64 `json:"publicationSeconds,omitempty"` + AdmissionSeconds float64 `json:"admissionSeconds,omitempty"` + PreparationServiceGroupID string `json:"preparationServiceGroupId,omitempty"` + PreparationServiceSlots int `json:"preparationServiceSlots,omitempty"` } type runnerDependency struct { - ProducerID string `json:"producerId"` - ConsumerID string `json:"consumerId"` - Bytes int64 `json:"bytes"` - PricePerByte float64 `json:"pricePerByte"` + ProducerID string `json:"producerId"` + ConsumerID string `json:"consumerId"` + Bytes int64 `json:"bytes"` + PricePerByte float64 `json:"pricePerByte"` + Strategy string `json:"strategy,omitempty"` + LogicalBytes int64 `json:"logicalBytes,omitempty"` + NetworkBytes int64 `json:"networkBytes,omitempty"` + RelayNetworkBytes int64 `json:"relayNetworkBytes,omitempty"` + Stages []materializationmodel.Stage `json:"stages,omitempty"` } type runnerLaneOrder struct { @@ -42,11 +57,25 @@ type runnerLaneOrder struct { } func buildRunnerInput(request ports.ExecutionRequest, referenceFLOPS float64) ([]byte, error) { + projected, err := domainworkflow.ProjectWorkspaceDependenciesForResourceTypes(request.Workflow, assignedResourceTypes(request)) + if err != nil { + return nil, fmt.Errorf("project workspace dependencies: %w", err) + } + request.Workflow = projected + preparationModel, err := materializationmodel.Decode(request.NetworkTopology.Metadata) + if err != nil { + return nil, err + } model := indexRequest(request) input := runnerInput{ RunID: request.Run.ID, PlanID: request.Plan.ID, DeadlineSeconds: request.Plan.DeadlineSeconds, Budget: request.Plan.Budget, } + if preparationModel != nil { + input.MaterializationModelVersion = preparationModel.SchemaVersion + input.MaterializationRelayResourceID = preparationModel.RelayResourceID + input.MaterializationModelSource = preparationModel.Source + } if raw := request.Plan.Metadata["interferenceMatrix"]; raw != nil { encoded, err := json.Marshal(raw) if err != nil { @@ -58,41 +87,13 @@ func buildRunnerInput(request ports.ExecutionRequest, referenceFLOPS float64) ([ } input.Interference = &matrix } - for _, activity := range request.Workflow.Activities { - assignment, exists := model.assignments[activity.ID] - if !exists { - return nil, fmt.Errorf("activity %q has no SimGrid assignment", activity.ID) - } - resource, exists := model.resources[assignment.ResourceID] - if !exists { - return nil, fmt.Errorf("assignment for %q references missing resource %q", activity.ID, assignment.ResourceID) - } - flops := activityFLOPS(activity) - overhead := resolveAssignmentOverhead(assignment, resource) - if flops <= 0 { - runtimeSeconds := resolveRuntime(activity, assignment, resource, model.profiles) - flops = runtimeSeconds * resourceFLOPS(resource, referenceFLOPS) - } - input.Tasks = append(input.Tasks, runnerTask{ - ID: activity.ID, ActivityTypeID: activity.ActivityTypeID, - AssignmentID: assignment.ID, ResourceID: resource.ID, - FLOPs: flops, OverheadSeconds: overhead, - PricePerSecond: resource.PricePerSecond, - }) + input.Tasks, err = buildRunnerTasks(request, referenceFLOPS, model, preparationModel) + if err != nil { + return nil, err } - dataBytes := make(map[string]int64) - for _, dependency := range request.Workflow.DataDependencies { - key := dependency.ProducerActivityID + "\x00" + dependency.ConsumerActivityID - dataBytes[key] += dependency.SizeBytes - } - for _, dependency := range request.Workflow.Dependencies { - producer := model.assignments[dependency.DependsOnActivityID] - consumer := model.assignments[dependency.ActivityID] - bytes := dataBytes[dependency.DependsOnActivityID+"\x00"+dependency.ActivityID] - input.Dependencies = append(input.Dependencies, runnerDependency{ - ProducerID: dependency.DependsOnActivityID, ConsumerID: dependency.ActivityID, - Bytes: bytes, PricePerByte: transferPrice(request.NetworkTopology.Links, producer.ResourceID, consumer.ResourceID), - }) + input.Dependencies, err = buildRunnerDependencies(request, model, preparationModel) + if err != nil { + return nil, err } for successor, predecessor := range model.lanePrevious { input.ResourceLaneOrders = append(input.ResourceLaneOrders, runnerLaneOrder{ @@ -137,3 +138,113 @@ func transferPrice(links []domain.NetworkLink, source, target string) float64 { } return cost } + +func buildRunnerTasks(request ports.ExecutionRequest, referenceFLOPS float64, model indexedRequest, preparationModel *materializationmodel.Model) ([]runnerTask, error) { + var tasks []runnerTask + for _, activity := range request.Workflow.Activities { + assignment, exists := model.assignments[activity.ID] + if !exists { + return nil, fmt.Errorf("activity %q has no SimGrid assignment", activity.ID) + } + resource, exists := model.resources[assignment.ResourceID] + if !exists { + return nil, fmt.Errorf("assignment for %q references missing resource %q", activity.ID, assignment.ResourceID) + } + flops := activityFLOPS(activity) + overhead := resolveAssignmentOverhead(assignment, resource) + if flops <= 0 { + runtimeSeconds := resolveRuntime(activity, assignment, resource, model.profiles) + flops = runtimeSeconds * resourceFLOPS(resource, referenceFLOPS) + } + preparationSeconds := 0.0 + publicationSeconds := 0.0 + admissionSeconds := 0.0 + preparationServiceGroupID, preparationServiceSlots := "", 0 + if preparationModel != nil { + service := preparationModel.Resources[resource.ID] + preparationServiceGroupID, preparationServiceSlots = service.BulkServiceGroupID, service.BulkServiceSlots + encoded, err := json.Marshal(activity.Metadata["workspaceProjectionVolumes"]) + if err != nil { + return nil, err + } + var volume domainworkflow.WorkspaceProjectionVolumes + if activity.Metadata["workspaceProjectionVolumes"] == nil { + return nil, fmt.Errorf("activity %q lacks projected materialization volumes", activity.ID) + } + if err = json.Unmarshal(encoded, &volume); err != nil { + return nil, err + } + preparationSeconds, err = preparationModel.PreparationSeconds(resource.ID, volume.InputTotalBytes, volume.InputUniqueBytes) + if err != nil { + return nil, fmt.Errorf("consumer %q merge: %w", activity.ID, err) + } + publicationSeconds, err = preparationModel.PublicationSeconds(resource.ID, volume.TerminalObservationBytes, volume.PublicationCleanupBytes) + if err != nil { + return nil, err + } + admissionSeconds, err = preparationModel.AdmissionSeconds(resource.ID) + if err != nil { + return nil, err + } + } + tasks = append(tasks, runnerTask{ + ID: activity.ID, ActivityTypeID: activity.ActivityTypeID, + AssignmentID: assignment.ID, ResourceID: resource.ID, + FLOPs: flops, OverheadSeconds: overhead, + PricePerSecond: resource.PricePerSecond, + PreparationSeconds: preparationSeconds, + PublicationSeconds: publicationSeconds, + AdmissionSeconds: admissionSeconds, + PreparationServiceGroupID: preparationServiceGroupID, + PreparationServiceSlots: preparationServiceSlots, + }) + } + return tasks, nil +} + +func buildRunnerDependencies(request ports.ExecutionRequest, model indexedRequest, preparationModel *materializationmodel.Model) ([]runnerDependency, error) { + var dependencies []runnerDependency + dataBytes := make(map[string]int64) + for _, dependency := range request.Workflow.DataDependencies { + key := dependency.ProducerActivityID + "\x00" + dependency.ConsumerActivityID + dataBytes[key] += dependency.SizeBytes + } + for _, dependency := range request.Workflow.Dependencies { + producer := model.assignments[dependency.DependsOnActivityID] + consumer := model.assignments[dependency.ActivityID] + bytes := dataBytes[dependency.DependsOnActivityID+"\x00"+dependency.ActivityID] + entry := runnerDependency{ + ProducerID: dependency.DependsOnActivityID, ConsumerID: dependency.ActivityID, + Bytes: bytes, PricePerByte: transferPrice(request.NetworkTopology.Links, producer.ResourceID, consumer.ResourceID), + } + if preparationModel != nil { + route, err := preparationModel.Route(producer.ResourceID, consumer.ResourceID, bytes) + if err != nil { + return nil, fmt.Errorf("dependency %s -> %s: %w", entry.ProducerID, entry.ConsumerID, err) + } + entry.Strategy, entry.LogicalBytes, entry.NetworkBytes, entry.RelayNetworkBytes, entry.Stages = route.Strategy, route.LogicalBytes, route.NetworkBytes, route.RelayNetworkBytes, route.Stages + transferCost := 0.0 + for _, stage := range route.Stages { + if _, exists := model.resources[stage.SourceResourceID]; !exists { + return nil, fmt.Errorf("materialization stage source %q absent from platform", stage.SourceResourceID) + } + if _, exists := model.resources[stage.TargetResourceID]; !exists { + return nil, fmt.Errorf("materialization stage target %q absent from platform", stage.TargetResourceID) + } + if stage.Bytes == 0 { + continue + } + _, cost, exists := resolveNetworkPath(request.NetworkTopology.Links, stage.SourceResourceID, stage.TargetResourceID, stage.Bytes) + if !exists { + return nil, fmt.Errorf("no materialization network path %q -> %q", stage.SourceResourceID, stage.TargetResourceID) + } + transferCost += cost + } + if bytes > 0 { + entry.PricePerByte = transferCost / float64(bytes) + } + } + dependencies = append(dependencies, entry) + } + return dependencies, nil +} diff --git a/internal/provider/simgrid/runner_contract_test.go b/internal/provider/simgrid/runner_contract_test.go index 30a335dd..42d74582 100644 --- a/internal/provider/simgrid/runner_contract_test.go +++ b/internal/provider/simgrid/runner_contract_test.go @@ -6,9 +6,144 @@ import ( "github.com/UFFeScience/akoflow/internal/application/ports" "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/materializationmodel" + domainworkflow "github.com/UFFeScience/akoflow/internal/domain/workflow" "github.com/stretchr/testify/require" ) +func materializedRequestFixture() Request { + request := processRequestFixture() + request.Workflow.Activities[0].Metadata = map[string]any{ + "workspaceProjection": domainworkflow.WorkspaceProjection{Contract: "whole-workspace-v1", Source: "baseline", Outputs: []domainworkflow.ProjectedFile{{Path: "data", SizeBytes: 100, ContentID: "output"}}}, + } + request.Workflow.Activities[1].Metadata = map[string]any{"workspaceProjection": domainworkflow.WorkspaceProjection{Contract: "whole-workspace-v1", Source: "baseline"}} + request.Resources = append(request.Resources, domain.Resource{ID: "relay", ComputeSpeedup: 1}) + request.NetworkTopology.Links = []domain.NetworkLink{ + {SourceResourceID: "edge", TargetResourceID: "relay", BandwidthBitsPerSecond: 8000, PricePerByte: .01}, + {SourceResourceID: "relay", TargetResourceID: "cloud", BandwidthBitsPerSecond: 8000, PricePerByte: .02}, + } + request.NetworkTopology.Metadata = map[string]any{"workspaceMaterializationModel": materializationmodel.Model{ + SchemaVersion: 1, Source: "baseline", RelayResourceID: "relay", Resources: map[string]materializationmodel.Resource{ + "edge": {Class: "hpc", Domain: "hpc"}, + "cloud": {Class: "micro", Domain: "cloud"}, + "relay": {Class: "relay", Domain: "relay"}, + }, + Profiles: []materializationmodel.Profile{ + {Strategy: "gateway", SourceClass: "hpc", TargetClass: "micro", Source: "baseline", Samples: 2, ArchiveToWorkspaceRatio: .5, IngressArchiveBytesPerSecond: 50, UploadArchiveBytesPerSecond: 25, IngressIncludesSourcePreparation: true}, + }, + MergeProfiles: []materializationmodel.MergeProfile{{TargetClass: "micro", Source: "baseline", Samples: 2, CheckSecondsPerInputByte: .01, CopySecondsPerUniqueByte: .02, VerifySecondsPerUniqueByte: .03}}, + }} + return request +} + +func TestRunnerContractMaterializationStagesAndConsumerMerge(t *testing.T) { + request := materializedRequestFixture() + payload, err := buildRunnerInput(request, 1e9) + require.NoError(t, err) + var input runnerInput + require.NoError(t, json.Unmarshal(payload, &input)) + require.Equal(t, 1, input.MaterializationModelVersion) + require.Len(t, input.Dependencies, 1) + dependency := input.Dependencies[0] + require.Equal(t, "gateway", dependency.Strategy) + require.Len(t, dependency.Stages, 2) + require.EqualValues(t, 50, dependency.Stages[0].Bytes) + require.EqualValues(t, 100, dependency.NetworkBytes) + require.EqualValues(t, 100, dependency.RelayNetworkBytes) + require.InDelta(t, 1.5, float64(dependency.Bytes)*dependency.PricePerByte, 1e-9) + require.InDelta(t, 6, input.Tasks[1].PreparationSeconds, 1e-9) + request.Resources = request.Resources[:2] + _, err = buildRunnerInput(request, 1e9) + require.ErrorContains(t, err, "absent from platform") +} + +func TestRunnerContractKeepsRuntimeAdmissionSeparate(t *testing.T) { + r := materializedRequestFixture() + m := r.NetworkTopology.Metadata["workspaceMaterializationModel"].(materializationmodel.Model) + resource := m.Resources["cloud"] + resource.ExecutionRuntimeID = "native-cloud" + m.Resources["cloud"] = resource + m.AdmissionProfiles = []materializationmodel.AdmissionProfile{{TargetClass: "micro", RuntimeID: "native-cloud", Source: "fixture", Samples: 1, FixedSeconds: 7}} + m.SchemaVersion = 2 + r.NetworkTopology.Metadata["workspaceMaterializationModel"] = m + payload, err := buildRunnerInput(r, 1e9) + require.NoError(t, err) + var input runnerInput + require.NoError(t, json.Unmarshal(payload, &input)) + require.Equal(t, 7.0, input.Tasks[1].AdmissionSeconds) + require.Equal(t, 6.0, input.Tasks[1].PreparationSeconds) + require.Zero(t, input.Tasks[0].AdmissionSeconds) +} + +func TestRunnerContractIncludesFixedMergeCommandServiceOnce(t *testing.T) { + request := materializedRequestFixture() + model := request.NetworkTopology.Metadata["workspaceMaterializationModel"].(materializationmodel.Model) + model.MergeProfiles[0].CommandOverheadSeconds = 2.5 + request.NetworkTopology.Metadata["workspaceMaterializationModel"] = model + payload, err := buildRunnerInput(request, 1e9) + require.NoError(t, err) + var input runnerInput + require.NoError(t, json.Unmarshal(payload, &input)) + require.Zero(t, input.Tasks[0].PreparationSeconds) + require.InDelta(t, 8.5, input.Tasks[1].PreparationSeconds, 1e-9) + require.EqualValues(t, 100, input.Dependencies[0].NetworkBytes) +} + +func TestRunnerContractIncludesInputObservationOnce(t *testing.T) { + request := materializedRequestFixture() + projection := request.Workflow.Activities[1].Metadata["workspaceProjection"].(domainworkflow.WorkspaceProjection) + terminalBytes := int64(300) + projection.TerminalObservationBytes = &terminalBytes + cleanupBytes := int64(100) + projection.PublicationCleanupBytes = &cleanupBytes + request.Workflow.Activities[1].Metadata["workspaceProjection"] = projection + model := request.NetworkTopology.Metadata["workspaceMaterializationModel"].(materializationmodel.Model) + model.InputObservationProfiles = []materializationmodel.InputObservationProfile{{TargetClass: "micro", Source: "queue-free-fixture", Samples: 1, SecondsPerUniqueByte: .1}} + model.PublicationProfiles = []materializationmodel.PublicationProfile{{TargetClass: "micro", Source: "queue-free-fixture", Samples: 1, ScanSecondsPerWorkspaceByte: .02, CleanupSecondsPerInputByte: .01}} + request.NetworkTopology.Metadata["workspaceMaterializationModel"] = model + payload, err := buildRunnerInput(request, 1e9) + require.NoError(t, err) + var input runnerInput + require.NoError(t, json.Unmarshal(payload, &input)) + require.Zero(t, input.Tasks[0].PreparationSeconds) + require.InDelta(t, 16, input.Tasks[1].PreparationSeconds, 1e-9) + require.InDelta(t, 7, input.Tasks[1].PublicationSeconds, 1e-9) + require.EqualValues(t, 100, input.Dependencies[0].NetworkBytes) +} + +func TestRunnerInputUsesWholeProducerWorkspaceNotDeclaredEdge(t *testing.T) { + request := ports.ExecutionRequest{ + Run: domain.ExecutionRun{ID: "run"}, + Plan: domain.SchedulePlan{Assignments: []domain.PlanAssignment{{ActivityID: "a", ResourceID: "source"}, {ActivityID: "b", ResourceID: "target"}}}, + Resources: []domain.Resource{{ID: "source", ComputeSpeedup: 1}, {ID: "target", ComputeSpeedup: 1}}, + Workflow: domain.WorkflowVersion{ + Activities: []domain.Activity{ + { + ID: "a", + Metadata: map[string]any{ + "workspaceProjection": domainworkflow.WorkspaceProjection{ + Contract: "whole-workspace-v1", + Source: "baseline", + Inputs: []domainworkflow.ProjectedFile{{Path: "in.fits", SizeBytes: 1000, ContentID: "in"}}, + Outputs: []domainworkflow.ProjectedFile{{Path: "out.fits", SizeBytes: 2000, ContentID: "estimated-output"}}, + }, + }, + }, + {ID: "b", Metadata: map[string]any{"workspaceProjection": domainworkflow.WorkspaceProjection{Contract: "whole-workspace-v1", Source: "baseline"}}}, + }, + Dependencies: []domain.ActivityDependency{{ActivityID: "b", DependsOnActivityID: "a"}}, + DataDependencies: []domain.ActivityDataDependency{{ProducerActivityID: "a", ConsumerActivityID: "b", SizeBytes: 12}}, + }, + } + payload, err := buildRunnerInput(request, 1e9) + require.NoError(t, err) + var input runnerInput + require.NoError(t, json.Unmarshal(payload, &input)) + require.Len(t, input.Dependencies, 1) + require.EqualValues(t, 3000, input.Dependencies[0].Bytes) + require.EqualValues(t, 12, request.Workflow.DataDependencies[0].SizeBytes) +} + func TestRunnerInputPrefersActivityRuntimeAndSeparatesOverhead(t *testing.T) { request := ports.ExecutionRequest{ Run: domain.ExecutionRun{ID: "run"}, diff --git a/internal/provider/simgrid/simulation_executor.go b/internal/provider/simgrid/simulation_executor.go index 316e89c7..42bc4b16 100644 --- a/internal/provider/simgrid/simulation_executor.go +++ b/internal/provider/simgrid/simulation_executor.go @@ -6,6 +6,7 @@ import ( "sort" "github.com/UFFeScience/akoflow/internal/domain" + domainworkflow "github.com/UFFeScience/akoflow/internal/domain/workflow" "github.com/UFFeScience/akoflow/internal/infrastructure/plugins/planning" ) @@ -18,6 +19,11 @@ func NewSimulationExecutor() *SimulationExecutor { } func (e *SimulationExecutor) Execute(ctx context.Context, request Request) (domain.ExecutionTrace, error) { + projected, err := domainworkflow.ProjectWorkspaceDependenciesForResourceTypes(request.Workflow, assignedResourceTypes(request)) + if err != nil { + return domain.ExecutionTrace{}, fmt.Errorf("project workspace dependencies: %w", err) + } + request.Workflow = projected if request.Run.Mode != domain.ExecutionModeSimulation { return domain.ExecutionTrace{}, fmt.Errorf("simulation executor cannot execute mode %q", request.Run.Mode) } @@ -38,6 +44,18 @@ func (e *SimulationExecutor) Execute(ctx context.Context, request Request) (doma return buildTrace(request, completed, transfers), nil } +func assignedResourceTypes(request Request) map[string]string { + resources := map[string]string{} + for _, resource := range request.Resources { + resources[resource.ID] = string(resource.Type) + } + assigned := map[string]string{} + for _, assignment := range request.Plan.Assignments { + assigned[assignment.ActivityID] = resources[assignment.ResourceID] + } + return assigned +} + type indexedRequest struct { resources map[string]domain.Resource assignments map[string]domain.PlanAssignment @@ -196,6 +214,7 @@ func buildTrace( tasks = append(tasks, task) } applyResourceActiveCosts(tasks, request.Resources) + applyCloudLifecycleCosts(tasks, request.Resources, request.Plan.LifecycleActions) for _, task := range tasks { metrics.MakespanSeconds = max(metrics.MakespanSeconds, task.FinishedAt) metrics.ComputeSeconds += task.RuntimeSeconds @@ -276,11 +295,14 @@ func allCompleted(ids []string, completed map[string]domain.TaskExecution) bool } func resolveRuntime(activity domain.Activity, assignment domain.PlanAssignment, resource domain.Resource, profiles map[string]domain.ActivityResourceProfile) float64 { + if profile, ok := profiles[activity.ActivityTypeID+"\x00"+resource.ID]; ok && profile.UsesWorkUnits() { + return profile.RuntimeFor(activity) + } if base, ok := numberMetadata(activity.Metadata, "baseRuntimeSeconds"); ok && resource.ComputeSpeedup > 0 { return base / resource.ComputeSpeedup } if profile, ok := profiles[activity.ActivityTypeID+"\x00"+resource.ID]; ok { - return profile.RuntimeSeconds + return profile.RuntimeFor(activity) } // Interference-aware planning stores a slowed predicted duration. Execution // must start from the resource-specific clean runtime and let the execution diff --git a/internal/provider/simgrid/stage_conformance.go b/internal/provider/simgrid/stage_conformance.go new file mode 100644 index 00000000..0b01263d --- /dev/null +++ b/internal/provider/simgrid/stage_conformance.go @@ -0,0 +1,45 @@ +package simgrid + +import ( + "fmt" + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/materializationmodel" + "math" +) + +// Reject success responses from a runner that silently ignores requested +// stages. Older runners remain compatible when no admission delay is requested. +func validateAdmissionTrace(request ports.ExecutionRequest, model *materializationmodel.Model, trace domain.ExecutionTrace) error { + if model == nil { + return nil + } + expected := map[string]float64{} + for _, a := range request.Plan.Assignments { + delay, err := model.AdmissionSeconds(a.ResourceID) + if err != nil { + return err + } + if delay > 0 { + expected[a.ActivityID] = delay + } + } + seen := map[string]bool{} + for _, t := range trace.Tasks { + want, required := expected[t.ActivityID] + if !required { + continue + } + got, ok := numberMetadata(t.Metadata, "externalAdmissionSeconds") + if seen[t.ActivityID] || !ok || math.IsNaN(got) || math.IsInf(got, 0) || math.Abs(got-want) > 1e-8*math.Max(1, want) { + return fmt.Errorf("SimGrid trace does not acknowledge requested admission stage for %s", t.ActivityID) + } + seen[t.ActivityID] = true + } + for id := range expected { + if !seen[id] { + return fmt.Errorf("SimGrid trace is missing admission stage for %s", id) + } + } + return nil +} diff --git a/internal/provider/simgrid/stage_conformance_test.go b/internal/provider/simgrid/stage_conformance_test.go new file mode 100644 index 00000000..ff221e3f --- /dev/null +++ b/internal/provider/simgrid/stage_conformance_test.go @@ -0,0 +1,46 @@ +package simgrid + +import ( + "context" + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/materializationmodel" + "github.com/stretchr/testify/require" + "math" + "testing" +) + +func admissionRequest() Request { + r := materializedRequestFixture() + m := r.NetworkTopology.Metadata["workspaceMaterializationModel"].(materializationmodel.Model) + resource := m.Resources["cloud"] + resource.ExecutionRuntimeID = "cloud-native" + m.Resources["cloud"] = resource + m.AdmissionProfiles = []materializationmodel.AdmissionProfile{{TargetClass: "micro", RuntimeID: "cloud-native", Source: "fixture", Samples: 1, FixedSeconds: 7}} + m.SchemaVersion = 2 + r.NetworkTopology.Metadata["workspaceMaterializationModel"] = m + return r +} + +func TestAdmissionRequiresNativeCapabilityBeforeExecution(t *testing.T) { + r := admissionRequest() + fake := &runnerCommandFake{capabilities: []byte(`{"workspaceMaterializationModelVersion":1,"workspaceMaterializationModelMaxVersion":2,"bulkServicePoolModelVersion":1}`)} + executor, err := NewProcessExecutor(fake, ProcessConfig{BinaryPath: "old-runner", Workspace: t.TempDir()}) + require.NoError(t, err) + _, err = executor.Execute(context.Background(), r) + require.ErrorContains(t, err, "runtime admission model") + require.Equal(t, []string{"--capabilities"}, fake.args) +} + +func TestAdmissionTraceRejectsIgnoredMissingAndDuplicateStages(t *testing.T) { + r := admissionRequest() + m, err := materializationmodel.Decode(r.NetworkTopology.Metadata) + require.NoError(t, err) + for _, v := range []any{nil, 0.0, math.NaN(), math.Inf(1)} { + require.Error(t, validateAdmissionTrace(r, m, domain.ExecutionTrace{Tasks: []domain.TaskExecution{{ActivityID: "b", Metadata: map[string]any{"externalAdmissionSeconds": v}}}})) + } + valid := domain.TaskExecution{ActivityID: "b", Metadata: map[string]any{"externalAdmissionSeconds": 7.0}} + require.NoError(t, validateAdmissionTrace(r, m, domain.ExecutionTrace{Tasks: []domain.TaskExecution{valid}})) + require.Error(t, validateAdmissionTrace(r, m, domain.ExecutionTrace{})) + require.Error(t, validateAdmissionTrace(r, m, domain.ExecutionTrace{Tasks: []domain.TaskExecution{valid, valid}})) + require.NoError(t, validateAdmissionTrace(r, nil, domain.ExecutionTrace{})) +} diff --git a/internal/provider/simgrid/workspace_queue.go b/internal/provider/simgrid/workspace_queue.go new file mode 100644 index 00000000..e0005df0 --- /dev/null +++ b/internal/provider/simgrid/workspace_queue.go @@ -0,0 +1,20 @@ +package simgrid + +import "github.com/UFFeScience/akoflow/internal/domain" + +// The native trace reports preparation separately. Waiting for a lane ends +// when preparation starts, not when computation starts after the merge. +func normalizeWorkspaceQueue(trace *domain.ExecutionTrace) { + trace.Executed.QueueSeconds = 0 + for i := range trace.Tasks { + task := &trace.Tasks[i] + if source, _ := task.Metadata["workspaceMaterializationModelSource"].(string); source != "" { + start, hasStart := task.Metadata["workspacePreparationStartedAt"].(float64) + ready, hasReady := task.Metadata["workspaceRouteReadyAt"].(float64) + if hasStart && hasReady { + task.QueueSeconds = max(0, start-ready) + } + } + trace.Executed.QueueSeconds += task.QueueSeconds + } +} diff --git a/internal/provider/simgrid/workspace_queue_test.go b/internal/provider/simgrid/workspace_queue_test.go new file mode 100644 index 00000000..80222195 --- /dev/null +++ b/internal/provider/simgrid/workspace_queue_test.go @@ -0,0 +1,21 @@ +package simgrid + +import ( + "github.com/UFFeScience/akoflow/internal/domain" + "testing" +) + +func TestWorkspaceMergeIsNotBilledAsQueueAndLegacyQueueIsPreserved(t *testing.T) { + trace := domain.ExecutionTrace{Tasks: []domain.TaskExecution{ + {QueueSeconds: 12, Metadata: map[string]any{"workspaceMaterializationModelSource": "frozen", "workspaceRouteReadyAt": 10.0, "workspacePreparationStartedAt": 13.0, "workspacePreparationSeconds": 9.0}}, + {QueueSeconds: 7}, + }} + normalizeWorkspaceQueue(&trace) + if trace.Tasks[0].QueueSeconds != 3 || trace.Tasks[1].QueueSeconds != 7 || trace.Executed.QueueSeconds != 10 { + t.Fatalf("queue classification=%+v", trace) + } + normalizeWorkspaceQueue(&trace) + if trace.Executed.QueueSeconds != 10 { + t.Fatal("normalization is not idempotent") + } +} diff --git a/internal/provider/slurm/adapter.go b/internal/provider/slurm/adapter.go index 5db8b5e5..e89f896e 100644 --- a/internal/provider/slurm/adapter.go +++ b/internal/provider/slurm/adapter.go @@ -2,6 +2,7 @@ package slurm import ( "context" + "crypto/sha256" "encoding/base64" "fmt" "os" @@ -12,18 +13,20 @@ import ( "sync" "time" + "github.com/UFFeScience/akoflow/internal/application/ports" "github.com/UFFeScience/akoflow/internal/domain" runtimecommon "github.com/UFFeScience/akoflow/internal/provider" ) type Adapter struct { - executor runtimecommon.CommandExecutor - partition string - scriptDirectory string - submitFromStdin bool - directMu sync.RWMutex - directResults map[string]directResult - directCancels map[string]context.CancelFunc + executor runtimecommon.CommandExecutor + partition string + scriptDirectory string + submitFromStdin bool + eligibleNodeFallback bool + directMu sync.RWMutex + directResults map[string]directResult + directCancels map[string]context.CancelFunc } type directResult struct { @@ -39,6 +42,8 @@ type Config struct { // the script stays with the engine while sbatch receives its contents on the // remote login node's stdin. SubmitFromStdin bool + // EligibleNodeFallback is opt-in and keeps the original node's feature set. + EligibleNodeFallback bool } func New(executor runtimecommon.CommandExecutor, partition string) *Adapter { @@ -47,7 +52,7 @@ func New(executor runtimecommon.CommandExecutor, partition string) *Adapter { func NewWithConfig(executor runtimecommon.CommandExecutor, config Config) *Adapter { return &Adapter{executor: executor, partition: config.Partition, - scriptDirectory: config.ScriptDirectory, submitFromStdin: config.SubmitFromStdin, + scriptDirectory: config.ScriptDirectory, submitFromStdin: config.SubmitFromStdin, eligibleNodeFallback: config.EligibleNodeFallback, directResults: make(map[string]directResult), directCancels: make(map[string]context.CancelFunc)} } @@ -68,21 +73,35 @@ func (a *Adapter) Start(ctx context.Context, execution domain.ActivityExecutionC if a.executor == nil { return domain.ActivityHandle{}, fmt.Errorf("slurm command executor is required") } - partition, node := a.partition, "" - if execution.Resource.Type == domain.ResourceHPCPartition && execution.Resource.ProviderID != "" { - partition = execution.Resource.ProviderID - } - if execution.Resource.Type == domain.ResourceHPCMachine && execution.Resource.ProviderID != "" { - node = execution.Resource.ProviderID - } + partition, node := slurmPlacement(a.partition, execution.Resource) activity := execution.Activity if activity.Command.WorkingDirectory == "" { activity.Command.WorkingDirectory = slurmArtifactRoot(execution.Run.ID, activity.ID) } + plannedNode := node + fallbackFeatures := "" + fallbackExcluded := "" + if a.eligibleNodeFallback && node != "" { + var err error + node, fallbackFeatures, fallbackExcluded, err = a.eligibleNodes(ctx, partition, node, execution.Resource) + if err != nil { + return domain.ActivityHandle{}, err + } + } script, err := batchScript(execution.Run.ID, activity, partition, node) if err != nil { return domain.ActivityHandle{}, err } + // Bind reconciliation to this exact run/activity/attempt, never an activity + // name shared by another run. The bounded hash also fits Slurm name limits. + jobName := fmt.Sprintf("akoflow-%x", sha256.Sum256([]byte(execution.HandleID()))) + script = strings.Replace(script, "#SBATCH --job-name="+shellToken("akoflow-"+activity.ID), "#SBATCH --job-name="+jobName, 1) + if a.eligibleNodeFallback && plannedNode != "" { + script, err = a.applyEligibleNodeGuard(ctx, execution, activity, script, plannedNode, node, partition, fallbackFeatures, fallbackExcluded) + if err != nil { + return domain.ActivityHandle{}, err + } + } scriptPath, err := a.saveScript(execution.Run.ID, activity.ID, ".sbatch", script) if err != nil { return domain.ActivityHandle{}, err @@ -99,11 +118,11 @@ func (a *Adapter) Start(ctx context.Context, execution domain.ActivityExecutionC } output, err := a.executor.Run(ctx, "sbatch", arguments, input) if err != nil { - return domain.ActivityHandle{}, err + return domain.ActivityHandle{}, a.reconcileSubmission(ctx, jobName, err) } jobID, err := parseJobID(output) if err != nil { - return domain.ActivityHandle{}, err + return domain.ActivityHandle{}, a.reconcileSubmission(ctx, jobName, err) } submittedAt := runtimecommon.UnixSeconds(time.Now()) return domain.ActivityHandle{ID: runtimecommon.NewID("activity"), RunID: execution.Run.ID, @@ -111,6 +130,7 @@ func (a *Adapter) Start(ctx context.Context, execution domain.ActivityExecutionC RuntimeID: execution.RuntimeID, ExternalID: jobID, Status: domain.HandleStarting, StartedAt: submittedAt, Metadata: map[string]any{"executionTarget": string(domain.ExecutionTargetBatch), "scriptPath": scriptPath, + "plannedNode": plannedNode, "eligibleNodes": node, "partition": partition, "eligibleNodeFallback": a.eligibleNodeFallback, "logPath": slurmLogPath(execution.Run.ID, activity.ID, jobID), "sentinelPath": slurmSentinelPath(execution.Run.ID, activity.ID, jobID), "metricsPath": slurmSentinelPath(execution.Run.ID, activity.ID, jobID) + ".metrics.tsv", @@ -118,6 +138,36 @@ func (a *Adapter) Start(ctx context.Context, execution domain.ActivityExecutionC "artifactObservationDriver": "filesystem-diff", "artifactObservationRoot": activity.Command.WorkingDirectory}}, nil } +func slurmPlacement(defaultPartition string, resource domain.Resource) (string, string) { + partition, node := defaultPartition, "" + if resource.Type == domain.ResourceHPCPartition && resource.ProviderID != "" { + partition = resource.ProviderID + } + if resource.Type == domain.ResourceHPCMachine && resource.ProviderID != "" { + node = resource.ProviderID + } + return partition, node +} + +// An SSH failure does not prove sbatch failed remotely. Cancel only jobs whose +// exact attempt-bound name matches, using a fresh bounded context even when +// the workflow context was cancelled. Never resubmit an uncertain submission. +func (a *Adapter) reconcileSubmission(ctx context.Context, jobName string, cause error) error { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + jobs, err := a.matchingSubmissionJobs(cleanup, jobName) + if err != nil { + return uncertainSubmission(jobName, fmt.Errorf("submission outcome uncertain; reconciliation unavailable: %v; original: %w", err, cause)) + } + if len(jobs) > 0 { + if _, err := a.executor.Run(cleanup, "scancel", jobs, nil); err != nil { + return uncertainSubmission(jobName, fmt.Errorf("submission outcome uncertain; cancellation failed: %v; original: %w", err, cause)) + } + return uncertainSubmission(jobName, fmt.Errorf("submission failed; cancellation of %d exact-attempt jobs is not yet confirmed: %w", len(jobs), cause)) + } + return fmt.Errorf("submission failed; exact-attempt reconciliation requested cancellation of %d jobs: %w", len(jobs), cause) +} + func validatePreparation(execution domain.ActivityExecutionContext) error { if execution.Preparation == nil { return nil @@ -144,9 +194,17 @@ func parseJobID(output []byte) (string, error) { } func (a *Adapter) Inspect(ctx context.Context, handle domain.ActivityHandle) (domain.ActivityHandle, error) { + if pending, _ := handle.Metadata[ports.RuntimeReconciliationPending].(bool); pending { + handle.Status = domain.HandleStarting + return handle, nil + } if handle.Metadata["executionTarget"] == string(domain.ExecutionTargetDirect) { return a.inspectDirect(handle), nil } + // Observation is read-only. A stuck accounting service or transport must + // not hold the supervisor's entire inspection batch indefinitely. + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() output, err := a.executor.Run(ctx, "sh", []string{"-s", "--", metadataString(handle, "logPath"), metadataString(handle, "metricsPath"), metadataString(handle, "sentinelPath"), handle.ExternalID}, []byte(slurmInspectionScript)) @@ -186,10 +244,14 @@ func (a *Adapter) Inspect(ctx context.Context, handle domain.ActivityHandle) (do const slurmInspectionScript = ` encode_file() { [ -n "$2" ] && [ -f "$2" ] && base64 < "$2" | tr -d '\n'; } -encode_command() { shift; "$@" 2>/dev/null | base64 | tr -d '\n' || true; } +encode_command() { shift; if command -v timeout >/dev/null 2>&1; then timeout 5 "$@" 2>/dev/null | base64 | tr -d '\n' || true; fi; } printf 'LOG='; encode_file LOG "$1"; printf '\n' printf 'METRICS='; encode_file METRICS "$2"; printf '\n' printf 'SENTINEL='; encode_file SENTINEL "$3"; printf '\n' +if [ -f "$3" ] && grep -q '^publication_complete=1$' "$3" && grep -Eq '^state=(completed|failed)$' "$3"; then + printf 'SACCT=\nSQUEUE=\nSCONTROL=\n' + exit 0 +fi printf 'SACCT='; encode_command SACCT sacct -j "$4" --noheader --parsable2 --format=State,ExitCode; printf '\n' printf 'SQUEUE='; encode_command SQUEUE squeue --noheader --jobs "$4" --format=%T; printf '\n' printf 'SCONTROL='; encode_command SCONTROL scontrol show job "$4" --oneliner; printf '\n' @@ -474,6 +536,9 @@ func slurmControlState(payload string) string { } func (a *Adapter) Stop(ctx context.Context, handle domain.ActivityHandle) error { + if pending, _ := handle.Metadata[ports.RuntimeReconciliationPending].(bool); pending { + return a.stopUncertainSubmission(ctx, handle) + } if handle.Metadata["executionTarget"] == string(domain.ExecutionTargetDirect) { a.directMu.Lock() cancel := a.directCancels[handle.ID] @@ -502,7 +567,7 @@ func (a *Adapter) startDirect(ctx context.Context, execution domain.ActivityExec return domain.ActivityHandle{}, err } startedAt := runtimecommon.UnixSeconds(time.Now()) - handle := domain.ActivityHandle{ID: execution.Run.ID + ":" + activity.ID, RunID: execution.Run.ID, + handle := domain.ActivityHandle{ID: execution.HandleID(), RunID: execution.Run.ID, ActivityID: activity.ID, ResourceID: execution.Resource.ID, RuntimeID: execution.RuntimeID, Status: domain.HandleRunning, StartedAt: startedAt, Metadata: map[string]any{ "executionTarget": string(domain.ExecutionTargetDirect), "slurmSubmission": "login-node", @@ -576,29 +641,7 @@ func batchScript(runID string, activity domain.Activity, partition, node string) return "", fmt.Errorf("activity entrypoint is required") } var script strings.Builder - script.WriteString("#!/bin/sh\n") - script.WriteString("#SBATCH --job-name=") - script.WriteString(shellToken("akoflow-" + activity.ID)) - script.WriteByte('\n') - script.WriteString("#SBATCH --output=") - script.WriteString(slurmLogPath(runID, activity.ID, "%j")) - script.WriteByte('\n') - if partition != "" { - script.WriteString("#SBATCH --partition=") - script.WriteString(shellToken(partition)) - script.WriteByte('\n') - } - if node != "" { - script.WriteString("#SBATCH --nodelist=") - script.WriteString(shellToken(node)) - script.WriteByte('\n') - } - if activity.Resources.CPU > 0 { - script.WriteString(fmt.Sprintf("#SBATCH --cpus-per-task=%d\n", int(activity.Resources.CPU+0.999))) - } - if activity.Resources.MemoryBytes > 0 { - script.WriteString(fmt.Sprintf("#SBATCH --mem=%dM\n", (activity.Resources.MemoryBytes+(1<<20)-1)/(1<<20))) - } + writeBatchDirectives(&script, runID, activity, partition, node) sentinelPrefix := slurmSentinelPrefix(runID, activity.ID) // Keep the sentinel in the submission directory. The activity command may // change into its workspace, and a relative sentinel path would otherwise @@ -633,7 +676,8 @@ for file do relative=${file#"$root"/}; size=$(wc -c < "$file") || exit 1; checks script.WriteString("finished_at=$(date +%s.%N); { printf 'state=%s\\nexit_code=%s\\nstarted_at=%s\\nfinished_at=%s\\nallocated_node=%s\\ncontainer_started_at=%s\\nartifact_root=%s\\n' \"$state\" \"$code\" \"$started_at\" \"$finished_at\" \"$allocated_node\" \"$container_started_at\" \"$artifact_root\"; ") script.WriteString("while IFS= read -r entry; do printf 'initial=%s\\n' \"$entry\"; done < \"$artifact_before\"; ") script.WriteString("while IFS= read -r entry; do printf 'final=%s\\n' \"$entry\"; done < \"$artifact_after\"; ") - script.WriteString("} > \"$sentinel\" || true; rm -f \"$artifact_before\" \"$artifact_after\" \"$container_start_marker\" \"$seed_links\" \"$seed_list\"; exit \"$code\"; }\n") + // Publish the complete manifest atomically before readers skip accounting. + script.WriteString("printf 'publication_complete=1\\n'; } > \"${sentinel}.publishing.$$\" && mv -f -- \"${sentinel}.publishing.$$\" \"$sentinel\"; rm -f \"${sentinel}.publishing.$$\" \"$artifact_before\" \"$artifact_after\" \"$container_start_marker\" \"$seed_links\" \"$seed_list\"; exit \"$code\"; }\n") script.WriteString("trap finish EXIT\nset -eu\n") image, err := slurmExecutable(activity.Command) if err != nil { @@ -784,3 +828,64 @@ func shellToken(value string) string { }, value) } func shellQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" } + +func (a *Adapter) applyEligibleNodeGuard(ctx context.Context, execution domain.ActivityExecutionContext, activity domain.Activity, script, plannedNode, node, partition, fallbackFeatures, fallbackExcluded string) (string, error) { + // Slurm's nodelist requests every listed node; it is not an alternatives + // list. Use feature constraints and exactly one node, then validate the + // allocated node against the verified eligible inventory before execution. + script = strings.Replace(script, "#SBATCH --nodelist="+node+"\n", "#SBATCH --nodes=1\n#SBATCH --constraint="+strings.ReplaceAll(fallbackFeatures, ",", "&")+"\n", 1) + // Feature matching alone admits nodes rejected by capacity/state checks. + // Exclude the inventory complement so Slurm cannot allocate such a node + // and fail the strict eligibility guard after the job has already started. + if fallbackExcluded != "" { + script = strings.Replace(script, "#SBATCH --nodes=1\n", "#SBATCH --nodes=1\n#SBATCH --exclude="+fallbackExcluded+"\n", 1) + } + // Fail closed on the allocated node before creating a missing workspace + // or invoking the activity. The marker proves shared filesystem access. + marker := filepath.Join(activity.Command.WorkingDirectory, ".akoflow-placement-"+shellToken(execution.HandleID())) + proof := runtimecommon.NewID("placement") + check := "set -eu\nmkdir -p -- " + shellQuote(activity.Command.WorkingDirectory) + "\ntest -d " + shellQuote(activity.Command.WorkingDirectory) + " || { echo 'Fallback workspace directory unavailable' >&2; exit 70; }\nprintf '%s' " + shellQuote(proof) + " > " + shellQuote(marker) + "\n" + if _, err := a.executor.Run(ctx, "sh", []string{"-s"}, []byte(check)); err != nil { + return "", fmt.Errorf("prepare fallback workspace proof: %w", err) + } + guard := "set -eu\ncase \"$(hostname -s)\" in " + strings.ReplaceAll(node, ",", "|") + ") ;; *) echo '[AKOFLOW ERROR] allocated fallback node is not eligible'; exit 70 ;; esac\n[ \"$(cat " + shellQuote(marker) + ")\" = " + shellQuote(proof) + " ] || { echo '[AKOFLOW ERROR] fallback workspace is not accessible'; exit 70; }\n" + image, _ := slurmExecutable(activity.Command) + if image != "" { + guard += "test -r " + shellQuote(image) + "\ncommand -v apptainer >/dev/null || command -v singularity >/dev/null\n" + } + guard += "printf '[AKOFLOW INFO] Eligible-node fallback: planned=%s allocated=%s partition=%s workspace verified\\n' " + shellQuote(plannedNode) + " \"$(hostname -s)\" " + shellQuote(partition) + "\n" + guard += "rm -f -- " + shellQuote(marker) + "\n" + script = strings.Replace(script, "trap finish EXIT\nset -eu\n", "trap finish EXIT\nset -eu\n"+guard, 1) + return script, nil +} + +func writeBatchDirectives(script *strings.Builder, runID string, activity domain.Activity, partition, node string) { + script.WriteString("#!/bin/sh\n") + script.WriteString("#SBATCH --job-name=") + script.WriteString(shellToken("akoflow-" + activity.ID)) + script.WriteByte('\n') + script.WriteString("#SBATCH --output=") + script.WriteString(slurmLogPath(runID, activity.ID, "%j")) + script.WriteByte('\n') + if partition != "" { + script.WriteString("#SBATCH --partition=") + script.WriteString(shellToken(partition)) + script.WriteByte('\n') + } + if node != "" { + script.WriteString("#SBATCH --nodelist=") + for index, candidate := range strings.Split(node, ",") { + if index > 0 { + script.WriteByte(',') + } + script.WriteString(shellToken(candidate)) + } + script.WriteByte('\n') + } + if activity.Resources.CPU > 0 { + script.WriteString(fmt.Sprintf("#SBATCH --cpus-per-task=%d\n", int(activity.Resources.CPU+0.999))) + } + if activity.Resources.MemoryBytes > 0 { + script.WriteString(fmt.Sprintf("#SBATCH --mem=%dM\n", (activity.Resources.MemoryBytes+(1<<20)-1)/(1<<20))) + } +} diff --git a/internal/provider/slurm/adapter_test.go b/internal/provider/slurm/adapter_test.go index 61ec2278..89381f11 100644 --- a/internal/provider/slurm/adapter_test.go +++ b/internal/provider/slurm/adapter_test.go @@ -2,6 +2,7 @@ package slurm import ( "context" + "crypto/sha256" "encoding/base64" "fmt" "os" @@ -24,6 +25,38 @@ type executorFake struct { type failingStatusExecutor struct{} +type lostSubmissionExecutor struct { + jobName string + cancelled []string + cleanupCancelled bool +} + +func (f *lostSubmissionExecutor) Run(ctx context.Context, name string, args []string, input []byte) ([]byte, error) { + switch name { + case "sbatch": + return nil, context.Canceled + case "squeue": + f.cleanupCancelled = ctx.Err() != nil + return []byte("123|" + f.jobName + "\n456|another-attempt\n"), nil + case "scancel": + f.cancelled = append(f.cancelled, args...) + return nil, nil + } + return nil, fmt.Errorf("unexpected command %s", name) +} + +func TestLostSubmissionCancelsOnlyExactAttemptWithFreshContext(t *testing.T) { + execution := domain.ActivityExecutionContext{Run: domain.ExecutionRun{ID: "run"}, Activity: domain.Activity{ID: "analysis", Command: domain.ActivityCommand{Entrypoint: "true"}}, Attempt: 2} + f := &lostSubmissionExecutor{jobName: fmt.Sprintf("akoflow-%x", sha256.Sum256([]byte(execution.HandleID())))} + a := NewWithConfig(f, Config{ScriptDirectory: t.TempDir(), SubmitFromStdin: true}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := a.Start(ctx, execution) + if err == nil || f.cleanupCancelled || len(f.cancelled) != 1 || f.cancelled[0] != "123" { + t.Fatalf("err=%v cleanupCancelled=%v cancelled=%v", err, f.cleanupCancelled, f.cancelled) + } +} + func (failingStatusExecutor) Run(_ context.Context, name string, _ []string, _ []byte) ([]byte, error) { return nil, fmt.Errorf("%s unavailable", name) } @@ -384,6 +417,76 @@ func TestSentinelAndScontrolStatusParsing(t *testing.T) { } } +func TestAtomicTerminalInspectionSkipsAccountingButLegacyDoesNot(t *testing.T) { + for _, atomic := range []bool{false, true} { + t.Run(fmt.Sprint(atomic), func(t *testing.T) { + dir := t.TempDir() + sentinel := filepath.Join(dir, "job.status") + payload := "state=completed\nexit_code=0\nstarted_at=100\nfinished_at=104\n" + if atomic { + payload += "publication_complete=1\n" + } + if err := os.WriteFile(sentinel, []byte(payload), 0600); err != nil { + t.Fatal(err) + } + marker := filepath.Join(dir, "accounting-called") + // The shell fixture is portable to hosts without GNU coreutils. + if err := os.WriteFile(filepath.Join(dir, "timeout"), []byte("#!/bin/sh\nshift\nexec \"$@\"\n"), 0700); err != nil { + t.Fatal(err) + } + for _, name := range []string{"sacct", "squeue", "scontrol"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte("#!/bin/sh\nprintf called >> '"+marker+"'\n"), 0700); err != nil { + t.Fatal(err) + } + } + cmd := exec.Command("sh", "-s", "--", "", "", sentinel, "1") + cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH")) + cmd.Stdin = strings.NewReader(slurmInspectionScript) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%s: %v", out, err) + } + sections := inspectionSections(string(out)) + observed, found := applySentinelStatus(domain.ActivityHandle{}, sections["SENTINEL"]) + if !found || observed.Status != domain.HandleCompleted || observed.FinishedAt != 104 { + t.Fatalf("lost completed sentinel: %+v", observed) + } + _, err = os.Stat(marker) + if atomic && !os.IsNotExist(err) { + t.Fatal("atomic terminal sentinel queried accounting") + } + if !atomic && err != nil { + t.Fatal("legacy sentinel skipped bounded accounting") + } + }) + } +} + +type deadlineInspectionExecutor struct{ deadline time.Time } + +func (e *deadlineInspectionExecutor) Run(ctx context.Context, _ string, _ []string, _ []byte) ([]byte, error) { + e.deadline, _ = ctx.Deadline() + return nil, context.DeadlineExceeded +} + +func TestInspectionDeadlineDoesNotTurnTimeoutIntoTerminalStatus(t *testing.T) { + executor := &deadlineInspectionExecutor{} + before := time.Now() + handle, err := New(executor, "").Inspect(context.Background(), domain.ActivityHandle{Status: domain.HandleRunning}) + if err != nil || handle.Status != domain.HandleRunning || handle.Metadata["statusQueryWarning"] == nil { + t.Fatalf("timeout changed state: %+v %v", handle, err) + } + if executor.deadline.IsZero() || executor.deadline.Sub(before) > 31*time.Second { + t.Fatal("inspection not bounded") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _, _ = New(executor, "").Inspect(ctx, handle) + if executor.deadline.Sub(time.Now()) > time.Second { + t.Fatal("inspection extended caller deadline") + } +} + func TestSentinelRecordsContainerStartupTiming(t *testing.T) { executor := &executorFake{output: []byte("state=completed\nexit_code=0\nstarted_at=100\nallocated_node=bora017\ncontainer_started_at=103.25\n")} handle, found := New(executor, "").sentinelStatus(context.Background(), domain.ActivityHandle{ diff --git a/internal/provider/slurm/connection_factory.go b/internal/provider/slurm/connection_factory.go index 35924361..63ccfa71 100644 --- a/internal/provider/slurm/connection_factory.go +++ b/internal/provider/slurm/connection_factory.go @@ -48,7 +48,7 @@ func (f ConnectionFactory) Build(runtime domain.EnvironmentRuntime, connection d partition = configString(connection.Configuration, "partition") } return NewWithConfig(executor, Config{Partition: partition, ScriptDirectory: scriptDirectory, - SubmitFromStdin: remote}), nil + SubmitFromStdin: remote, EligibleNodeFallback: configBool(runtime.Configuration, "eligibleNodeFallback", configBool(connection.Configuration, "eligibleNodeFallback", false))}), nil } func knownHostsFile(connection domain.EnvironmentConnection) string { diff --git a/internal/provider/slurm/console_runner.go b/internal/provider/slurm/console_runner.go index 9f9523d3..a6fb9fce 100644 --- a/internal/provider/slurm/console_runner.go +++ b/internal/provider/slurm/console_runner.go @@ -23,6 +23,9 @@ func (r ConsoleRunner) RunConsoleCommand(ctx context.Context, connection domain. if connection.Type == domain.ConnectionSSH { executor = runtimecommon.NewSSHCommandExecutor(executor, connection) } + if direct, _ := connection.Configuration["skipSchedulerCheck"].(bool); direct { + resource.ExecutionTarget = domain.ExecutionTargetDirect + } script := consoleScript(resource, command) output, err := executor.Run(ctx, "/bin/sh", []string{"-s"}, []byte(script)) if err != nil { diff --git a/internal/provider/slurm/console_runner_test.go b/internal/provider/slurm/console_runner_test.go index 73a253bf..b8741d40 100644 --- a/internal/provider/slurm/console_runner_test.go +++ b/internal/provider/slurm/console_runner_test.go @@ -1,6 +1,7 @@ package slurm import ( + "context" "strings" "testing" @@ -8,6 +9,28 @@ import ( domainconsole "github.com/UFFeScience/akoflow/internal/domain/console" ) +type consoleCaptureExecutor struct{ script string } + +func (e *consoleCaptureExecutor) Run(_ context.Context, _ string, _ []string, input []byte) ([]byte, error) { + e.script = string(input) + return nil, nil +} + +func TestConsoleSkipsSchedulerOnDirectConnection(t *testing.T) { + executor := &consoleCaptureExecutor{} + _, _, _, _, err := (ConsoleRunner{ + Executor: executor, + }).RunConsoleCommand(context.Background(), domain.EnvironmentConnection{ + Type: domain.ConnectionLocal, + Configuration: map[string]any{"skipSchedulerCheck": true}, + }, domain.Resource{}, domainconsole.Command{ + Command: "hostname", + }) + if err != nil || strings.Contains(executor.script, "srun") || !strings.Contains(executor.script, "exec /bin/sh -lc") { + t.Fatalf("script=%q err=%v", executor.script, err) + } +} + func TestConsoleScriptRunsDirectlyOnLoginNode(t *testing.T) { script := consoleScript(domain.Resource{ExecutionTarget: domain.ExecutionTargetDirect}, domainconsole.Command{ Command: "hostname && pwd", WorkingDirectory: "/scratch/a b", Environment: map[string]string{"RUN": "one two"}, diff --git a/internal/provider/slurm/eligible_nodes.go b/internal/provider/slurm/eligible_nodes.go new file mode 100644 index 00000000..b36a9178 --- /dev/null +++ b/internal/provider/slurm/eligible_nodes.go @@ -0,0 +1,116 @@ +package slurm + +import ( + "context" + "fmt" + "sort" + "strconv" + "strings" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +// Discover within the requested partition, never across heterogeneous families. +func (a *Adapter) eligibleNodes(ctx context.Context, partition, original string, resource domain.Resource) (string, string, string, error) { + if partition == "" { + return "", "", "", fmt.Errorf("eligible-node fallback requires an explicit partition") + } + originalOutput, err := a.executor.Run(ctx, "scontrol", []string{"show", "node", original, "--oneliner"}, nil) + if err != nil { + return "", "", "", fmt.Errorf("inspect planned node: %w", err) + } + fields := slurmFields(string(originalOutput)) + features := fields["AvailableFeatures"] + if fields["NodeName"] != original || features == "" || features == "(null)" { + return "", "", "", fmt.Errorf("planned node %q has no verified feature set", original) + } + for _, feature := range strings.Split(features, ",") { + if feature == "" || shellToken(feature) != feature { + return "", "", "", fmt.Errorf("planned node %q has unsupported feature syntax", original) + } + } + output, err := a.executor.Run(ctx, "sinfo", []string{"--partition", partition, "--Node", "--noheader", "--format=%N|%c|%m|%f|%t"}, nil) + if err != nil { + return "", "", "", fmt.Errorf("discover eligible nodes: %w", err) + } + nodes, err := selectEligibleNodes(string(output), features, resource) + if err != nil { + return "", "", "", err + } + allowed := map[string]bool{} + for _, node := range strings.Split(nodes, ",") { + allowed[node] = true + } + excluded := []string{} + seen := map[string]bool{} + for _, row := range strings.Split(string(output), "\n") { + cells := strings.Split(strings.TrimSpace(row), "|") + // The SSH executor merges stderr into stdout. Ignore diagnostic lines, + // just as selectEligibleNodes does; only complete inventory rows name + // nodes. A malformed name within an inventory row still fails closed. + if len(cells) != 5 { + continue + } + name := strings.TrimSpace(cells[0]) + if name == "" { + continue + } + if shellToken(name) != name { + return "", "", "", fmt.Errorf("invalid node inventory name") + } + if !allowed[name] && !seen[name] { + excluded = append(excluded, name) + seen[name] = true + } + } + sort.Strings(excluded) + return nodes, features, strings.Join(excluded, ","), nil +} + +func slurmFields(output string) map[string]string { + result := map[string]string{} + for _, field := range strings.Fields(output) { + key, value, ok := strings.Cut(field, "=") + if ok { + result[key] = value + } + } + return result +} + +func selectEligibleNodes(output, features string, resource domain.Resource) (string, error) { + nodes := map[string]bool{} + for _, row := range strings.Split(output, "\n") { + cells := strings.Split(strings.TrimSpace(row), "|") + if len(cells) != 5 { + continue + } + cpus, errCPU := strconv.Atoi(cells[1]) + memory, errMemory := strconv.ParseInt(cells[2], 10, 64) + if errCPU != nil || errMemory != nil || cpus < resource.CPUCores || memory < (resource.MemoryBytes+(1<<20)-1)/(1<<20) { + continue + } + state := strings.TrimRight(cells[4], "~#$@+") + if state != "idle" && state != "mix" && state != "alloc" { + continue + } + compatible := true + for _, feature := range strings.Split(features, ",") { + if !strings.Contains(","+cells[3]+",", ","+feature+",") { + compatible = false + } + } + if compatible && shellToken(cells[0]) == cells[0] && cells[0] != "" { + nodes[cells[0]] = true + } + } + selected := []string{} + for node := range nodes { + selected = append(selected, node) + } + sort.Strings(selected) + if len(selected) == 0 { + return "", fmt.Errorf("no eligible nodes preserve the planned resource characteristics") + } + return strings.Join(selected, ","), nil +} diff --git a/internal/provider/slurm/eligible_nodes_test.go b/internal/provider/slurm/eligible_nodes_test.go new file mode 100644 index 00000000..d31e1049 --- /dev/null +++ b/internal/provider/slurm/eligible_nodes_test.go @@ -0,0 +1,92 @@ +package slurm + +import ( + "context" + "github.com/UFFeScience/akoflow/internal/domain" + "strings" + "testing" +) + +type placementExecutor struct { + script string + proof bool + proofScript string +} + +func (e *placementExecutor) Run(_ context.Context, name string, _ []string, input []byte) ([]byte, error) { + switch name { + case "scontrol": + return []byte("NodeName=bora001 AvailableFeatures=bora,intel,cascadelake,omnipath"), nil + case "sinfo": + return []byte( + ("SSH diagnostic: merged stderr line\nbora003|36|191000|bora,intel,cascadelake,omnipath|idle\nbora006|36" + + "|191000|bora,intel,cascadelake,omnipath|idle\nbora021|36|191000|bora,intel,cascadelake,omnipath|drain" + + "\nbora022|16|191000|bora,intel,cascadelake,omnipath|idle"), + ), nil + case "sh": + e.proof = true + e.proofScript = string(input) + return nil, nil + case "sbatch": + e.script = string(input) + return []byte("123"), nil + } + return nil, nil +} + +func TestFallbackSubmissionKeepsRequestsAndGuardsBeforeCommand(t *testing.T) { + executor := &placementExecutor{} + adapter := NewWithConfig(executor, Config{Partition: "routage", ScriptDirectory: t.TempDir(), SubmitFromStdin: true, EligibleNodeFallback: true}) + handle, err := adapter.Start(context.Background(), domain.ActivityExecutionContext{ + Run: domain.ExecutionRun{ID: "run"}, RuntimeID: "slurm", + Resource: domain.Resource{ID: "bora", ProviderID: "bora001", Type: domain.ResourceHPCMachine, CPUCores: 36}, + Activity: domain.Activity{ID: "activity", Resources: domain.ActivityResources{CPU: 1, MemoryBytes: 256 << 20}, Command: domain.ActivityCommand{Entrypoint: "/bin/true", WorkingDirectory: "/shared/workspace"}}, + }) + if err != nil { + t.Fatal(err) + } + for _, expected := range []string{ + "#SBATCH --partition=routage", + "#SBATCH --nodes=1", + "#SBATCH --exclude=bora021,bora022", + "#SBATCH --constraint=bora&intel&cascadelake&omnipath", + "#SBATCH --cpus-per-task=1", + "#SBATCH --mem=256M", + "fallback workspace is not accessible", + "Eligible-node fallback", + } { + if !strings.Contains(executor.script, expected) { + t.Fatalf("missing %q", expected) + } + } + if strings.Contains(executor.script, "#SBATCH --nodelist=") { + t.Fatal("fallback must not request all eligible nodes") + } + if !executor.proof || handle.Metadata["plannedNode"] != "bora001" || handle.Metadata["eligibleNodes"] != "bora003,bora006" { + t.Fatalf("missing proof or provenance: %+v", handle) + } + if !strings.Contains(executor.proofScript, "mkdir -p -- '/shared/workspace'") { + t.Fatal("new root activity workspace must be initialized before shared access proof") + } + if strings.Index(executor.script, "fallback workspace is not accessible") > strings.LastIndex(executor.script, "/bin/true") { + t.Fatal("workspace guard runs after activity") + } +} + +func TestEligibleNodesPreserveFamilyCapacityAndState(t *testing.T) { + rows := ("bora003|36|191000|bora,intel,cascadelake,omnipath|idle\narm01|224|255000|arm,cavium|idle\nbora004|36|1" + + "91000|bora,intel,cascadelake,omnipath|drain$\nbora005|16|191000|bora,intel,cascadelake,omnipath|idle\n" + + "bora006|36|100|bora,intel,cascadelake,omnipath|idle\nbora007|36|191000|bora,intel,cascadelake,omnipat" + + "h|alloc") + got, err := selectEligibleNodes(rows, "bora,intel,cascadelake,omnipath", domain.Resource{CPUCores: 36, MemoryBytes: 191000 << 20}) + if err != nil || got != "bora003,bora007" { + t.Fatalf("nodes=%s err=%v", got, err) + } +} + +func TestEligibleNodesFailClosed(t *testing.T) { + _, err := selectEligibleNodes("arm01|224|255000|arm,cavium|idle", "amd,zen2,diablo", domain.Resource{}) + if err == nil || !strings.Contains(err.Error(), "no eligible") { + t.Fatal(err) + } +} diff --git a/internal/provider/slurm/submission_reconciliation.go b/internal/provider/slurm/submission_reconciliation.go new file mode 100644 index 00000000..e2bc5ce2 --- /dev/null +++ b/internal/provider/slurm/submission_reconciliation.go @@ -0,0 +1,72 @@ +package slurm + +import ( + "context" + "crypto/sha256" + "fmt" + "strconv" + "strings" + "time" + + "github.com/UFFeScience/akoflow/internal/application/ports" + "github.com/UFFeScience/akoflow/internal/domain" +) + +func uncertainSubmission(jobName string, cause error) error { + return ports.NewRuntimeReconciliationError(cause, map[string]any{ + "slurmSubmissionName": jobName, "executionTarget": string(domain.ExecutionTargetBatch), + }) +} + +func (a *Adapter) matchingSubmissionJobs(ctx context.Context, jobName string) ([]string, error) { + output, err := a.executor.Run(ctx, "squeue", []string{"--me", "--noheader", "--format=%i|%j"}, nil) + if err != nil { + return nil, err + } + var jobs []string + for _, line := range strings.Split(string(output), "\n") { + id, name, ok := strings.Cut(strings.TrimSpace(line), "|") + if !ok || name != jobName { + continue + } + numericID, parseErr := strconv.ParseUint(id, 10, 64) + if parseErr != nil || numericID == 0 { + return nil, fmt.Errorf("invalid exact-attempt Slurm job ID") + } + jobs = append(jobs, id) + } + return jobs, nil +} + +func (a *Adapter) stopUncertainSubmission(ctx context.Context, handle domain.ActivityHandle) error { + jobName, _ := handle.Metadata["slurmSubmissionName"].(string) + if handle.ID == "" || jobName != fmt.Sprintf("akoflow-%x", sha256.Sum256([]byte(handle.ID))) { + return fmt.Errorf("uncertain Slurm submission identity does not match exact attempt") + } + if a.executor == nil { + return fmt.Errorf("slurm command executor is required") + } + cleanup, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + jobs, err := a.matchingSubmissionJobs(cleanup, jobName) + if err != nil { + return fmt.Errorf("inspect uncertain submission: %w", err) + } + if len(jobs) > 0 { + if _, err := a.executor.Run(cleanup, "scancel", jobs, nil); err != nil { + return fmt.Errorf("cancel uncertain submission: %w", err) + } + } + for len(jobs) > 0 { + select { + case <-cleanup.Done(): + return fmt.Errorf("exact-attempt cancellation not confirmed: %w", cleanup.Err()) + case <-time.After(100 * time.Millisecond): + } + jobs, err = a.matchingSubmissionJobs(cleanup, jobName) + if err != nil { + return fmt.Errorf("verify uncertain submission cancellation: %w", err) + } + } + return nil +} diff --git a/internal/provider/slurm/uncertain_submission_test.go b/internal/provider/slurm/uncertain_submission_test.go new file mode 100644 index 00000000..a65120e4 --- /dev/null +++ b/internal/provider/slurm/uncertain_submission_test.go @@ -0,0 +1,89 @@ +package slurm + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "reflect" + "testing" + + "github.com/UFFeScience/akoflow/internal/application/ports" + + "github.com/UFFeScience/akoflow/internal/domain" +) + +type unavailableSubmissionExecutor struct{} + +func (unavailableSubmissionExecutor) Run(context.Context, string, []string, []byte) ([]byte, error) { + return nil, fmt.Errorf("transport unavailable after submission") +} + +type reconciliationExecutor struct { + name string + queries int + cancelled []string + verificationFailure bool +} + +func (e *reconciliationExecutor) Run(_ context.Context, command string, args []string, _ []byte) ([]byte, error) { + if command == "scancel" { + e.cancelled = append(e.cancelled, args...) + return nil, nil + } + e.queries++ + if e.queries > 1 { + if e.verificationFailure { + return nil, fmt.Errorf("queue verification unavailable") + } + return []byte("456|unrelated-attempt\n"), nil + } + return []byte("123|" + e.name + "\n456|unrelated-attempt\n"), nil +} + +func TestUncertainSubmissionStopRequiresExactTerminalProof(t *testing.T) { + for _, failure := range []bool{false, true} { + t.Run(fmt.Sprint(failure), func(t *testing.T) { + id := "run:activity:attempt-2" + name := fmt.Sprintf("akoflow-%x", sha256.Sum256([]byte(id))) + executor := &reconciliationExecutor{name: name, verificationFailure: failure} + a := New(executor, "routage") + handle := domain.ActivityHandle{ID: id, Status: domain.HandleStarting, Metadata: map[string]any{ + ports.RuntimeReconciliationPending: true, "slurmSubmissionName": name, + }} + err := a.Stop(context.Background(), handle) + if (err != nil) != failure { + t.Fatalf("terminal proof mismatch: %v", err) + } + if !reflect.DeepEqual(executor.cancelled, []string{"123"}) { + t.Fatalf("wrong jobs cancelled: %v", executor.cancelled) + } + }) + } +} + +func TestUncertainSubmissionRejectsMismatchedIdentity(t *testing.T) { + executor := &reconciliationExecutor{} + a := New(executor, "routage") + err := a.Stop(context.Background(), domain.ActivityHandle{ID: "attempt", Metadata: map[string]any{ + ports.RuntimeReconciliationPending: true, "slurmSubmissionName": "another-attempt", + }}) + if err == nil || executor.queries != 0 || len(executor.cancelled) != 0 { + t.Fatal("identity mismatch must fail before external commands") + } +} + +func TestLostSubmissionExportsReconciliationEvidenceWhenQueueUnavailable(t *testing.T) { + a := NewWithConfig(unavailableSubmissionExecutor{}, Config{ScriptDirectory: t.TempDir(), SubmitFromStdin: true}) + execution := domain.ActivityExecutionContext{ + Run: domain.ExecutionRun{ID: "run"}, Activity: domain.Activity{ID: "activity", Command: domain.ActivityCommand{Entrypoint: "true"}}, Attempt: 2, + } + _, err := a.Start(context.Background(), execution) + var pending interface{ RuntimeReconciliationMetadata() map[string]any } + if !errors.As(err, &pending) { + t.Fatalf("lost submission must retain durable reconciliation evidence: %v", err) + } + if pending.RuntimeReconciliationMetadata()["slurmSubmissionName"] == nil { + t.Fatal("exact attempt identity missing") + } +} diff --git a/internal/provider/ssh_executor.go b/internal/provider/ssh_executor.go index 5584dba8..e2d31db5 100644 --- a/internal/provider/ssh_executor.go +++ b/internal/provider/ssh_executor.go @@ -16,16 +16,17 @@ import ( // to SSH configuration, an agent, or a configured identity file; no secret is // stored in the environment definition. type SSHCommandExecutor struct { - Executor CommandExecutor - ConnectionID string - Endpoint string - Username string - Port int - IdentityFile string - ProxyCommand string - HostKeyAlias string - KnownHostsFile string - ForwardAgent bool + Executor CommandExecutor + ConnectionID string + Endpoint string + Username string + Port int + IdentityFile string + ProxyCommand string + HostKeyAlias string + KnownHostsFile string + ForwardAgent bool + StrictHostKeyChecking bool } // NewSSHCommandExecutor builds the canonical daemon-side SSH transport for an @@ -45,12 +46,13 @@ func NewSSHCommandExecutor(executor CommandExecutor, connection domain.Environme } return SSHCommandExecutor{ Executor: executor, ConnectionID: connection.ID, Endpoint: connection.Endpoint, Username: connection.Username, - Port: connectionInt(connection.Configuration, "port"), - IdentityFile: identity, - ProxyCommand: connectionString(connection.Configuration, "proxyCommand"), - HostKeyAlias: connectionString(connection.Configuration, "hostKeyAlias"), - KnownHostsFile: knownHosts, - ForwardAgent: connectionBool(connection.Configuration, "forwardAgent"), + Port: connectionInt(connection.Configuration, "port"), + IdentityFile: identity, + ProxyCommand: connectionString(connection.Configuration, "proxyCommand"), + HostKeyAlias: connectionString(connection.Configuration, "hostKeyAlias"), + KnownHostsFile: knownHosts, + ForwardAgent: connectionBool(connection.Configuration, "forwardAgent"), + StrictHostKeyChecking: connectionBool(connection.Configuration, "strictHostKeyChecking"), } } @@ -73,7 +75,12 @@ func (e SSHCommandExecutor) Run(ctx context.Context, name string, args []string, if err := os.MkdirAll(filepath.Dir(e.KnownHostsFile), 0o700); err != nil { return nil, fmt.Errorf("create SSH known-hosts directory: %w", err) } - sshArgs = append(sshArgs, "-o", "UserKnownHostsFile="+e.KnownHostsFile, "-o", "StrictHostKeyChecking=accept-new") + policy := "accept-new" + if e.StrictHostKeyChecking { + policy = "yes" + sshArgs = append(sshArgs, "-o", "HostKeyAlgorithms=ssh-ed25519") + } + sshArgs = append(sshArgs, "-o", "UserKnownHostsFile="+e.KnownHostsFile, "-o", "StrictHostKeyChecking="+policy) } if e.Port > 0 { sshArgs = append(sshArgs, "-p", strconv.Itoa(e.Port)) diff --git a/internal/provider/ssh_session.go b/internal/provider/ssh_session.go index b32bd70d..8fdb894b 100644 --- a/internal/provider/ssh_session.go +++ b/internal/provider/ssh_session.go @@ -14,6 +14,7 @@ import ( const defaultSSHControlPersistSeconds = 180 const SSHChannelLimit = 4 +const SSHBulkChannelLimit = SSHChannelLimit - 1 // SSHSessionKey contains every setting that changes the effective SSH // transport. Its digest is safe to use as a Unix socket name and does not @@ -32,14 +33,16 @@ type SSHSessionKey struct { } type sshSessionManager struct { - mu sync.Mutex - locks map[string]*sync.Mutex - channels map[string]chan struct{} + mu sync.Mutex + locks map[string]*sync.Mutex + channels map[string]chan struct{} + bulkChannels map[string]chan struct{} } var sharedSSHSessions = sshSessionManager{ - locks: make(map[string]*sync.Mutex), - channels: make(map[string]chan struct{}), + locks: make(map[string]*sync.Mutex), + channels: make(map[string]chan struct{}), + bulkChannels: make(map[string]chan struct{}), } // AcquireSSHChannel reserves one remote command session on a multiplexed SSH @@ -60,6 +63,34 @@ func AcquireSSHChannel(ctx context.Context, controlPath string) (func(), error) } } +// AcquireSSHBulkChannel keeps one of the existing session slots available for +// runtime inspection and control commands. It never increases server load or +// opens a separate transport to evade the shared session limit. +func AcquireSSHBulkChannel(ctx context.Context, controlPath string) (func(), error) { + if strings.TrimSpace(controlPath) == "" { + return func() {}, nil + } + sharedSSHSessions.mu.Lock() + queue := sharedSSHSessions.bulkChannels[controlPath] + if queue == nil { + queue = make(chan struct{}, SSHBulkChannelLimit) + sharedSSHSessions.bulkChannels[controlPath] = queue + } + sharedSSHSessions.mu.Unlock() + select { + case queue <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + release, err := AcquireSSHChannel(ctx, controlPath) + if err != nil { + <-queue + return nil, err + } + var once sync.Once + return func() { once.Do(func() { release(); <-queue }) }, nil +} + func (m *sshSessionManager) channelQueue(path string) chan struct{} { m.mu.Lock() defer m.mu.Unlock() diff --git a/internal/provider/ssh_session_test.go b/internal/provider/ssh_session_test.go index 3b81fb51..2470332f 100644 --- a/internal/provider/ssh_session_test.go +++ b/internal/provider/ssh_session_test.go @@ -72,6 +72,75 @@ func TestSSHChannelQueueHonorsContextCancellation(t *testing.T) { } } +func TestSSHBulkChannelsReserveControlCapacity(t *testing.T) { + path := filepath.Join(t.TempDir(), "master") + var releases []func() + for range SSHBulkChannelLimit { + release, err := AcquireSSHBulkChannel(context.Background(), path) + if err != nil { + t.Fatal(err) + } + releases = append(releases, release) + defer release() + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + control, err := AcquireSSHChannel(ctx, path) + if err != nil { + t.Fatalf("bulk starved control: %v", err) + } + control() + blocked, stop := context.WithCancel(context.Background()) + stop() + if release, err := AcquireSSHBulkChannel(blocked, path); err == nil { + release() + t.Fatal("bulk exceeded reserved limit") + } + other, err := AcquireSSHBulkChannel(ctx, filepath.Join(t.TempDir(), "other")) + if err != nil { + t.Fatalf("independent transport blocked: %v", err) + } + other() + releases[0]() + releases[0]() // Release must remain idempotent. + replacement, err := AcquireSSHBulkChannel(ctx, path) + if err != nil { + t.Fatalf("released bulk slot unavailable: %v", err) + } + replacement() +} + +func TestSSHBulkCancellationReleasesReservationWhileTotalFull(t *testing.T) { + path := filepath.Join(t.TempDir(), "master") + var controls []func() + for range SSHChannelLimit { + release, err := AcquireSSHChannel(context.Background(), path) + if err != nil { + t.Fatal(err) + } + controls = append(controls, release) + defer release() + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + if release, err := AcquireSSHBulkChannel(ctx, path); err == nil { + release() + t.Fatal("bulk acquired a full transport") + } + for _, release := range controls { + release() + } + available, stop := context.WithTimeout(context.Background(), time.Second) + defer stop() + for range SSHBulkChannelLimit { + release, err := AcquireSSHBulkChannel(available, path) + if err != nil { + t.Fatalf("cancelled bulk reservation leaked: %v", err) + } + defer release() + } +} + func TestSSHSessionPathUsesAllEffectiveConnectionSettings(t *testing.T) { directory := t.TempDir() t.Setenv("AKOFLOW_SSH_CONTROL_DIRECTORY", directory) diff --git a/internal/provider/storage/sshfilesystem/archive_test.go b/internal/provider/storage/sshfilesystem/archive_test.go new file mode 100644 index 00000000..66359637 --- /dev/null +++ b/internal/provider/storage/sshfilesystem/archive_test.go @@ -0,0 +1,59 @@ +package sshfilesystem + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/provider" +) + +func TestNativeArchivePublishesOnlyAfterRemoteValidation(t *testing.T) { + executor := &commandFake{output: []byte{}} + driver := New(connectionStoreFake{connection: testConnection()}, executor) + storage := testStorage() + result, err := driver.ArchiveDirectory(context.Background(), storage, "runs/run") + if err != nil || result != "runs/run.tar.gz" || executor.calls != 1 { + t.Fatalf("archive=%q calls=%d error=%v", result, executor.calls, err) + } + executor.err = errors.New("archive integrity check failed") + if result, err := driver.ArchiveDirectory(context.Background(), storage, "runs/run"); err == nil || result != "" { + t.Fatal("failed archive must not be published") + } + for _, value := range []string{"", "..", "../outside"} { + if _, err := driver.ArchiveDirectory(context.Background(), storage, value); err == nil { + t.Fatalf("accepted archive path %q", value) + } + } + storage.ReadOnly = true + if _, err := driver.ArchiveDirectory(context.Background(), storage, "runs/run"); err == nil { + t.Fatal("read-only archive accepted") + } +} + +func TestNativeArchiveRealLinuxFilesystem(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("worker filesystem scripts require GNU realpath/find") + } + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "runs", "run"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "runs", "run", "output.txt"), []byte("preserved output"), 0600); err != nil { + t.Fatal(err) + } + driver := New(connectionStoreFake{connection: &domain.EnvironmentConnection{Type: domain.ConnectionLocal}}, provider.OSCommandExecutor{}) + storage := domain.StorageResource{Endpoint: root, Metadata: map[string]any{"connectionId": "local"}} + archive, err := driver.ArchiveDirectory(context.Background(), storage, "runs/run") + if err != nil { + t.Fatal(err) + } + output, err := (provider.OSCommandExecutor{}).Run(context.Background(), "tar", []string{"-xOzf", filepath.Join(root, archive), "runs/run/output.txt"}, nil) + if err != nil || string(output) != "preserved output" { + t.Fatalf("archive contents=%q error=%v", output, err) + } +} diff --git a/internal/provider/storage/sshfilesystem/binary_write_test.go b/internal/provider/storage/sshfilesystem/binary_write_test.go new file mode 100644 index 00000000..6929b2bd --- /dev/null +++ b/internal/provider/storage/sshfilesystem/binary_write_test.go @@ -0,0 +1,24 @@ +package sshfilesystem + +import ( + "bytes" + "context" + "testing" + + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/provider" +) + +func TestRealShellWritePreservesBinaryStdin(t *testing.T) { + root := t.TempDir() + storage := domain.StorageResource{ID: "local", Type: domain.StorageSSH, Endpoint: root, + Metadata: map[string]any{"connectionId": "local"}} + driver := New(connectionStoreFake{connection: &domain.EnvironmentConnection{Type: domain.ConnectionLocal}}, provider.OSCommandExecutor{}) + want := []byte{0, 1, 255, 10, 13, 0, 42} + // Exercise the actual shell transport independently of GNU realpath, + // which is required by the Linux worker's filesystem scripts. + got, err := driver.runInput(context.Background(), storage, "cat", want, root, "binary.tar.gz") + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("binary payload mismatch: length=%d error=%v", len(got), err) + } +} diff --git a/internal/provider/storage/sshfilesystem/budget.go b/internal/provider/storage/sshfilesystem/budget.go new file mode 100644 index 00000000..6dda9127 --- /dev/null +++ b/internal/provider/storage/sshfilesystem/budget.go @@ -0,0 +1,123 @@ +package sshfilesystem + +import ( + "context" + "crypto/sha256" + "fmt" + "math" + "strconv" + "strings" + "time" + + "github.com/UFFeScience/akoflow/internal/domain" + "github.com/UFFeScience/akoflow/internal/domain/storagebudget" +) + +// WorkspaceBudget reads the mounted filesystem and its current user's quota. +// Unknown quota formats fail closed; filesystem free space alone cannot prove +// the user can write. Project/group quotas require a separate policy observer. +func (d *Driver) WorkspaceBudget(ctx context.Context, storage domain.StorageResource, value string) (storagebudget.Snapshot, error) { + root, relative, err := browseRoot(storage, value) + if err != nil { + return storagebudget.Snapshot{}, err + } + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + output, err := d.run(ctx, storage, workspaceBudgetScript, root, relative) + if err != nil { + return storagebudget.Snapshot{}, fmt.Errorf("observe filesystem/user quota: %w", err) + } + available, filesystem, uid, err := parseWorkspaceBudget(string(output)) + if err != nil { + return storagebudget.Snapshot{}, err + } + // Account identity is shared across logical nodes of the same connection; + // do not expose a remote filesystem hostname or user name in the snapshot. + identity := metadataString(storage.Metadata, "connectionId") + "\x00" + filesystem + "\x00" + uid + return storagebudget.Snapshot{StorageID: fmt.Sprintf("ssh-user-quota-%x", sha256.Sum256([]byte(identity))), AvailableBytes: available, ObservedAt: time.Now().UTC(), Source: "live-posix-filesystem-and-user-quota-v1"}, nil +} + +const workspaceBudgetScript = commonScript + "\n" + ` +set -eu +safe_existing || exit 65 +target="$resolved" +test -d "$target" +usage=$(LC_ALL=C df -Pk -- "$target") +mount=$(printf '%s\n' "$usage" | awk 'NR==2 { print $6 }') +[ -n "$mount" ] || exit 65 +printf '__AKO_DF__\n%s\n__AKO_UID__\n' "$usage" +id -u +printf '__AKO_QUOTA__\n' +if LC_ALL=C quota -w -v -p -f "$mount" 2>&1; then + : +else + result=$? + # Over-quota tools may return 1 with valid data. Other errors are unknown. + [ "$result" -eq 1 ] || exit "$result" +fi +printf '__AKO_END__\n' +` + +func parseWorkspaceBudget(output string) (int64, string, string, error) { + section, filesystem, uid := "", "", "" + var available int64 + matched := false + number := func(value string) (int64, error) { + return strconv.ParseInt(strings.TrimSuffix(value, "*"), 10, 64) + } + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + switch line { + case "__AKO_DF__", "__AKO_UID__", "__AKO_QUOTA__", "__AKO_END__": + section = line + continue + } + fields := strings.Fields(line) + switch section { + case "__AKO_DF__": + if len(fields) != 6 || fields[0] == "Filesystem" { + continue + } + free, err := number(fields[3]) + if err != nil || free < 0 || free > math.MaxInt64/1024 || filesystem != "" { + return 0, "", "", fmt.Errorf("invalid or ambiguous filesystem capacity") + } + filesystem, available = fields[0], free*1024 + case "__AKO_UID__": + if _, err := strconv.ParseUint(line, 10, 64); err != nil || uid != "" { + return 0, "", "", fmt.Errorf("invalid quota account identity") + } + uid = line + case "__AKO_QUOTA__": + if len(fields) == 0 || fields[0] != filesystem { + continue + } + if len(fields) != 9 || matched { + return 0, "", "", fmt.Errorf("invalid or ambiguous quota row") + } + used, err := number(fields[1]) + if err != nil || used < 0 { + return 0, "", "", fmt.Errorf("invalid quota usage") + } + for _, index := range []int{2, 3} { + limit, err := number(fields[index]) + if err != nil || limit < 0 { + return 0, "", "", fmt.Errorf("invalid quota limit") + } + if limit == 0 { + continue + } // Explicit unlimited, not missing data. + free := max(int64(0), limit-used) + if free > math.MaxInt64/1024 { + return 0, "", "", fmt.Errorf("quota capacity overflow") + } + available = min(available, free*1024) + } + matched = true + } + } + if !matched || filesystem == "" || uid == "" || section != "__AKO_END__" { + return 0, "", "", fmt.Errorf("effective user quota unavailable for workspace filesystem") + } + return available, filesystem, uid, nil +} diff --git a/internal/provider/storage/sshfilesystem/budget_test.go b/internal/provider/storage/sshfilesystem/budget_test.go new file mode 100644 index 00000000..59834d4e --- /dev/null +++ b/internal/provider/storage/sshfilesystem/budget_test.go @@ -0,0 +1,61 @@ +package sshfilesystem + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const quotaFixture = `__AKO_DF__ +Filesystem 1024-blocks Used Available Capacity Mounted on +server:/home 100000000 5000000 95000000 5% /home +__AKO_UID__ +12345 +__AKO_QUOTA__ +quota: Cannot resolve unrelated mount: Permission denied +Disk quotas for user researcher: +Filesystem blocks quota limit grace files quota limit grace +server:/home 20971340 20971520 20971520 0 2894 4294967295 4294967295 0 +__AKO_END__ +` + +func TestEffectiveWorkspaceBudgetUsesPersonalQuotaNotFilesystemFree(t *testing.T) { + bytes, _, _, err := parseWorkspaceBudget(quotaFixture) + require.NoError(t, err) + require.Equal(t, int64(180*1024), bytes) + for _, replacement := range []string{"20971520* 20971520 20971520", "20971521* 20971520 20971520"} { + value := strings.Replace(quotaFixture, "20971340 20971520 20971520", replacement, 1) + bytes, _, _, err = parseWorkspaceBudget(value) + require.NoError(t, err) + require.Zero(t, bytes) + } +} + +func TestBudgetRejectsUnprovenQuotaMappingOrTruncatedOutput(t *testing.T) { + for _, value := range []string{ + strings.Replace(quotaFixture, "server:/home 20971340", "server:/different 20971340", 1), + strings.Replace(quotaFixture, "__AKO_END__", "", 1), + strings.Replace(quotaFixture, "20971340", "invalid", 1), + } { + _, _, _, err := parseWorkspaceBudget(value) + require.Error(t, err) + } +} + +func TestWorkspaceBudgetHasBoundedReadOnlySSHAndStableAccountIdentity(t *testing.T) { + executor := &commandFake{output: []byte(quotaFixture)} + driver := New(connectionStoreFake{connection: testConnection()}, executor) + value, err := driver.WorkspaceBudget(context.Background(), testStorage(), "") + require.NoError(t, err) + require.Equal(t, int64(180*1024), value.AvailableBytes) + require.NotContains(t, value.StorageID, "server") + storage := testStorage() + storage.ID = "another-logical-node" + second, err := driver.WorkspaceBudget(context.Background(), storage, "") + require.NoError(t, err) + require.Equal(t, value.StorageID, second.StorageID) + _, err = driver.WorkspaceBudget(context.Background(), storage, "../../etc") + require.Error(t, err) +} diff --git a/internal/provider/storage/sshfilesystem/driver.go b/internal/provider/storage/sshfilesystem/driver.go index 17bff5d8..3b15cad9 100644 --- a/internal/provider/storage/sshfilesystem/driver.go +++ b/internal/provider/storage/sshfilesystem/driver.go @@ -26,6 +26,12 @@ type ConnectionStore interface { type Driver struct { connections ConnectionStore executor provider.CommandExecutor + cloud ports.CloudConfigurationStore +} + +func (d *Driver) WithCloudStore(store ports.CloudConfigurationStore) *Driver { + d.cloud = store + return d } func New(connections ConnectionStore, executor provider.CommandExecutor) *Driver { @@ -142,6 +148,23 @@ func (d *Driver) Write(ctx context.Context, storage domain.StorageResource, valu return err } +// ArchiveDirectory packs within the verified source filesystem, avoiding one +// SSH round trip per file. It publishes only a validated, nonempty archive. +func (d *Driver) ArchiveDirectory(ctx context.Context, storage domain.StorageResource, value string) (string, error) { + if storage.ReadOnly { + return "", fmt.Errorf("storage is read-only") + } + root, relative, err := browseRoot(storage, value) + if err != nil || relative == "" { + return "", fmt.Errorf("archive requires a bounded subdirectory") + } + _, err = d.run(ctx, storage, archiveScript, root, relative) + if err != nil { + return "", err + } + return relative + ".tar.gz", nil +} + func (d *Driver) run(ctx context.Context, storage domain.StorageResource, script string, root, relative string) ([]byte, error) { return d.runInput(ctx, storage, script, nil, root, relative) } @@ -158,10 +181,34 @@ func (d *Driver) runInput(ctx context.Context, storage domain.StorageResource, s // remote command through a shell, so this avoids a second round of parsing // for function declarations, quotes, and newlines in the fixed program. program := base64.StdEncoding.EncodeToString([]byte(prefix + "\n" + script)) - return exec.Run(ctx, "sh", []string{"-c", "printf %s " + program + " | base64 -d | sh"}, input) + // Decode the program into the command argument, not the child's stdin: + // writeScript must receive the caller's binary payload on stdin. + return exec.Run(ctx, "sh", []string{"-c", "sh -c \"$(printf %s " + program + " | base64 -d)\""}, input) } func (d *Driver) connectionExecutor(ctx context.Context, storage domain.StorageResource) (provider.CommandExecutor, error) { + if instanceID := metadataString(storage.Metadata, "cloudInstanceId"); instanceID != "" { + if d.cloud == nil { + return nil, fmt.Errorf("cloud storage identity resolver is unavailable") + } + instance, err := d.cloud.FindProvisionedInstance(ctx, instanceID) + if err != nil || instance == nil || instance.Status != "ready" { + return nil, fmt.Errorf("cloud storage instance is not ready") + } + alias, knownHosts, err := instance.VerifiedSSHTrust() + if err != nil { + return nil, fmt.Errorf("cloud storage trust: %w", err) + } + connection := domain.EnvironmentConnection{ + ID: instance.ID, Type: domain.ConnectionSSH, Endpoint: instance.PublicAddress, + Username: instance.SSHUsername, CredentialRef: instance.SSHCredentialRef, + Configuration: map[string]any{"port": 22, "hostKeyAlias": alias, "knownHostsFile": knownHosts, "strictHostKeyChecking": true}, + } + if connection.Endpoint == "" { + return nil, fmt.Errorf("cloud storage has no relay-accessible address") + } + return provider.NewSSHCommandExecutor(d.executor, connection), nil + } if d.connections == nil { return nil, fmt.Errorf("SSH storage connection store is required") } @@ -297,4 +344,14 @@ target="$parent/$(basename -- "$candidate")" tmp="$target.akoflow-copy-part" cat > "$tmp" && mv -f -- "$tmp" "$target"` +const archiveScript = commonScript + "\n" + `safe_existing || exit 3 +test -d "$candidate" || exit 5 +test -z "$(find "$candidate" -type l -print -quit)" || exit 6 +target="$candidate.tar.gz" +tmp=$(mktemp "$target.part.XXXXXXXX") || exit 7 +trap 'rm -f -- "$tmp"' EXIT HUP INT TERM +tar -C "$root" -czf "$tmp" -- "$rel" || exit 8 +test -s "$tmp" && gzip -t "$tmp" && tar -tzf "$tmp" >/dev/null || exit 9 +mv -f -- "$tmp" "$target"` + var _ ports.StorageBrowser = (*Driver)(nil) diff --git a/internal/provider/telemetry/docker.go b/internal/provider/telemetry/docker.go index df0eec97..5203f908 100644 --- a/internal/provider/telemetry/docker.go +++ b/internal/provider/telemetry/docker.go @@ -13,6 +13,19 @@ import ( const dockerSampleInterval = 10 * time.Second +func dockerAttempt(handle domain.ActivityHandle) int { + if attempt := int(number(handle.Metadata["executionAttempt"])); attempt > 0 { + return attempt + } + // Persisted legacy handles already carry the authoritative attempt suffix. + if _, suffix, ok := strings.Cut(handle.ID, ":attempt-"); ok { + if attempt, err := strconv.Atoi(suffix); err == nil && attempt > 0 { + return attempt + } + } + return 1 +} + // DockerRunner is satisfied by both local and SSH-backed command executors. type DockerRunner func(context.Context, string, []string, []byte) ([]byte, error) @@ -46,7 +59,7 @@ func ObserveDocker(ctx context.Context, handle *domain.ActivityHandle, container cpuSeconds += observed.cpuCores * math.Max(0, now-previousAt) } handle.Metrics = []domain.ActivityMetricSample{{RunID: handle.RunID, ActivityID: handle.ActivityID, - Attempt: 1, ObservedAt: now, CPUSeconds: cpuSeconds, MemoryBytes: observed.memoryBytes, + Attempt: dockerAttempt(*handle), ObservedAt: now, CPUSeconds: cpuSeconds, MemoryBytes: observed.memoryBytes, ReadBytes: observed.readBytes, WriteBytes: observed.writeBytes, Source: "docker-stats-estimated-cpu", Scope: "container"}} handle.Metadata["dockerMetricObservedAt"] = now diff --git a/internal/provider/telemetry/docker_test.go b/internal/provider/telemetry/docker_test.go index afcc4f46..4f4a5af0 100644 --- a/internal/provider/telemetry/docker_test.go +++ b/internal/provider/telemetry/docker_test.go @@ -19,6 +19,21 @@ func TestParseDockerStats(t *testing.T) { } } +func TestDockerMetricsKeepRecoveryAttemptsSeparate(t *testing.T) { + for _, attempt := range []int{1, 3} { + handle := domain.ActivityHandle{RunID: "run", ActivityID: "same-activity", Metadata: map[string]any{"executionAttempt": attempt}} + ObserveDocker(context.Background(), &handle, "container", func(context.Context, string, []string, []byte) ([]byte, error) { + return []byte(`{"CPUPerc":"50%","MemUsage":"1MiB / 1GiB","BlockIO":"0B / 0B"}`), nil + }) + if len(handle.Metrics) != 1 || handle.Metrics[0].Attempt != attempt { + t.Fatalf("attempt %d mixed: %+v", attempt, handle.Metrics) + } + } + if dockerAttempt(domain.ActivityHandle{ID: "run:activity:attempt-3"}) != 3 || dockerAttempt(domain.ActivityHandle{}) != 1 { + t.Fatal("legacy handle attribution failed") + } +} + func TestObserveDockerIsBoundedAndOptional(t *testing.T) { calls := 0 run := func(_ context.Context, _ string, _ []string, _ []byte) ([]byte, error) { diff --git a/scripts/gcp_pricing_probe.py b/scripts/gcp_pricing_probe.py new file mode 100644 index 00000000..9f80f38c --- /dev/null +++ b/scripts/gcp_pricing_probe.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Read public regional disk SKU provenance without logging credentials.""" +import argparse +import json +import urllib.request +import urllib.parse +from gcp_worker_image import token + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--credential", required=True) + parser.add_argument("--region", required=True) + args = parser.parse_args() + with open(args.credential) as stream: + access = token(json.load(stream), "https://www.googleapis.com/auth/cloud-platform") + root = "https://cloudbilling.googleapis.com/v1/services/6F81-5844-456A/skus" + page = "" + while True: + query = urllib.parse.urlencode({"currencyCode": "USD", "pageSize": 5000, "pageToken": page}) + request = urllib.request.Request(root+"?"+query, headers={"Authorization": "Bearer "+access}) + with urllib.request.urlopen(request, timeout=60) as response: + result = json.load(response) + for sku in result.get("skus", []): + description = sku.get("description", "").lower() + if args.region not in sku.get("serviceRegions", []): + continue + if "capacity" not in description or "pd" not in description: + continue + expression = sku.get("pricingInfo", [{}])[-1].get("pricingExpression", {}) + print(json.dumps({"skuId": sku.get("skuId"), "description": sku.get("description"), + "category": sku.get("category"), "usageUnit": expression.get("usageUnit"), + "rates": expression.get("tieredRates")})) + page = result.get("nextPageToken", "") + if not page: + break + +if __name__ == "__main__": + main() diff --git a/scripts/gcp_worker_image.py b/scripts/gcp_worker_image.py new file mode 100644 index 00000000..f9bf5009 --- /dev/null +++ b/scripts/gcp_worker_image.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Capture a stopped, disposable worker's boot disk; never print credentials. + +Run in the server container (cryptography is supplied by its Ansible runtime). +VM configuration, stopping and destruction remain Akoflow API operations. +""" +import argparse +import base64 +import json +import re +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid + + +def image_payload(project, zone, instance, vm, image_name): + if not re.fullmatch(r"cloud-instance-[a-z0-9-]+", instance): + raise ValueError("Expected an explicit managed builder instance") + if not re.fullmatch(r"akoflow-montage-docker-[a-z0-9-]+", image_name): + raise ValueError("Expected a versioned Montage worker image name") + if vm.get("name") != instance or vm.get("status") != "TERMINATED": + raise ValueError("Builder identity must match and VM must be stopped") + disks = [d for d in vm.get("disks", []) if d.get("boot")] + if len(disks) != 1: + raise ValueError("Builder must have exactly one boot disk") + source = disks[0].get("source", "") + expected = f"https://www.googleapis.com/compute/v1/projects/{project}/zones/{zone}/disks/" + if not source.startswith(expected) or not source.removeprefix(expected).startswith(instance): + raise ValueError("Boot disk must belong to the explicit builder and zone") + return {"name": image_name, "sourceDisk": source, + "description": "AKO-136 Docker-ready Montage worker; sanitized disposable builder", + "labels": {"managed-by": "akoflow", "purpose": "montage-worker", "issue": "ako-136"}, + "storageLocations": [zone.rsplit("-", 1)[0]]} + + +def token(credential, scope="https://www.googleapis.com/auth/compute"): + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import padding + def b64(data): + return base64.urlsafe_b64encode(data).rstrip(b"=") + now = int(time.time()) + claims = {"iss": credential["client_email"], "aud": credential["token_uri"], + "scope": scope, "iat": now, "exp": now + 3600} + unsigned = b64(b'{"alg":"RS256","typ":"JWT"}') + b"." + b64(json.dumps(claims).encode()) + key = serialization.load_pem_private_key(credential["private_key"].encode(), None) + assertion = unsigned + b"." + b64(key.sign(unsigned, padding.PKCS1v15(), hashes.SHA256())) + body = urllib.parse.urlencode({"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + "assertion": assertion.decode()}).encode() + request = urllib.request.Request(credential["token_uri"], data=body) + with urllib.request.urlopen(request, timeout=45) as response: + return json.load(response)["access_token"] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=["create", "status"]) + parser.add_argument("--credential", required=True) + parser.add_argument("--project", required=True) + parser.add_argument("--zone", required=True) + parser.add_argument("--instance", required=True) + parser.add_argument("--image", required=True) + args = parser.parse_args() + with open(args.credential) as stream: + credential = json.load(stream) + access = token(credential) + root = "https://compute.googleapis.com/compute/v1/projects/" + urllib.parse.quote(args.project, safe="") + def call(path, body=None): + headers = {"Authorization": "Bearer " + access, "Content-Type": "application/json"} + request = urllib.request.Request(root + path, headers=headers, + data=None if body is None else json.dumps(body).encode()) + with urllib.request.urlopen(request, timeout=45) as response: + return json.load(response) + path = "/global/images/" + urllib.parse.quote(args.image, safe="") + if args.action == "status": + image = call(path) + print(json.dumps({k: image.get(k) for k in ("name", "status", "selfLink", "creationTimestamp", "diskSizeGb")})) + return + vm = call("/zones/" + args.zone + "/instances/" + args.instance) + body = image_payload(args.project, args.zone, args.instance, vm, args.image) + try: + existing = call(path) + except urllib.error.HTTPError as error: + if error.code != 404: + raise + else: + if existing.get("sourceDisk") != body["sourceDisk"] or existing.get("labels", {}).get("issue") != "ako-136": + raise ValueError("Existing image has a different source/ownership; refusing overwrite") + print(json.dumps({"name": existing["name"], "status": existing["status"], "reused": True})) + return + request_id = uuid.uuid5(uuid.NAMESPACE_URL, root + path) + operation = call("/global/images?requestId=" + str(request_id), body) + print(json.dumps({"operation": operation.get("name"), "status": operation.get("status"), "image": args.image})) + + +if __name__ == "__main__": + try: + main() + except urllib.error.HTTPError as error: + raise SystemExit("GCP request failed with HTTP " + str(error.code)) diff --git a/scripts/gcp_worker_image_audit.py b/scripts/gcp_worker_image_audit.py new file mode 100644 index 00000000..84d3d55a --- /dev/null +++ b/scripts/gcp_worker_image_audit.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Read-only proof of a managed worker's actual boot image, with no addresses.""" +import argparse +import json +import re +import urllib.parse +import urllib.request +from gcp_worker_image import token + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--credential", required=True) + parser.add_argument("--project", required=True) + parser.add_argument("--zone", required=True) + parser.add_argument("--image", required=True) + parser.add_argument("instances", nargs="+") + args = parser.parse_args() + with open(args.credential) as stream: + access = token(json.load(stream)) + root = "https://compute.googleapis.com/compute/v1/projects/"+urllib.parse.quote(args.project, safe="") + expected = "https://www.googleapis.com/compute/v1/projects/"+args.project+"/global/images/"+args.image + def get(endpoint): + request = urllib.request.Request(endpoint, headers={"Authorization": "Bearer "+access}) + with urllib.request.urlopen(request, timeout=45) as response: + return json.load(response) + for instance in args.instances: + if not re.fullmatch(r"cloud-instance-[a-z0-9-]+", instance): + raise ValueError("An explicit managed instance is required") + vm = get(root+"/zones/"+args.zone+"/instances/"+instance) + disks = [disk for disk in vm.get("disks", []) if disk.get("boot")] + if len(disks) != 1 or not disks[0]["source"].startswith("https://www.googleapis.com/compute/v1/projects/"+args.project+"/zones/"+args.zone+"/disks/"): + raise ValueError("Unexpected boot disk identity") + disk = get(disks[0]["source"]) + proof = {"instanceId": instance, "status": vm.get("status"), + "createdAt": vm.get("creationTimestamp"), "imageMatches": disk.get("sourceImage") == expected, + "sourceImageId": disk.get("sourceImageId"), "diskSizeGiB": disk.get("sizeGb")} + print(json.dumps(proof)) + if not proof["imageMatches"]: + raise ValueError("Boot disk does not match the requested immutable image") + +if __name__ == "__main__": + main() diff --git a/scripts/gcp_worker_image_test.py b/scripts/gcp_worker_image_test.py new file mode 100644 index 00000000..b2a80d1f --- /dev/null +++ b/scripts/gcp_worker_image_test.py @@ -0,0 +1,29 @@ +import unittest +from gcp_worker_image import image_payload + + +class WorkerImageTest(unittest.TestCase): + def vm(self): + return {"name": "cloud-instance-builder", "status": "TERMINATED", + "disks": [{"boot": True, "source": "https://www.googleapis.com/compute/v1/projects/p/zones/us-central1-a/disks/cloud-instance-builder"}]} + + def capture(self, vm): + return image_payload("p", "us-central1-a", "cloud-instance-builder", vm, "akoflow-montage-docker-v1") + + def test_stopped_explicit_boot_disk(self): + self.assertEqual(["us-central1"], self.capture(self.vm())["storageLocations"]) + + def test_reject_running_wrong_identity_and_wrong_disk(self): + for field, value in [("status", "RUNNING"), ("name", "other")]: + vm = self.vm() + vm[field] = value + with self.assertRaises(ValueError): + self.capture(vm) + vm = self.vm() + vm["disks"][0]["source"] = vm["disks"][0]["source"].replace("/projects/p/", "/projects/other/") + with self.assertRaises(ValueError): + self.capture(vm) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/refresh_montage_prices.sh b/scripts/refresh_montage_prices.sh new file mode 100644 index 00000000..d93ae475 --- /dev/null +++ b/scripts/refresh_montage_prices.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# AKO-140: refresh tariffs without resetting the experimental CPU model. +set -eu +api=${AKOFLOW_API_URL:-http://127.0.0.1:8080/akoflow-api} +: "${AKOFLOW_API_TOKEN:?Load the API token without printing it}" +get() { curl -fsS -H "Authorization: Bearer $AKOFLOW_API_TOKEN" "$api/$1/"; } +post() { curl --fail-with-body -sS -H "Authorization: Bearer $AKOFLOW_API_TOKEN" -H 'Content-Type: application/json' --data-binary @- "$api/$1/"; } +catalog=$(get environments/gcp-environment/cloud-catalog | jq -c '{region,discoveredAt,disks,machines:[.machines[]|select(.providerTypeId=="e2-micro" or .providerTypeId=="e2-medium")]}') +targets=$(get environments/gcp-environment/cloud-capacity-targets) +for id in montage-gcp-e2-medium-1 montage-gcp-e2-medium-2 montage-gcp-fog-e2-micro-01 montage-gcp-fog-e2-micro-02 montage-gcp-fog-e2-micro-03 montage-gcp-fog-e2-micro-04 montage-gcp-fog-e2-micro-05 montage-gcp-fog-e2-micro-06 montage-gcp-fog-e2-micro-07 montage-gcp-fog-e2-micro-08 montage-gcp-fog-e2-micro-09 montage-gcp-fog-e2-micro-10; do + original=$(get resources/$id) + target=$(printf '%s' "$targets" | jq -ce --arg id "$id" '.[]|select(.id==$id)') + printf '%s' "$target" | jq -e --argjson catalog "$catalog" ' + . as $target | $catalog.region==$target.region and + ([$catalog.machines[] | select(.providerTypeId==$target.providerMachineType and .pricePerHour>0)]|length)==1 and + ([$catalog.disks[] | select(.providerTypeId==$target.configuration.diskType and .pricePerGiBMonth>0)]|length)==1' >/dev/null + updated=$(printf '%s' "$target" | jq --argjson catalog "$catalog" '.configuration.priceObservedAt=$catalog.discoveredAt | del(.createdAt)' | post environments/gcp-environment/cloud-capacity-targets) + printf '%s' "$original" | jq --argjson target "$updated" ' + .pricePerSecond=($target.configuration.pricePerHour/3600) | + .metadata.priceCurrency=$target.configuration.priceCurrency | + .metadata.priceSource=$target.configuration.priceSource | + .metadata.priceObservedAt=$target.configuration.priceObservedAt | + .metadata.diskPricePerGiBMonth=$target.configuration.diskPricePerGiBMonth | + .metadata.pricingStatus="estimated"' | post resources >/dev/null + printf '%s' "$updated" | jq '{id,pricePerHour:.configuration.pricePerHour,diskPricePerGiBMonth:.configuration.diskPricePerGiBMonth,currency:.configuration.priceCurrency}' +done diff --git a/scripts/submit_montage_docker_image.sh b/scripts/submit_montage_docker_image.sh new file mode 100644 index 00000000..873a08a3 --- /dev/null +++ b/scripts/submit_montage_docker_image.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Submit a fresh run from live catalog, selected priced plan and immutable image targets. +set -eu +mode=${1:?simulation or real} +case "$mode" in simulation|real) ;; *) exit 2 ;; esac +: "${AKOFLOW_API_TOKEN:?Load the API token without printing it}" +: "${RUN_ID:?Use a new explicit experiment identity}" +api=${AKOFLOW_API_URL:-http://127.0.0.1:8080/akoflow-api} +base=storage/montage-image-20261010 +get() { curl -fsS -H "Authorization: Bearer $AKOFLOW_API_TOKEN" "$api/$1/"; } +test "$(get execution-runs | jq '[.[]|select(.status=="running" or .status=="created")]|length')" = 0 +if [ "$mode" = real ]; then + test "$(get cloud-operations | jq '[.[]|select(.status=="running" or .status=="queued")]|length')" = 0 + test "$(get environments/gcp-environment/cloud-instances | jq '[.[]|select(.status!="destroyed")]|length')" = 0 +fi +gcp=$(get environments/gcp-environment) +hpc=$(get environments/plafrim-execution) +jq --arg mode "$mode" --arg id "$RUN_ID" --argjson gcp "$gcp" --argjson hpc "$hpc" --slurpfile baseline storage/montage-fog-20261006/real-request.json --slurpfile plan "$base/selected-priced-v2.json" --slurpfile scope "$base/scope.json" --slurpfile network "$base/network.json" ' + .run={id:$id,mode:$mode,seed:1,schedulePlanId:$plan[0].id} | + .plan=$plan[0] | .executionScope=$scope[0] | .networkTopology=$network[0] | + (.plan.assignments|map(.resourceId)|unique) as $ids | + .resources=([$gcp.resources[],$hpc.resources[]]|map(select(.id as $resource|$ids|index($resource)))) | + ($baseline[0].runtimes[]|select(.id=="montage-simgrid")) as $simulation | + .runtimes=([$gcp.runtimes[],$hpc.runtimes[],$simulation]|unique_by(.id)) | + .runtimeBindings=(if $mode=="simulation" then [$ids[]|{resourceId:.,runtimeId:"montage-simgrid",enabled:true}] + else [$gcp.runtimeBindings[]?,$gcp.resourceRuntimeBindings[]?,$hpc.runtimeBindings[]?,$hpc.resourceRuntimeBindings[]?]|map(select(.resourceId as $resource|$ids|index($resource)))|unique_by(.resourceId+":"+.runtimeId) end) +' storage/montage-fog-20261006/single-lane-real-request.json | + tee "$base/$RUN_ID-request.json" | + curl --fail-with-body -sS -H "Authorization: Bearer $AKOFLOW_API_TOKEN" -H 'Content-Type: application/json' --data-binary @- "$api/execution-runs/" > "$base/$RUN_ID-submitted.json" +jq '{id,status,error}' "$base/$RUN_ID-submitted.json" diff --git a/scripts/submit_montage_measured.sh b/scripts/submit_montage_measured.sh new file mode 100644 index 00000000..fc0180a1 --- /dev/null +++ b/scripts/submit_montage_measured.sh @@ -0,0 +1,39 @@ +#!/bin/sh +set -eu +mode=${1:?simulation or real} +case "$mode" in simulation|real) ;; *) exit 2 ;; esac +: "${AKOFLOW_API_TOKEN:?Load token without printing it}" +: "${RUN_ID:?Use a new holdout run identity}" +base=${MONTAGE_EXPERIMENT_DIR:-storage/montage-measured-20261011} +compute_model=${MONTAGE_COMPUTE_MODEL:-experiments/montage-micro-hpc-e2/compute-model-v2.json} +api=${AKOFLOW_API_URL:-http://127.0.0.1:8080/akoflow-api} +get() { curl -fsS -H "Authorization: Bearer $AKOFLOW_API_TOKEN" "$api/$1/"; } +get execution-runs > "$base/runs-before-$RUN_ID.json" +test "$(jq '[.[]|select(.status=="running" or .status=="created")]|length' "$base/runs-before-$RUN_ID.json")" = 0 +test "$(jq --arg id "$RUN_ID" '[.[]|select(.id==$id)]|length' "$base/runs-before-$RUN_ID.json")" = 0 +if [ "$mode" = real ]; then + test "$(get cloud-operations | jq '[.[]|select(.status=="running" or .status=="queued")]|length')" = 0 + test "$(get environments/gcp-environment/cloud-instances | jq '[.[]|select(.status!="destroyed")]|length')" = 0 +fi +get environments/gcp-environment > "$base/gcp-for-$RUN_ID.json" +get environments/plafrim-execution > "$base/hpc-for-$RUN_ID.json" +jq -n --arg mode "$mode" --arg id "$RUN_ID" \ + --slurpfile gcp "$base/gcp-for-$RUN_ID.json" --slurpfile hpc "$base/hpc-for-$RUN_ID.json" \ + --slurpfile plan "$base/selected-plan.json" --slurpfile workflow "$base/workflow-created.json" \ + --slurpfile scope "$base/scope.json" --slurpfile network "$base/network.json" \ + --slurpfile relay "$base/relay-resource.json" \ + --slurpfile compute "$compute_model" \ + --slurpfile baseline storage/montage-fog-20261006/real-request.json ' + ($plan[0].assignments|map(.resourceId)|unique) as $ids | + ($baseline[0].runtimes[]|select(.id=="montage-simgrid")) as $simulation | + {run:{id:$id,mode:$mode,seed:1,schedulePlanId:$plan[0].id}, + workflow:$workflow[0].version,plan:$plan[0],executionScope:$scope[0],networkTopology:$network[0], + resources:(([$gcp[0].resources[],$hpc[0].resources[]]|map(select(.id as $r|$ids|index($r))))+[$relay[0]]), + activityProfiles:($compute[0].activityProfiles|map(select(.resourceId as $r|$ids|index($r)))), + runtimes:([$gcp[0].runtimes[],$hpc[0].runtimes[],$simulation]|unique_by(.id)), + runtimeBindings:(if $mode=="simulation" then [$ids[]|{resourceId:.,runtimeId:"montage-simgrid",enabled:true}] + else [$gcp[0].runtimeBindings[]?,$gcp[0].resourceRuntimeBindings[]?,$hpc[0].runtimeBindings[]?,$hpc[0].resourceRuntimeBindings[]?]|map(select(.resourceId as $r|$ids|index($r)))|unique_by(.resourceId+":"+.runtimeId) end)} +' > "$base/$RUN_ID-request.json" +curl --fail-with-body -sS -H "Authorization: Bearer $AKOFLOW_API_TOKEN" -H 'Content-Type: application/json' \ + --data-binary "@$base/$RUN_ID-request.json" "$api/execution-runs/" > "$base/$RUN_ID-submitted.json" +jq '{id,status,error}' "$base/$RUN_ID-submitted.json" diff --git a/server b/server deleted file mode 100755 index 0e7ce7ab..00000000 Binary files a/server and /dev/null differ diff --git a/simgrid-runner/src/main.cpp b/simgrid-runner/src/main.cpp index e71a6e7d..f7a364c4 100644 --- a/simgrid-runner/src/main.cpp +++ b/simgrid-runner/src/main.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,13 @@ struct TaskModel { double price_per_second = 0; double flops = 0; double overhead_seconds = 0; + double preparation_seconds = 0; + double admission_seconds = 0; + double publication_seconds = 0; + std::string preparation_service_group; + int preparation_service_slots = 0; + double preparation_started_at = 0; + double preparation_finished_at = 0; double started_at = 0; double compute_started_at = 0; double compute_finished_at = 0; @@ -93,6 +101,22 @@ static InterferenceModel create_interference(const json& input) return interference; } +struct TransferStage { + std::string service_group; + int service_slots = 0; + double service_queued_at = 0; + double service_wait_seconds = 0; + std::string name; + std::string source; + std::string target; + long long bytes = 0; + double rate = 0; + double before_seconds = 0; + double after_seconds = 0; + double started_at = 0; + double finished_at = 0; +}; + struct TransferModel { std::string producer_id; std::string consumer_id; @@ -102,6 +126,10 @@ struct TransferModel { double price_per_byte = 0; double started_at = 0; double finished_at = 0; + std::string strategy; + long long network_bytes = 0; + long long relay_network_bytes = 0; + std::vector stages; }; static Arguments parse_arguments(int argc, char** argv) @@ -154,6 +182,17 @@ static std::map create_tasks(sg4::Engine& engine, const engine.host_by_name(task.resource_id); task.flops = std::max(0.0, value.at("flops").get()); task.overhead_seconds = std::max(0.0, value.value("overheadSeconds", 0.0)); + task.preparation_seconds = value.value("preparationSeconds", 0.0); + task.admission_seconds = value.value("admissionSeconds", 0.0); + if (!std::isfinite(task.admission_seconds) || task.admission_seconds < 0) + throw std::runtime_error("invalid external admission delay"); + task.publication_seconds = value.value("publicationSeconds",0.0); + if (!std::isfinite(task.publication_seconds) || task.publication_seconds < 0) + throw std::runtime_error("invalid publication service"); + task.preparation_service_group = value.value("preparationServiceGroupId", ""); + task.preparation_service_slots = value.value("preparationServiceSlots", 0); + if (!std::isfinite(task.preparation_seconds) || task.preparation_seconds < 0) + throw std::runtime_error("invalid consumer preparation duration"); if (!tasks.emplace(task.id, task).second) throw std::runtime_error("duplicate task: " + task.id); } @@ -169,7 +208,9 @@ static std::vector create_transfers(const json& input, const std: auto& producer = tasks.at(producer_id); auto& consumer = tasks.at(consumer_id); const long long bytes = value.value("bytes", 0LL); - if (bytes <= 0 || producer.resource_id == consumer.resource_id) + if (bytes < 0) + throw std::runtime_error("negative dependency bytes"); + if ((bytes == 0 && !value.contains("stages")) || producer.resource_id == consumer.resource_id) continue; TransferModel transfer; transfer.producer_id = producer_id; @@ -178,6 +219,57 @@ static std::vector create_transfers(const json& input, const std: transfer.target_resource_id = consumer.resource_id; transfer.bytes = bytes; transfer.price_per_byte = value.value("pricePerByte", 0.0); + transfer.strategy = value.value("strategy", ""); + transfer.network_bytes = value.value("networkBytes",input.value("materializationModelVersion",0)>0 ? 0LL : bytes); + transfer.relay_network_bytes = value.value("relayNetworkBytes", 0LL); + for (const auto& stage_value : value.value("stages", json::array())) { + TransferStage stage; + stage.service_group = stage_value.value("serviceGroupId", ""); + stage.service_slots = stage_value.value("serviceSlots", 0); + stage.name = stage_value.at("name").get(); + stage.source = stage_value.at("sourceResourceId").get(); + stage.target = stage_value.at("targetResourceId").get(); + stage.bytes = stage_value.value("bytes", 0LL); + stage.rate = stage_value.value("effectiveBytesPerSecond", 0.0); + stage.before_seconds = stage_value.value("beforeSeconds", 0.0); + stage.after_seconds = stage_value.value("afterSeconds", 0.0); + if (stage.bytes < 0 || !std::isfinite(stage.rate) || stage.rate < 0 || + (stage.bytes > 0 && stage.rate <= 0) || !std::isfinite(stage.before_seconds) || + stage.before_seconds < 0 || !std::isfinite(stage.after_seconds) || stage.after_seconds < 0) + throw std::runtime_error("invalid materialization stage"); + transfer.stages.push_back(stage); + } + if (input.value("materializationModelVersion", 0) > 0 && transfer.stages.empty()) + throw std::runtime_error("materialization dependency lacks stages"); + if (!transfer.stages.empty()) { + if (transfer.stages.front().source != producer.resource_id || transfer.stages.back().target != consumer.resource_id) + throw std::runtime_error("materialization stages do not match dependency endpoints"); + long long network_bytes = 0; + for (size_t index=0; index0 && transfer.stages[index-1].target != transfer.stages[index].source) + throw std::runtime_error("materialization stages are not contiguous"); + if (transfer.stages[index].bytes > std::numeric_limits::max()-network_bytes) + throw std::runtime_error("materialization network bytes overflow"); + network_bytes += transfer.stages[index].bytes; + } + if (network_bytes != transfer.network_bytes) + throw std::runtime_error("materialization network byte count differs from stages"); + if (transfer.strategy == "gateway") { + const auto relay=input.value("materializationRelayResourceId",std::string{}); + if (relay.empty() || transfer.stages.size()!=2 || transfer.stages[0].target != relay || transfer.stages[1].source != relay || transfer.relay_network_bytes != network_bytes) + throw std::runtime_error("gateway materialization must use both relay hops"); + } else if (transfer.strategy == "direct-runtime") { + const bool receiver_stage = transfer.stages.size()==2 && transfer.stages[1].bytes==0 && + transfer.stages[1].source==consumer.resource_id && transfer.stages[1].target==consumer.resource_id; + if ((transfer.stages.size()!=1 && !receiver_stage) || transfer.relay_network_bytes!=0) + throw std::runtime_error("invalid peer/shared materialization stages"); + } else if (transfer.strategy == "shared-storage") { + if (transfer.stages.size()!=1 || transfer.relay_network_bytes!=0 || network_bytes!=0) + throw std::runtime_error("invalid peer/shared materialization stages"); + } else { + throw std::runtime_error("unsupported materialization strategy"); + } + } transfers.push_back(transfer); } return transfers; @@ -195,17 +287,52 @@ static void run_simulation(sg4::Engine& engine, const json& input, const InterferenceModel& interference) { const auto run_id = input.at("runId").get(); + std::map service_pools; + std::map service_capacities; + auto register_service = [&](const std::string& group, int slots) { + if (group.empty()) { + if (slots != 0) throw std::runtime_error("service capacity without group"); + return; + } + if (slots < 1) throw std::runtime_error("invalid service capacity"); + auto existing = service_capacities.find(group); + if (existing != service_capacities.end() && existing->second != slots) + throw std::runtime_error("conflicting shared service capacities"); + if (existing == service_capacities.end()) { + service_capacities[group] = slots; + service_pools[group] = sg4::Semaphore::create(slots); + } + }; + for (const auto& [id, task] : tasks) + register_service(task.preparation_service_group, task.preparation_service_slots); + for (const auto& transfer : transfers) + for (const auto& stage : transfer.stages) + register_service(stage.service_group, stage.service_slots); std::map>> incoming_dependencies; std::map> outgoing_dependency_signals; + std::map> outgoing_consumers; std::map> incoming_lanes; std::map> outgoing_lanes; std::map> active_by_resource; + std::set> data_edges; + std::map remaining_producers; + std::map inbound_flow_count; + std::map preparation_gates; + const bool materialization_enabled = input.value("materializationModelVersion",0) > 0; + for (const auto& transfer : transfers) { + data_edges.insert({transfer.producer_id,transfer.consumer_id}); + inbound_flow_count[transfer.consumer_id]++; + } for (const auto& dependency : input.at("dependencies")) { const auto producer = dependency.at("producerId").get(); const auto consumer = dependency.at("consumerId").get(); incoming_dependencies[consumer].push_back({producer, mailbox_name(run_id, "data", producer, consumer)}); outgoing_dependency_signals[producer].push_back(mailbox_name(run_id, "ready", producer, consumer)); + outgoing_consumers[producer].push_back(consumer); + remaining_producers[consumer]++; + if (materialization_enabled && preparation_gates.count(consumer)==0) + preparation_gates[consumer] = sg4::Semaphore::create(0); } for (const auto& order : input.at("resourceLaneOrders")) { const auto predecessor = order.at("predecessorId").get(); @@ -218,11 +345,54 @@ static void run_simulation(sg4::Engine& engine, const json& input, for (auto& transfer : transfers) { auto* transfer_model = &transfer; auto* source_host = engine.host_by_name(transfer.source_resource_id); - sg4::Actor::create("transfer-" + transfer.producer_id + "-" + transfer.consumer_id, source_host, - [transfer_model, run_id]() { + engine.add_actor("transfer-" + transfer.producer_id + "-" + transfer.consumer_id, source_host, + [transfer_model, run_id, &engine, &preparation_gates, &service_pools, materialization_enabled]() { sg4::Mailbox::by_name(mailbox_name(run_id, "ready", transfer_model->producer_id, transfer_model->consumer_id))->get(); + if (materialization_enabled) + preparation_gates.at(transfer_model->consumer_id)->acquire(); transfer_model->started_at = sg4::Engine::get_clock(); + if (!transfer_model->stages.empty()) { + for (size_t stage_index=0; stage_indexstages.size(); ++stage_index) { + auto& stage=transfer_model->stages[stage_index]; + stage.service_queued_at = sg4::Engine::get_clock(); + if (!stage.service_group.empty()) service_pools.at(stage.service_group)->acquire(); + stage.service_wait_seconds = sg4::Engine::get_clock() - stage.service_queued_at; + stage.started_at = sg4::Engine::get_clock(); + if (stage.bytes > 0) { + auto* stage_model=&stage; + const auto channel=mailbox_name(run_id,"stage-"+std::to_string(stage_index),transfer_model->producer_id,transfer_model->consumer_id); + auto receiver=engine.add_actor("receive-"+channel,engine.host_by_name(stage.target),[stage_model,channel]() { + sg4::Mailbox::by_name(channel)->get(); + if (stage_model->after_seconds > 0) + sg4::this_actor::sleep_for(stage_model->after_seconds); + }); + auto sender=engine.add_actor("send-"+channel,engine.host_by_name(stage.source),[stage_model,channel]() { + if (stage_model->before_seconds > 0) + sg4::this_actor::sleep_for(stage_model->before_seconds); + auto communication=sg4::Mailbox::by_name(channel)->put_init(&mailbox_token,stage_model->bytes); + communication->set_rate(stage_model->rate); + communication->wait(); + }); + sender->join(); + receiver->join(); + } else { + if (stage.before_seconds+stage.after_seconds > 0) + sg4::this_actor::sleep_for(stage.before_seconds+stage.after_seconds); + } + stage.finished_at = sg4::Engine::get_clock(); + if (!stage.service_group.empty()) service_pools.at(stage.service_group)->release(); + } + transfer_model->finished_at = sg4::Engine::get_clock(); + // Notify on the destination host after all stages. This zero-byte + // control message does not send the payload a third time via relay. + auto notification = engine.add_actor("prepared-"+transfer_model->producer_id+"-"+transfer_model->consumer_id, + engine.host_by_name(transfer_model->target_resource_id),[transfer_model,run_id]() { + sg4::Mailbox::by_name(mailbox_name(run_id,"data",transfer_model->producer_id,transfer_model->consumer_id))->put(&mailbox_token,0); + }); + notification->join(); + return; + } sg4::Mailbox::by_name(mailbox_name(run_id, "data", transfer_model->producer_id, transfer_model->consumer_id)) ->put(&mailbox_token, transfer_model->bytes); @@ -233,27 +403,38 @@ static void run_simulation(sg4::Engine& engine, const json& input, for (auto& [id, task] : tasks) { auto* task_model = &task; auto* host = engine.host_by_name(task.resource_id); - sg4::Actor::create("task-" + id, host, + engine.add_actor("task-" + id, host, [task_model, id, &incoming_dependencies, &outgoing_dependency_signals, &incoming_lanes, &outgoing_lanes, &tasks, &active_by_resource, - &interference, run_id]() { + &interference, &data_edges, &remaining_producers, &inbound_flow_count, + &preparation_gates, &service_pools, &outgoing_consumers, materialization_enabled, run_id]() { const auto& dependencies = incoming_dependencies[id]; std::vector received_payloads(dependencies.size(), nullptr); std::vector incoming_transfers; incoming_transfers.reserve(dependencies.size()); for (size_t index = 0; index < dependencies.size(); ++index) { const auto& [producer_id, mailbox] = dependencies[index]; - if (tasks.at(producer_id).resource_id != task_model->resource_id) + if (data_edges.count({producer_id,id}) > 0) incoming_transfers.push_back(sg4::Mailbox::by_name(mailbox)->get_async(&received_payloads[index])); } for (const auto& [producer_id, mailbox] : dependencies) { - if (tasks.at(producer_id).resource_id == task_model->resource_id) + if (data_edges.count({producer_id,id}) == 0) sg4::Mailbox::by_name(mailbox_name(run_id, "ready", producer_id, id))->get(); } for (const auto& transfer : incoming_transfers) transfer->wait(); for (const auto& mailbox : incoming_lanes[id]) sg4::Mailbox::by_name(mailbox)->get(); + task_model->preparation_started_at = sg4::Engine::get_clock(); + if (task_model->preparation_seconds > 0 && !task_model->preparation_service_group.empty()) + service_pools.at(task_model->preparation_service_group)->acquire(); + if (task_model->preparation_seconds > 0) + sg4::this_actor::sleep_for(task_model->preparation_seconds); + if (task_model->preparation_seconds > 0 && !task_model->preparation_service_group.empty()) + service_pools.at(task_model->preparation_service_group)->release(); + task_model->preparation_finished_at = sg4::Engine::get_clock(); + if (task_model->admission_seconds > 0) + sg4::this_actor::sleep_for(task_model->admission_seconds); task_model->started_at = sg4::Engine::get_clock(); if (task_model->overhead_seconds > 0) sg4::this_actor::sleep_for(task_model->overhead_seconds); @@ -334,7 +515,17 @@ static void run_simulation(sg4::Engine& engine, const json& input, } active_by_resource[task_model->resource_id].erase(id); task_model->compute_finished_at = sg4::Engine::get_clock(); + if (task_model->publication_seconds > 0) + sg4::this_actor::sleep_for(task_model->publication_seconds); task_model->finished_at = sg4::Engine::get_clock(); + if (materialization_enabled) { + for (const auto& consumer : outgoing_consumers[id]) { + if (--remaining_producers[consumer] == 0) { + for (int flow = 0; flow < inbound_flow_count[consumer]; ++flow) + preparation_gates.at(consumer)->release(); + } + } + } for (const auto& mailbox : outgoing_dependency_signals[id]) sg4::Mailbox::by_name(mailbox)->put_init(&mailbox_token, 0)->detach(); for (const auto& mailbox : outgoing_lanes[id]) @@ -436,10 +627,17 @@ static json build_result(const json& input, const std::map() + ":" + id}, {"executionRunId", input.at("runId")}, {"planAssignmentId", task.assignment_id}, {"activityId", id}, {"plannedResourceId", task.resource_id}, {"allocatedResourceId", task.resource_id}, - {"attempt", 1}, {"status", "completed"}, {"readyAt", ready}, {"dataReadyAt", ready}, + {"attempt", 1}, {"status", "completed"}, {"readyAt", ready}, + {"dataReadyAt",input.value("materializationModelVersion",0)>0 ? task.preparation_finished_at : ready}, {"queuedAt", ready}, {"startedAt", start}, {"finishedAt", finish}, {"runtimeSeconds", duration}, {"queueSeconds", queued}, {"transferSeconds", inbound_transfer[id]}, {"interferenceSeconds", task.interference_seconds}, {"overheadSeconds", task.overhead_seconds}, {"cost", costs.at(id)}, + {"metadata", {{"workspacePublicationSeconds",task.publication_seconds},{"workspacePreparationSeconds",task.preparation_seconds}, + {"externalAdmissionSeconds",task.admission_seconds}, + {"workspaceMaterializationModelSource",input.value("materializationModelSource",std::string{})}, + {"workspaceRouteReadyAt",ready}, + {"workspacePreparationStartedAt",task.preparation_started_at}, + {"workspacePreparationFinishedAt",task.preparation_finished_at}}}, }); } for (const auto& transfer : transfers) { @@ -449,12 +647,20 @@ static json build_result(const json& input, const std::map(transfer.bytes) * transfer.price_per_byte; transfer_seconds += duration; total_cost += cost; + json phases=json::array(); + for (const auto& stage : transfer.stages) + phases.push_back({{"name",stage.name},{"startedAt",stage.started_at},{"finishedAt",stage.finished_at},{"durationSeconds",stage.finished_at-stage.started_at}, + {"serviceGroupId",stage.service_group},{"serviceSlots",stage.service_slots}, + {"serviceQueuedAt",stage.service_queued_at},{"serviceWaitSeconds",stage.service_wait_seconds}}); result["transfers"].push_back({ {"id", input.at("runId").get() + ":" + transfer.producer_id + ":" + transfer.consumer_id}, {"executionRunId", input.at("runId")}, {"producerActivityId", transfer.producer_id}, {"consumerActivityId", transfer.consumer_id}, {"sourceResourceId", transfer.source_resource_id}, {"targetResourceId", transfer.target_resource_id}, {"bytes", transfer.bytes}, {"startedAt", start}, {"finishedAt", finish}, {"durationSeconds", duration}, {"cost", cost}, + {"strategy",transfer.strategy},{"logicalBytes",transfer.bytes},{"networkBytes",transfer.network_bytes}, + {"route",{{"strategy",transfer.strategy},{"phases",phases},{"serverNetworkBytes",transfer.relay_network_bytes}, + {"reason",transfer.stages.empty() ? "legacy edge-volume model" : "frozen materialization phase model"}}}, }); } const double deadline = input.value("deadlineSeconds", 0.0); @@ -474,8 +680,14 @@ static json build_result(const json& input, const std::map + + + + + + + + + + + + + + + + diff --git a/simgrid-runner/tests/materialization-simulation.json b/simgrid-runner/tests/materialization-simulation.json new file mode 100644 index 00000000..d1f39eb9 --- /dev/null +++ b/simgrid-runner/tests/materialization-simulation.json @@ -0,0 +1,21 @@ +{ + "materializationModelVersion": 1, + "materializationRelayResourceId": "relay", + "runId": "materialization-test", + "planId": "materialization-plan", + "tasks": [ + {"id":"a","assignmentId":"a","resourceId":"cloud-a","flops":1000000000}, + {"id":"c","assignmentId":"c","resourceId":"hpc","flops":5000000000}, + {"id":"b","assignmentId":"b","resourceId":"cloud-b","flops":1000000000,"preparationSeconds":2} + ], + "dependencies": [ + {"producerId":"a","consumerId":"b","bytes":200,"networkBytes":100,"strategy":"direct-runtime", + "stages":[{"name":"peer","sourceResourceId":"cloud-a","targetResourceId":"cloud-b","bytes":100,"effectiveBytesPerSecond":100}]}, + {"producerId":"c","consumerId":"b","bytes":200,"networkBytes":200,"relayNetworkBytes":200,"strategy":"gateway", + "stages":[ + {"name":"relay-pack-and-receive","sourceResourceId":"hpc","targetResourceId":"relay","bytes":100,"effectiveBytesPerSecond":100,"afterSeconds":1}, + {"name":"relay-upload-and-extract","sourceResourceId":"relay","targetResourceId":"cloud-b","bytes":100,"effectiveBytesPerSecond":100,"afterSeconds":2} + ]} + ], + "resourceLaneOrders": [] +} diff --git a/simgrid-runner/tests/publication-simulation.json b/simgrid-runner/tests/publication-simulation.json new file mode 100644 index 00000000..9e653002 --- /dev/null +++ b/simgrid-runner/tests/publication-simulation.json @@ -0,0 +1,19 @@ +{ + "materializationModelVersion":1, + "materializationRelayResourceId":"relay", + "runId":"publication-test", + "planId":"publication-plan", + "tasks":[ + {"id":"a","assignmentId":"a","resourceId":"cloud-a","flops":1000000000,"publicationSeconds":1}, + {"id":"c","assignmentId":"c","resourceId":"hpc","flops":5000000000}, + {"id":"b","assignmentId":"b","resourceId":"cloud-b","flops":1000000000,"preparationSeconds":2,"publicationSeconds":2} + ], + "dependencies":[ + {"producerId":"a","consumerId":"b","bytes":200,"networkBytes":100,"strategy":"direct-runtime","stages":[{"name":"peer","sourceResourceId":"cloud-a","targetResourceId":"cloud-b","bytes":100,"effectiveBytesPerSecond":100}]}, + {"producerId":"c","consumerId":"b","bytes":200,"networkBytes":200,"relayNetworkBytes":200,"strategy":"gateway","stages":[ + {"name":"relay-pack-and-receive","sourceResourceId":"hpc","targetResourceId":"relay","bytes":100,"effectiveBytesPerSecond":100,"afterSeconds":1}, + {"name":"relay-upload-and-extract","sourceResourceId":"relay","targetResourceId":"cloud-b","bytes":100,"effectiveBytesPerSecond":100,"afterSeconds":2} + ]} + ], + "resourceLaneOrders":[] +} diff --git a/simgrid-runner/tests/test_materialization.py b/simgrid-runner/tests/test_materialization.py new file mode 100644 index 00000000..5f83b2b9 --- /dev/null +++ b/simgrid-runner/tests/test_materialization.py @@ -0,0 +1,191 @@ +import copy +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +class NativeMaterializationTest(unittest.TestCase): + image = os.environ.get("SIMGRID_TEST_IMAGE", "akoflow-simgrid-ako141-build") + root = Path(__file__).resolve().parent + + def run_model(self, model, platform=None, success=True): + with tempfile.TemporaryDirectory(prefix="akoflow-native-test-") as directory: + case = Path(directory) + (case / "input.json").write_text(json.dumps(model)) + (case / "platform.xml").write_text(platform or (self.root / "materialization-platform.xml").read_text()) + command = ["docker", "run", "--rm", "-v", f"{directory}:/case", self.image, + "/build/output/akoflow-simgrid-runner", "--platform", "/case/platform.xml", + "--input", "/case/input.json", "--output", "/case/output.json"] + result = subprocess.run(command, capture_output=True, text=True, timeout=30) + if not success: + self.assertNotEqual(result.returncode, 0) + return result.stderr + self.assertEqual(result.returncode, 0, result.stderr) + return json.loads((case / "output.json").read_text()) + + def fixture(self): + return json.loads((self.root / "materialization-simulation.json").read_text()) + + def test_mixed_origins_serial_relay_and_single_consumer_merge(self): + result = self.run_model(self.fixture()) + tasks = {task["activityId"]: task for task in result["tasks"]} + transfers = {flow["producerActivityId"]: flow for flow in result["transfers"]} + last_producer = max(tasks["a"]["finishedAt"], tasks["c"]["finishedAt"]) + # Both routes wait for all producers, not only their own source. + self.assertAlmostEqual(transfers["a"]["startedAt"], last_producer, places=6) + self.assertAlmostEqual(transfers["c"]["startedAt"], last_producer, places=6) + phases = transfers["c"]["route"]["phases"] + self.assertEqual(len(phases), 2) + self.assertAlmostEqual(phases[0]["finishedAt"], phases[1]["startedAt"], places=6) + self.assertGreaterEqual(phases[0]["durationSeconds"], 1.999) + self.assertGreaterEqual(phases[1]["durationSeconds"], 2.999) + prepared = tasks["b"]["metadata"] + self.assertAlmostEqual(prepared["workspacePreparationSeconds"], 2) + self.assertAlmostEqual(prepared["workspacePreparationStartedAt"], transfers["c"]["finishedAt"], places=6) + self.assertAlmostEqual(tasks["b"]["startedAt"], transfers["c"]["finishedAt"] + 2, places=6) + self.assertAlmostEqual(result["executed"]["makespanSeconds"], 13, places=5) + self.assertEqual(transfers["c"]["networkBytes"], 200) + self.assertEqual(transfers["c"]["route"]["serverNetworkBytes"], 200) + self.assertEqual(transfers["a"]["route"]["serverNetworkBytes"], 0) + + def test_shared_link_contention_is_not_ignored(self): + model = self.fixture() + model["tasks"][0]["resourceId"] = "hpc" + model["tasks"][0]["flops"] = 1000000000 + model["tasks"][1]["flops"] = 1000000000 + second = copy.deepcopy(model["dependencies"][1]) + second["producerId"] = "a" + for flow in [second, model["dependencies"][1]]: + for stage in flow["stages"]: + stage["effectiveBytesPerSecond"] = 1000 + stage["afterSeconds"] = 0 + model["dependencies"][0] = second + platform = (self.root / "materialization-platform.xml").read_text().replace('bandwidth="1000000000Bps"', 'bandwidth="100Bps"') + result = self.run_model(model, platform) + for flow in result["transfers"]: + self.assertGreater(flow["durationSeconds"], 3.9) # 200B ingress then 200B egress sharing. + + def test_independent_source_preparation_matches_planner_without_extra_bytes(self): + model = self.fixture() + stage = model["dependencies"][1]["stages"][0] + stage["beforeSeconds"] = 2 + result = self.run_model(model) + self.assertAlmostEqual(result["executed"]["makespanSeconds"], 15, places=5) + flow = next(f for f in result["transfers"] if f["producerActivityId"] == "c") + self.assertEqual(flow["networkBytes"], 200) + self.assertEqual(flow["route"]["phases"][0]["serviceWaitSeconds"], 0) + + def test_shared_service_slot_serializes_distinct_routes_and_merge(self): + model = self.fixture() + # Both routes and the final merge use one control transport slot. + # There is no network bottleneck in this fixture. + for flow in model["dependencies"]: + for stage in flow["stages"]: + stage["serviceGroupId"] = "control-transport" + stage["serviceSlots"] = 1 + consumer = next(task for task in model["tasks"] if task["id"] == "b") + consumer["preparationServiceGroupId"] = "control-transport" + consumer["preparationServiceSlots"] = 1 + result = self.run_model(model) + # roots 5 + peer 1 + relay 2+3 + merge 2 + compute 1. + self.assertAlmostEqual(result["executed"]["makespanSeconds"], 14, places=5) + model["dependencies"][0]["stages"][0]["serviceSlots"] = 2 + self.assertIn("conflicting shared service capacities", self.run_model(model, success=False)) + + def test_six_predecessors_use_two_waves_of_three_transport_slots(self): + model = self.fixture() + model["tasks"] = [{"id": f"p{i}", "assignmentId": f"p{i}", + "resourceId": "hpc", "flops": 1000000000} for i in range(6)] + model["resourceLaneOrders"] = [{"predecessorId": f"p{i-1}", "successorId": f"p{i}"} for i in range(1, 6)] + model["tasks"].append({"id": "b", "assignmentId": "b", "resourceId": "cloud-b", + "flops": 1000000000, "preparationSeconds": 2, + "preparationServiceGroupId": "cloud-session", "preparationServiceSlots": 3}) + model["dependencies"] = [{"producerId": f"p{i}", "consumerId": "b", "bytes": 100, + "networkBytes": 0, "relayNetworkBytes": 0, "strategy": "gateway", "stages": [ + {"name": "source-pack", "sourceResourceId": "hpc", "targetResourceId": "relay", + "bytes": 0, "beforeSeconds": 10, "serviceGroupId": "hpc-session", "serviceSlots": 3}, + {"name": "target-extract", "sourceResourceId": "relay", "targetResourceId": "cloud-b", + "bytes": 0, "afterSeconds": 1, "serviceGroupId": "cloud-session", "serviceSlots": 3} + ]} for i in range(6)] + result = self.run_model(model) + starts = sorted(flow["route"]["phases"][0]["startedAt"] for flow in result["transfers"]) + self.assertEqual(starts, [6, 6, 6, 16, 16, 16]) + waits = sorted(flow["route"]["phases"][0]["serviceWaitSeconds"] for flow in result["transfers"]) + self.assertEqual(waits, [0, 0, 0, 10, 10, 10]) + self.assertAlmostEqual(result["executed"]["makespanSeconds"], 30, places=5) + + def test_peer_extraction_contends_on_destination_not_source_session(self): + model = self.fixture() + model["tasks"] = [{"id": f"p{i}", "assignmentId": f"p{i}", "resourceId": "cloud-a", "flops": 0} for i in range(6)] + model["tasks"].append({"id": "b", "assignmentId": "b", "resourceId": "cloud-b", "flops": 1000000000, + "preparationSeconds": 2, "preparationServiceGroupId": "destination", "preparationServiceSlots": 3}) + model["dependencies"] = [{"producerId": f"p{i}", "consumerId": "b", "bytes": 100, "networkBytes": 100, + "strategy": "direct-runtime", "stages": [ + {"name": "peer-upload", "sourceResourceId": "cloud-a", "targetResourceId": "cloud-b", + "bytes": 100, "effectiveBytesPerSecond": 100, "serviceGroupId": "source", "serviceSlots": 6}, + {"name": "peer-extract", "sourceResourceId": "cloud-b", "targetResourceId": "cloud-b", + "bytes": 0, "beforeSeconds": 10, "serviceGroupId": "destination", "serviceSlots": 3} + ]} for i in range(6)] + result = self.run_model(model) + self.assertAlmostEqual(result["executed"]["makespanSeconds"], 24, places=5) + self.assertEqual(sum(flow["networkBytes"] for flow in result["transfers"]), 600) + self.assertEqual(sum(flow["route"]["serverNetworkBytes"] for flow in result["transfers"]), 0) + + def test_legacy_zero_byte_control_dependency_does_not_deadlock(self): + model = self.fixture() + model.pop("materializationModelVersion") + for flow in model["dependencies"]: + flow.pop("stages") + flow["bytes"] = 0 + result = self.run_model(model) + self.assertEqual(len(result["transfers"]), 0) + self.assertEqual(len(result["tasks"]), 3) + + def test_proven_shared_storage_stage_does_not_move_network_bytes(self): + model = self.fixture() + model["tasks"] = [model["tasks"][1], model["tasks"][2]] + model["tasks"][1]["resourceId"] = "hpc-b" + model["tasks"][1]["preparationSeconds"] = 0 + flow = model["dependencies"][1] + flow.update(strategy="shared-storage", networkBytes=0, relayNetworkBytes=0) + flow["stages"] = [{"name":"shared-copy", "sourceResourceId":"hpc", "targetResourceId":"hpc-b", + "bytes":0, "beforeSeconds":1, "afterSeconds":2}] + model["dependencies"] = [flow] + platform = (self.root / "materialization-platform.xml").read_text().replace( + '\n