From 50caebdb9c493c4b35bb89180e596d2f9ab8e50e Mon Sep 17 00:00:00 2001 From: Sylphira-ovo Date: Mon, 19 Oct 2026 11:42:15 +0800 Subject: [PATCH] research: publish vehicle NONT-D Val397 results and audited artifacts --- research/vehicle_program_v1/.gitattributes | 4 + research/vehicle_program_v1/.gitignore | 7 + research/vehicle_program_v1/BRIEF.zh-CN.md | 11 + .../vehicle_program_v1/CROSSCHECK.zh-CN.md | 19 + research/vehicle_program_v1/README.md | 37 + research/vehicle_program_v1/REPORT.zh-CN.md | 39 + research/vehicle_program_v1/REPRODUCE.md | 38 + .../vehicle_program_v1/code/Night-Control.ps1 | 34 + research/vehicle_program_v1/code/README.md | 11 + .../vehicle_program_v1/code/api_backend.py | 282 ++++ research/vehicle_program_v1/code/api_check.py | 69 + .../vehicle_program_v1/code/api_discover.py | 48 + .../code/api_offline_audit.py | 69 + .../vehicle_program_v1/code/api_parallel.py | 276 ++++ .../vehicle_program_v1/code/api_prepare.py | 31 + research/vehicle_program_v1/code/artifacts.py | 109 ++ research/vehicle_program_v1/code/backends.py | 40 + research/vehicle_program_v1/code/bundle.py | 28 + research/vehicle_program_v1/code/night_ops.py | 83 ++ .../vehicle_program_v1/code/night_prepare.py | 89 ++ .../vehicle_program_v1/code/night_resume.py | 111 ++ .../vehicle_program_v1/code/night_runner.py | 248 ++++ .../code/night_supervisor.py | 145 ++ .../code/pinned-inputs.json | 12 + research/vehicle_program_v1/code/prepare.py | 106 ++ research/vehicle_program_v1/code/replay.py | 50 + research/vehicle_program_v1/code/runner.py | 249 ++++ research/vehicle_program_v1/code/score.py | 102 ++ .../vehicle_program_v1/code/tests/test_api.py | 190 +++ .../code/tests/test_night.py | 173 +++ .../code/tests/test_night_recovery.py | 180 +++ .../code/tests/test_parallel.py | 167 +++ .../code/tests/test_vehicle.py | 243 ++++ .../vehicle_program_v1/code/upstream.json | 124 ++ .../code/upstream/backend/download_model.py | 118 ++ .../code/upstream/backend/local_smoke.py | 250 ++++ .../upstream/runtime/program_loop/__init__.py | 5 + .../upstream/runtime/program_loop/__main__.py | 3 + .../upstream/runtime/program_loop/_vendor.py | 9 + .../runtime/program_loop/catalog_context.py | 41 + .../code/upstream/runtime/program_loop/cli.py | 72 + .../runtime/program_loop/contracts.py | 59 + .../runtime/program_loop/controller.py | 113 ++ .../runtime/program_loop/execution.py | 105 ++ .../runtime/program_loop/failure_policy.py | 60 + .../upstream/runtime/program_loop/planner.py | 121 ++ .../program_loop/planning_contracts.py | 30 + .../runtime/program_loop/prompt_encoding.py | 96 ++ .../runtime/program_loop/recovery_context.py | 41 + .../runtime/program_loop/rejection_context.py | 36 + .../runtime/program_loop/validation.py | 108 ++ .../code/upstream/runtime/runtime_identity.py | 31 + .../upstream/runtime/tests/test_grounding.py | 41 + .../runtime/tests/test_integration.py | 59 + .../code/upstream/runtime/vendor/common.py | 118 ++ .../runtime/vendor/comparison_output.py | 48 + .../upstream/runtime/vendor/evidence_roles.py | 77 ++ .../runtime/vendor/grounded_semantics.py | 64 + .../runtime/vendor/pool_availability.py | 115 ++ .../upstream/runtime/vendor/pool_bindings.py | 56 + .../upstream/runtime/vendor/pool_catalog.py | 111 ++ .../upstream/runtime/vendor/pool_metadata.py | 111 ++ .../upstream/runtime/vendor/pool_runtime.py | 564 ++++++++ .../runtime/vendor/reference_skills_63.json | 74 + .../vendor/runtime/code/agent/__init__.py | 3 + .../code/agent/orchestration/__init__.py | 35 + .../code/agent/orchestration/agent_loop.py | 1029 ++++++++++++++ .../orchestration/artifact_projection.py | 67 + .../code/agent/orchestration/catalog.py | 105 ++ .../agent/orchestration/category_catalog.py | 100 ++ .../code/agent/orchestration/contracts.py | 120 ++ .../agent/orchestration/metadata_lookup.py | 56 + .../code/agent/orchestration/metrics.py | 48 + .../orchestration/model_skill_executor.py | 837 +++++++++++ .../code/agent/orchestration/planner.py | 405 ++++++ .../code/agent/orchestration/retrieval.py | 335 +++++ .../code/agent/orchestration/runtime.py | 173 +++ .../code/agent/orchestration/skill_loader.py | 159 +++ .../agent/orchestration/table_field_lookup.py | 89 ++ .../agent/orchestration/topk_selection.py | 61 + .../code/agent/orchestration/validator.py | 131 ++ .../code/script/orchestration/__init__.py | 2 + .../orchestration/run_agent_loop_batch.py | 1223 +++++++++++++++++ ...dence_reasoning_ASSESS_CATEGORY_SUPPORT.md | 17 + ...ce_reasoning_CHECK_EVIDENCE_SUFFICIENCY.md | 19 + ...e_reasoning_COMPARE_CATEGORY_HYPOTHESES.md | 17 + ...e_reasoning_DECIDE_NEXT_EVIDENCE_SOURCE.md | 17 + ...dence_reasoning_FINALIZE_CLASSIFICATION.md | 19 + ...vidence_reasoning_FUSE_CONTEXT_EVIDENCE.md | 17 + ...idence_reasoning_FUSE_DATABASE_EVIDENCE.md | 17 + .../evidence_reasoning_FUSE_FIELD_EVIDENCE.md | 17 + ...idence_reasoning_FUSE_METADATA_EVIDENCE.md | 17 + ...idence_reasoning_FUSE_STANDARD_EVIDENCE.md | 17 + .../evidence_reasoning_FUSE_TABLE_EVIDENCE.md | 17 + ...nce_reasoning_IDENTIFY_MISSING_EVIDENCE.md | 17 + .../evidence_reasoning_INFER_CATEGORY.md | 19 + ...ence_reasoning_INFER_CATEGORY_FROM_TOPK.md | 16 + .../evidence_reasoning_INFER_GRADING_LEVEL.md | 19 + ...easoning_RESOLVE_FIELD_CONTEXT_CONFLICT.md | 17 + ...nce_reasoning_RESOLVE_METADATA_CONFLICT.md | 17 + ...nce_reasoning_RESOLVE_STANDARD_CONFLICT.md | 17 + ...reasoning_VERIFY_CATEGORY_WITH_STANDARD.md | 17 + ...RIFY_CLASSIFICATION_GRADING_CONSISTENCY.md | 17 + .../evidence_reasoning_WEIGH_EVIDENCE.md | 17 + ...ta_acquisition_ACQUIRE_MISSING_METADATA.md | 17 + ...acquisition_CHECK_METADATA_AVAILABILITY.md | 19 + ...ata_acquisition_EXPAND_SELECTED_CONTEXT.md | 17 + ...quisition_GET_COMPLETE_DATABASE_CONTEXT.md | 17 + ..._acquisition_GET_COMPLETE_TABLE_CONTEXT.md | 17 + ...uisition_GET_CONTEXT_FIELD_DESCRIPTIONS.md | 17 + ...ata_acquisition_GET_CONTEXT_FIELD_TYPES.md | 17 + ...ta_acquisition_GET_CONTEXT_FIELD_VALUES.md | 17 + ...ta_acquisition_GET_DATABASE_DESCRIPTION.md | 17 + ...adata_acquisition_GET_FIELD_DESCRIPTION.md | 17 + ...metadata_acquisition_GET_FIELD_METADATA.md | 21 + ...adata_acquisition_GET_TABLE_DESCRIPTION.md | 17 + .../metadata_acquisition_GET_TABLE_FIELDS.md | 17 + ...adata_acquisition_SELECT_CONTEXT_FIELDS.md | 17 + ...rpretation_DISAMBIGUATE_FIELD_SEMANTICS.md | 19 + ...rpretation_EXPAND_DATABASE_ABBREVIATION.md | 17 + ...nterpretation_EXPAND_FIELD_ABBREVIATION.md | 19 + ...nterpretation_EXPAND_TABLE_ABBREVIATION.md | 17 + ...interpretation_INTERPRET_CONTEXT_FIELDS.md | 17 + ...retation_INTERPRET_DATABASE_DESCRIPTION.md | 17 + ..._interpretation_INTERPRET_DATABASE_NAME.md | 17 + ...emantic_interpretation_INTERPRET_DOMAIN.md | 19 + ...erpretation_INTERPRET_FIELD_DESCRIPTION.md | 19 + ...tic_interpretation_INTERPRET_FIELD_NAME.md | 19 + ...tic_interpretation_INTERPRET_FIELD_TYPE.md | 17 + ...ic_interpretation_INTERPRET_FIELD_VALUE.md | 17 + ...erpretation_INTERPRET_TABLE_DESCRIPTION.md | 21 + ...tic_interpretation_INTERPRET_TABLE_NAME.md | 17 + ...pretation_SYNTHESIZE_METADATA_SEMANTICS.md | 17 + ...ard_alignment_ALIGN_CONTEXT_TO_STANDARD.md | 17 + ...rd_alignment_ALIGN_DATABASE_TO_STANDARD.md | 17 + ...alignment_ALIGN_DESCRIPTION_TO_STANDARD.md | 17 + ...ndard_alignment_ALIGN_FIELD_TO_STANDARD.md | 17 + ...ndard_alignment_ALIGN_TABLE_TO_STANDARD.md | 17 + ...ndard_alignment_ALIGN_VALUE_TO_STANDARD.md | 17 + ...d_alignment_COMPARE_STANDARD_CANDIDATES.md | 19 + ...tandard_alignment_REFINE_STANDARD_QUERY.md | 17 + .../standard_alignment_RESEARCH_STANDARD.md | 17 + .../standard_alignment_SEARCH_STANDARD.md | 19 + ...rd_alignment_SEARCH_STANDARD_BY_CONTEXT.md | 17 + ...d_alignment_SEARCH_STANDARD_BY_DATABASE.md | 17 + ...dard_alignment_SEARCH_STANDARD_BY_FIELD.md | 17 + ...dard_alignment_SEARCH_STANDARD_BY_TABLE.md | 17 + ...standard_alignment_SELECT_CATEGORY_TOPK.md | 16 + ...d_alignment_SUMMARIZE_STANDARD_EVIDENCE.md | 17 + .../runtime/vendor/skill_reference.json | 380 +++++ .../code/upstream/runtime/vendor/study.py | 323 +++++ .../code/vendor_snapshot.py | 47 + .../publication_tests/test_publication.py | 69 + .../results/code-publication.json | 157 +++ .../vehicle_program_v1/results/diagnosis.json | 71 + .../results/evidence-summary.json | 96 ++ .../results/files.sha256.json | 164 +++ .../results/model-roles.json | 50 + .../results/publication-verification.json | 39 + .../results/trajectory-export.json | 17 + .../tools/redact_trajectories.py | 56 + .../tools/verify_release.py | 24 + .../trajectories/val397.redacted.jsonl | 397 ++++++ 163 files changed, 14180 insertions(+) create mode 100644 research/vehicle_program_v1/.gitattributes create mode 100644 research/vehicle_program_v1/.gitignore create mode 100644 research/vehicle_program_v1/BRIEF.zh-CN.md create mode 100644 research/vehicle_program_v1/CROSSCHECK.zh-CN.md create mode 100644 research/vehicle_program_v1/README.md create mode 100644 research/vehicle_program_v1/REPORT.zh-CN.md create mode 100644 research/vehicle_program_v1/REPRODUCE.md create mode 100644 research/vehicle_program_v1/code/Night-Control.ps1 create mode 100644 research/vehicle_program_v1/code/README.md create mode 100644 research/vehicle_program_v1/code/api_backend.py create mode 100644 research/vehicle_program_v1/code/api_check.py create mode 100644 research/vehicle_program_v1/code/api_discover.py create mode 100644 research/vehicle_program_v1/code/api_offline_audit.py create mode 100644 research/vehicle_program_v1/code/api_parallel.py create mode 100644 research/vehicle_program_v1/code/api_prepare.py create mode 100644 research/vehicle_program_v1/code/artifacts.py create mode 100644 research/vehicle_program_v1/code/backends.py create mode 100644 research/vehicle_program_v1/code/bundle.py create mode 100644 research/vehicle_program_v1/code/night_ops.py create mode 100644 research/vehicle_program_v1/code/night_prepare.py create mode 100644 research/vehicle_program_v1/code/night_resume.py create mode 100644 research/vehicle_program_v1/code/night_runner.py create mode 100644 research/vehicle_program_v1/code/night_supervisor.py create mode 100644 research/vehicle_program_v1/code/pinned-inputs.json create mode 100644 research/vehicle_program_v1/code/prepare.py create mode 100644 research/vehicle_program_v1/code/replay.py create mode 100644 research/vehicle_program_v1/code/runner.py create mode 100644 research/vehicle_program_v1/code/score.py create mode 100644 research/vehicle_program_v1/code/tests/test_api.py create mode 100644 research/vehicle_program_v1/code/tests/test_night.py create mode 100644 research/vehicle_program_v1/code/tests/test_night_recovery.py create mode 100644 research/vehicle_program_v1/code/tests/test_parallel.py create mode 100644 research/vehicle_program_v1/code/tests/test_vehicle.py create mode 100644 research/vehicle_program_v1/code/upstream.json create mode 100644 research/vehicle_program_v1/code/upstream/backend/download_model.py create mode 100644 research/vehicle_program_v1/code/upstream/backend/local_smoke.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/program_loop/__init__.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/program_loop/__main__.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/program_loop/_vendor.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/program_loop/catalog_context.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/program_loop/cli.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/program_loop/contracts.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/program_loop/controller.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/program_loop/execution.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/program_loop/failure_policy.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/program_loop/planner.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/program_loop/planning_contracts.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/program_loop/prompt_encoding.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/program_loop/recovery_context.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/program_loop/rejection_context.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/program_loop/validation.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/runtime_identity.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/tests/test_grounding.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/tests/test_integration.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/common.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/comparison_output.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/evidence_roles.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/grounded_semantics.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/pool_availability.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/pool_bindings.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/pool_catalog.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/pool_metadata.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/pool_runtime.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/reference_skills_63.json create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/__init__.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/__init__.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/agent_loop.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/artifact_projection.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/catalog.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/category_catalog.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/contracts.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/metadata_lookup.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/metrics.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/model_skill_executor.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/planner.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/retrieval.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/runtime.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/skill_loader.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/table_field_lookup.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/topk_selection.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/validator.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/script/orchestration/__init__.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/script/orchestration/run_agent_loop_batch.py create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_ASSESS_CATEGORY_SUPPORT.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_CHECK_EVIDENCE_SUFFICIENCY.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_COMPARE_CATEGORY_HYPOTHESES.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_DECIDE_NEXT_EVIDENCE_SOURCE.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FINALIZE_CLASSIFICATION.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_CONTEXT_EVIDENCE.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_DATABASE_EVIDENCE.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_FIELD_EVIDENCE.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_METADATA_EVIDENCE.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_STANDARD_EVIDENCE.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_TABLE_EVIDENCE.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_IDENTIFY_MISSING_EVIDENCE.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY_FROM_TOPK.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_GRADING_LEVEL.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_FIELD_CONTEXT_CONFLICT.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_METADATA_CONFLICT.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_STANDARD_CONFLICT.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CATEGORY_WITH_STANDARD.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CLASSIFICATION_GRADING_CONSISTENCY.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_WEIGH_EVIDENCE.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_ACQUIRE_MISSING_METADATA.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_CHECK_METADATA_AVAILABILITY.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_EXPAND_SELECTED_CONTEXT.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_DATABASE_CONTEXT.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_TABLE_CONTEXT.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_DESCRIPTIONS.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_TYPES.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_VALUES.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_DATABASE_DESCRIPTION.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_DESCRIPTION.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_METADATA.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_DESCRIPTION.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_FIELDS.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_SELECT_CONTEXT_FIELDS.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_DISAMBIGUATE_FIELD_SEMANTICS.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_DATABASE_ABBREVIATION.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_FIELD_ABBREVIATION.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_TABLE_ABBREVIATION.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_CONTEXT_FIELDS.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_DESCRIPTION.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_NAME.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DOMAIN.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_DESCRIPTION.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_NAME.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_TYPE.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_VALUE.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_DESCRIPTION.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_NAME.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_SYNTHESIZE_METADATA_SEMANTICS.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_CONTEXT_TO_STANDARD.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DATABASE_TO_STANDARD.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DESCRIPTION_TO_STANDARD.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_FIELD_TO_STANDARD.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_TABLE_TO_STANDARD.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_VALUE_TO_STANDARD.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_COMPARE_STANDARD_CANDIDATES.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_REFINE_STANDARD_QUERY.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_RESEARCH_STANDARD.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_CONTEXT.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_DATABASE.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_FIELD.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_TABLE.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SELECT_CATEGORY_TOPK.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SUMMARIZE_STANDARD_EVIDENCE.md create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/skill_reference.json create mode 100644 research/vehicle_program_v1/code/upstream/runtime/vendor/study.py create mode 100644 research/vehicle_program_v1/code/vendor_snapshot.py create mode 100644 research/vehicle_program_v1/publication_tests/test_publication.py create mode 100644 research/vehicle_program_v1/results/code-publication.json create mode 100644 research/vehicle_program_v1/results/diagnosis.json create mode 100644 research/vehicle_program_v1/results/evidence-summary.json create mode 100644 research/vehicle_program_v1/results/files.sha256.json create mode 100644 research/vehicle_program_v1/results/model-roles.json create mode 100644 research/vehicle_program_v1/results/publication-verification.json create mode 100644 research/vehicle_program_v1/results/trajectory-export.json create mode 100644 research/vehicle_program_v1/tools/redact_trajectories.py create mode 100644 research/vehicle_program_v1/tools/verify_release.py create mode 100644 research/vehicle_program_v1/trajectories/val397.redacted.jsonl diff --git a/research/vehicle_program_v1/.gitattributes b/research/vehicle_program_v1/.gitattributes new file mode 100644 index 0000000..f25acf0 --- /dev/null +++ b/research/vehicle_program_v1/.gitattributes @@ -0,0 +1,4 @@ +# Preserve exact publication bytes for SHA verification on every platform. +* -text whitespace=blank-at-eol,blank-at-eof,space-before-tab,cr-at-eol +# Frozen upstream includes three existing extra EOF blank lines; retain hashes. +code/upstream/** whitespace=blank-at-eol,space-before-tab,cr-at-eol,-blank-at-eof diff --git a/research/vehicle_program_v1/.gitignore b/research/vehicle_program_v1/.gitignore new file mode 100644 index 0000000..806d880 --- /dev/null +++ b/research/vehicle_program_v1/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.pyc +.pytest_cache/ +.env +credentials.md +private/ +!trajectories/*.redacted.jsonl diff --git a/research/vehicle_program_v1/BRIEF.zh-CN.md b/research/vehicle_program_v1/BRIEF.zh-CN.md new file mode 100644 index 0000000..1b49541 --- /dev/null +++ b/research/vehicle_program_v1/BRIEF.zh-CN.md @@ -0,0 +1,11 @@ +# 汽车 NONT-D 完成汇报 + +陈一鸿，2026-10-19。 + +已完成汽车Val397无训练Program基线：Qwen3.5-9B、F66、SiliconFlow API。分类33/397=8.31%，分级53/397=13.35%，联合26/397=6.55%，合法终局115/397=28.97%。265条预算耗尽、17条后端错误均保留在397分母。 + +累计usage与未知结果预留估算49.07元（非账单），生成墙钟约9小时28分，含熔断后等待/恢复。保留9条先导，后续388条生成；未按结果重抽。397份收据与主指标复算、账本对账通过；380份非后端错误轨迹完整回放通过。 + +主要可见瓶颈是候选标准覆盖、比较no_match后的恢复规划，以及合法终局中的错误选择。结果较低，如实作为基线保留。汽车59标准条目对应44叶子，多条目可共享分类或等级；API精度/revision未核实。A/B/C合表口径待与赵天畅核对。 + +代码、汇总指标、脱敏轨迹和验证材料见本目录[README](README.md)。本次只有NONT-D，未完成汽车SFT/RL；Val397不是独立Test。公开轨迹为有损投影，不能用于训练或完整回放。 diff --git a/research/vehicle_program_v1/CROSSCHECK.zh-CN.md b/research/vehicle_program_v1/CROSSCHECK.zh-CN.md new file mode 100644 index 0000000..cc92254 --- /dev/null +++ b/research/vehicle_program_v1/CROSSCHECK.zh-CN.md @@ -0,0 +1,19 @@ +# 与汽车 NONT-A/B/C 合表前的核验清单 + +状态：待双方核对，不能据本清单宣称已经完成公平对照。 + +| 项目 | 本次NONT-D | 需A/B/C确认 | +|---|---|---| +| 数据 | DocAILab/DCG@84a9a462，Val397 | 相同revision、ID集合和分母；不混Test395 | +| 输入权限 | 初始domain/field_name，按需field_description | 模型不得看到分类、分级或样本顶层标准匹配信息 | +| 标准 | 59条目/44叶子，category内部为条目ID | Top-5按条目还是去重类别；同叶子多等级如何处理 | +| 模型 | SiliconFlow Qwen3.5-9B，revision/精度未核实 | 是否同服务和模型；本地FP16不能默认等价 | +| 生成 | 32K上下文，512输出，temperature=0，thinking=false | 相同或明确披露差异 | +| C/D控制 | 同F66；D最多3Program/4规划/12Skill，每Program10步 | 共同Skill后端、总执行预算、工具版本和反馈规则 | +| 主评分 | 路径分类、分级、联合；失败全部入397 | 独立重算，合法终局率不冒充类别取值合法率 | +| F1 | 支持14叶子及固定44叶子分别报告 | 标签全集、无效预测处理；不照搬固定56联合标签 | +| 来源历史 | 9先导＋388后续，显式基础设施恢复 | 先导是否纳入、失败是否重跑、费用/耗时口径 | + +A=直接回答；B=固定Top-5→Top-1两个核心Skill；C=逐步选Skill；D=有序Program及预算内重规划。四组均未训练。C/D是编排方式对照，D/后续TRAIN-A是Planner SFT对照。 + +项目指标表中Weighted-F1、分级/联合Macro-F1、Macro-Precision/Recall及answer/level解析率等，需按双方统一评测定义补齐；本次不将Program validated_terminal强行改名为另一种输出schema的解析率，也不复制金融固定标签数。 diff --git a/research/vehicle_program_v1/README.md b/research/vehicle_program_v1/README.md new file mode 100644 index 0000000..43b69a3 --- /dev/null +++ b/research/vehicle_program_v1/README.md @@ -0,0 +1,37 @@ +# 汽车 NONT-D / ProgSkill：Val397 无训练结果 + +陈一鸿，2026-10-19。固定汽车开发验证集397条，使用未微调的 Qwen3.5-9B Planner 与模型型 Skill，F66 Program 框架。完整处理并验收；**性能较低的基线结果如实保留**。 + +| 指标 | 数量 / 全量分母 | 比例 | +|---|---:|---:| +| 分类正确 | 33/397 | 8.31% | +| 分级正确 | 53/397 | 13.35% | +| 联合正确 | 26/397 | 6.55% | +| 合法终局 | 115/397 | 28.97% | +| 任次Top-5叶子命中 | 214/397 | 53.90% | +| 任次Top-5含分类及等级兼容条目 | 104/397 | 26.20% | + +265条预算耗尽、17条来源后端错误全部计入397分母，不重抽、不删失败。分类Macro-F1：14个有支持叶子0.212365，固定44叶子0.067571。分级来自选中标准条目。 + +累计usage及未知结果预留估算 **49.0696005元**，非供应商账单；2582个记录调用入口（含先导及本地发送前拒绝），已知输入29,999,427、输出271,045 tokens。夜间账本2536次HTTP，先导另有44次；二者不能与2582入口次数混淆。 + +## 材料入口 + +- [中文实验报告](REPORT.zh-CN.md)：范围、结果、失败分析、限制。 +- [汇报摘要](BRIEF.zh-CN.md)：便于项目文档引用。 +- [复现与发布边界](REPRODUCE.md)：CPU检查、受控重建、身份差异。 +- [指标JSON](results/evidence-summary.json)、[模型与预算](results/model-roles.json)、[聚合诊断](results/diagnosis.json)。 +- [汽车A/B/C交叉核验清单](CROSSCHECK.zh-CN.md)：尚未完成跨方法口径核验。 +- `code/`：汽车适配、执行、API限流/恢复、评分及回放代码和测试，含固定上游F66代码。 +- `trajectories/val397.redacted.jsonl`：397条有损公开投影，包含全部失败；不是训练集或完整回放输入。 +- `results/trajectory-export.json`、`results/publication-verification.json`、`results/files.sha256.json`：导出清单、检查结果、发布文件哈希。 + +## 解释边界 + +数据固定 `DocAILab/DCG@84a9a462dcb828332f980ef121f40e1281ce1794`，Val397不是Test395。模型服务为SiliconFlow，精度与权重revision未核实，不能宣称等同金融本地FP16。59标准条目对应44叶子，Top-5条目不保证是5个不同类别。 + +先导9条全部保留，剩余388条生成；332条时接口错误触发熔断，经基础设施冷却恢复修复后补完65条。Prompt、模型名称、Program预算和评分不变，费用及原始计时未重置。先导与后续调度条件不同，不能称397条均首次在同一profile运行。 + +397份收据文件校验、逐条主指标复算及2536笔夜间请求账本对账通过；380份非后端错误轨迹完整回放通过，退出码0。公开投影不含真实元数据、原始消息、推理文本、gold或源ID；分类与等级使用不可反查的公开别名。原始证据受控保留。 + +仅发布汽车NONT-D；A/B/C不在本次结果内，SFT/RL未运行。clone/import及CPU测试不会启动真实模型或训练。 diff --git a/research/vehicle_program_v1/REPORT.zh-CN.md b/research/vehicle_program_v1/REPORT.zh-CN.md new file mode 100644 index 0000000..1b71346 --- /dev/null +++ b/research/vehicle_program_v1/REPORT.zh-CN.md @@ -0,0 +1,39 @@ +# 汽车 NONT-D 实验报告 + +日期：2026-10-19；执行者：陈一鸿。目标：接用团队F66 Program框架，完成汽车领域未训练Planner基线，为后续同接口训练对照提供可复核起点。 + +## 方法与固定条件 + +上游代码来自团队commit `9473b2c2a330b7f48dc0c1831bda11e36d428379`，117个上游文件保持一致。Planner生成有序Program与状态引用，执行遇到失败后在余下预算内重新规划；不是随机抽取Skill。 + +初始输入只有domain与field_name，工具能获取当前来源的field_name和field_description；无表库上下文，不补造缺失元数据。样本分类、分级及顶层标准匹配说明不进入推理输入；gold仅用于独立评分。标准目录保持59个条目、44个叶子完整路径。 + +Planner与模型型Skill均为SiliconFlow `Qwen/Qwen3.5-9B`，无SFT/RL、无adapter。temperature=0，thinking关闭，输出512，上下文32768。官方tokenizer仅作参考容量校验；API端精度、模型revision和确定性未获验证。历史代码中的local后端FP16/seed参数不代表API服务实际身份。 + +预算：最多3个Program、每Program10步、累计12个Skill步骤、4次Planner尝试、同证据链相同成功调用限1次。夜间2路并发、72000 TPM；70元含先导估算停止线、原始14小时墙钟限制。HTTP错误不自动重试旧来源。 + +## 结果及完整性 + +主指标见[README](README.md)及[机器可读结果](results/evidence-summary.json)。全部397条均有收据；115条合法终局、219条Program预算耗尽、46条Planner预算耗尽、17条来源后端错误。失败均进入全量分母。合法终局中26/115=22.61%联合正确。 + +原始计时00:50:48至10:18:33生成结束，共9小时27分44秒，包含停机等待与恢复；10:20:43本机评分回放结束。费用核算为已知夜间usage47.5121280元＋12笔未知结果预留0.654336元＋先导0.9031365元＝49.0696005元，不是账单。 + +9条先导来自1条canary和两批各4条测速，全部原样保留（含5条后端失败）。夜间生成其余388条；332条时503/超时触发连续错误熔断，原请求尾部冷却校验缺陷修复后显式迁移至v8续跑65条。原332份收据和2095笔账目不变，未改模型任务逻辑、Prompt或预算，未根据正确性重抽。不同阶段服务与调度条件须随结果披露。 + +原始397条数据与study/gold的ID、标签和允许元数据逐条对应；主指标另行逐条复算一致。全部收据绑定文件hash通过，2536笔夜间请求hash/结果/usage/金额对账通过，380份非后端错误轨迹完整控制器回放通过。回放验证执行一致性，不等于专家确认标注或算法设计正确。 + +## 低分分析 + +已观察路径中，104条曾检索到分类与等级兼容条目，其中58条形成合法终局，26条联合正确。既存在联合标准候选覆盖不足，也存在有候选但未完成或选错的问题。 + +176条遇到比较no_match，之后仅18条合法终止；142条重复同有效输入比较被拦截。51条实际比较候选已含gold兼容条目仍no_match，其中44条最终未形成合法终局。此处“兼容”仅指与提供gold一致，不是独立专家裁决。共享提示词已经说明比较非所有路径必选，但未训练Planner常未有效切换路径。 + +379条成功获取字段元数据；98条尝试字段类型解释失败。412组五条目候选中191组不足5个不同叶子，平均4.279个不同叶子；候选单位应与汽车A/B/C核对，不能直接套用金融类别Top-5口径。 + +剔除17个后端错误，联合仍只有26/380=6.84%；即使这17条全答对，也仅43/397=10.83%。12个调用触及输出长度限制，涉及10个来源，不能解释大多数失败。完全相同允许输入仅1组2条标签冲突，也不足以解释整体低分。 + +## 结论与后续 + +本轮可作为如实报告的汽车无训练基线。低合法率与合法答案内低正确率同时存在，不支持仅增加预算或原样重跑。金融团队同框架发布的联合18.72%、合法45.26%只作外部参照；数据、标准空间、元数据和后端不同，不能据此单独归因于领域难度。 + +先与汽车A/B/C核对候选单位、输入权限、模型服务及评分，尤其C/D的F66和总预算。后续只在隔离训练来源准备Planner示范和训练，不把Val397诊断轨迹转作SFT数据。本发布未运行训练或新的模型请求。 diff --git a/research/vehicle_program_v1/REPRODUCE.md b/research/vehicle_program_v1/REPRODUCE.md new file mode 100644 index 0000000..2a84385 --- /dev/null +++ b/research/vehicle_program_v1/REPRODUCE.md @@ -0,0 +1,38 @@ +# Reproduction and publication boundaries + +This release includes code, aggregate metrics and 397 lossy trajectory projections. It does not include controlled data/gold, private source-ID mappings, raw prompts/replies, tokenizer/model assets, credentials or historical machine paths. + +## Offline checks + +Python 3.11+; publication checks were run with Python 3.14 on Windows. Run these commands separately from this directory: + +```text +python -B -m unittest discover -s code/tests -v +python -B -m unittest discover -s code/upstream/runtime/tests -v +python -B -m unittest discover -s publication_tests -v +python -B tools/verify_release.py +``` + +The tests use fictional fixtures and scripted/mock transports. They do not call live models or train. Windows supervisor/status operations require psutil. Actual API reference token counting additionally requires tokenizers and jinja2 and the separately authorized pinned reference tokenizer. Retained local GPU code is not evidence that this API experiment used local weights. + +## Code provenance + +`results/code-publication.json` binds all 145 files in the historical final runtime. The 117 vendored upstream files remain byte-identical to team commit `9473b2c2a330b7f48dc0c1831bda11e36d428379`. + +The public copy changes exactly three historical files: `code/README.md`, the local credential-file default in `api_backend.py`, and the local tokenizer-path default in `night_prepare.py`. Environment variables VEHICLE_NONTD_CREDENTIAL_FILE and VEHICLE_NONTD_TOKENIZER accept local asset paths, not API keys. These publication changes do not change Planner/Skill prompts, execution budgets, inference policy or scoring. New manifests are required; the public copy must not be inserted into the immutable historical run. Original code hashes and private run/score/replay hashes are retained in the aggregate evidence. + +## Controlled data and fresh runs + +Obtain the authorized DocAILab/DCG dataset at revision `84a9a462dcb828332f980ef121f40e1281ce1794` independently. `code/pinned-inputs.json` provides five input file hashes. From `code/`, `prepare.py --source --output ` builds allowlisted inference data and an evaluator-only gold file. The supplied tests exercise the same path with fictional inputs. Preparation does not start a model. + +`runner.py` provides mock/local/API execution; `score.py` performs evaluator-only scoring; `replay.py` re-executes a recorded complete controller trace without a model. `night_prepare.py`, `night_runner.py`, `night_supervisor.py` and `Night-Control.ps1` contain the bounded Windows API orchestration and recovery checks. Help and import do not start paid work. Do not infer permission for a new live run from this publication. + +Exact historical replay requires the original immutable runtime and manifests, controlled study/registry, full traces and call records. Independent numerical scoring additionally requires controlled gold. These are intentionally absent. Hosted model weight revision/precision is not exposed, so a new API run is not guaranteed to reproduce the same outputs. + +## Public trajectory schema + +Rows are `val-0001` through `val-0397` in fixed evaluation order. Only known tool names, integer budget/attempt counters, enumerated statuses, termination flags and opaque predicted category/grade aliases are retained. All arguments (including state references), messages, raw outputs, observations, summaries, source metadata and gold are omitted. Invalid Planner replies retain their attempt/status, never the raw text. A trace SHA binds each public row to privately retained evidence. Aliases are consistent within this release; no mapping to private source IDs or standard labels is published. + +All 397 receipts are represented, including 17 backend errors and 265 nonterminal budget failures. Public projections allow auditing aggregate completion and tool-sequence behavior; they cannot independently establish semantic correctness or serve as full replay/SFT data. + +`results/files.sha256.json` lists every publication file except itself. Compare these hashes after download. `results/publication-verification.json` records the actual local checks and privacy scan scope; no GPU/API experiment is part of publication. diff --git a/research/vehicle_program_v1/code/Night-Control.ps1 b/research/vehicle_program_v1/code/Night-Control.ps1 new file mode 100644 index 0000000..32747fc --- /dev/null +++ b/research/vehicle_program_v1/code/Night-Control.ps1 @@ -0,0 +1,34 @@ +param( + [Parameter(Mandatory=$true)][string]$Plan, + [ValidateSet('Check','Start','Status','Stop','Resume')][string]$Action='Status', + [switch]$Live +) +$ErrorActionPreference='Stop' +$nightPlan = Get-Content -LiteralPath $Plan -Raw -Encoding UTF8 | ConvertFrom-Json +$ops = Join-Path $PSScriptRoot 'night_ops.py' +$supervisor = Join-Path $PSScriptRoot 'night_supervisor.py' +if ($Action -in @('Check','Status','Stop')) { + & $nightPlan.python -B $ops --plan $Plan --action $Action.ToLower() + exit $LASTEXITCODE +} +if ($nightPlan.mode -eq 'api' -and -not $Live) { throw 'Live start requires -Live; preparation never starts model calls.' } +$preflight = if ($Action -eq 'Resume') { 'resume-check' } else { 'check' } +& $nightPlan.python -B $ops --plan $Plan --action $preflight +if ($LASTEXITCODE -ne 0) { throw 'Offline preflight failed.' } +$taskArgs = '"' + $supervisor + '" --plan "' + $Plan + '"' +if ($Action -eq 'Resume') { $taskArgs += ' --resume' } +$taskAction = New-ScheduledTaskAction -Execute $nightPlan.pythonw -Argument $taskArgs -WorkingDirectory $PSScriptRoot +$taskSettings = New-ScheduledTaskSettingsSet -Hidden -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Hours 16) -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries +$principal = New-ScheduledTaskPrincipal -UserId ([System.Security.Principal.WindowsIdentity]::GetCurrent().Name) -LogonType Interactive -RunLevel Limited +$existing = Get-ScheduledTask -TaskName $nightPlan.task_name -ErrorAction SilentlyContinue +if ($existing) { + if ($existing.Actions[0].Execute -ne $nightPlan.pythonw -or $existing.Actions[0].Arguments -notlike ('*"' + $Plan + '"*')) { + throw 'Task name belongs to a different action.' + } + if ($existing.State -eq 'Running') { throw 'Night task is already running.' } + Unregister-ScheduledTask -TaskName $nightPlan.task_name -Confirm:$false +} +# No time trigger: the human must explicitly request Start or Resume. +Register-ScheduledTask -TaskName $nightPlan.task_name -Action $taskAction -Settings $taskSettings -Principal $principal | Out-Null +Start-ScheduledTask -TaskName $nightPlan.task_name +Write-Output ('Started independent task: ' + $nightPlan.task_name) diff --git a/research/vehicle_program_v1/code/README.md b/research/vehicle_program_v1/code/README.md new file mode 100644 index 0000000..4f95765 --- /dev/null +++ b/research/vehicle_program_v1/code/README.md @@ -0,0 +1,11 @@ +# Vehicle NONT-D execution code + +This is a publication copy of the final v8 runtime, not the identity of the historical immutable run. See ../REPRODUCE.md and ../results/code-publication.json. + +The upstream F66 runtime is byte-identical to team commit 9473b2c2a330b7f48dc0c1831bda11e36d428379. Only this README and local asset-path defaults in api_backend.py/night_prepare.py differ from the final runtime. Model prompts, controller logic, budgets and scoring are unchanged. + +Credentials are read only by explicit live operations, from VEHICLE_NONTD_CREDENTIAL_FILE or ~/.config/vehicle-nontd/credentials.md. The existing parser expects the dedicated heading 汽车 NONT-D SiliconFlow API and one API Key line. Never commit that file. VEHICLE_NONTD_TOKENIZER selects a separately authorized local reference tokenizer. No credentials, tokenizer assets, real study, gold or raw traces are included. + +Python 3.11+; offline tests use the standard library. Actual Windows status/supervision also requires psutil; real reference counting requires tokenizers and jinja2. Public tests never request a real model. The retained local GPU backend was not used for the reported API result. + +Run `python -B -m unittest discover -s tests -v` from this directory. New controlled runs require newly prepared manifests; do not insert this changed code into the old run or weaken identity checks. diff --git a/research/vehicle_program_v1/code/api_backend.py b/research/vehicle_program_v1/code/api_backend.py new file mode 100644 index 0000000..6a23c8a --- /dev/null +++ b/research/vehicle_program_v1/code/api_backend.py @@ -0,0 +1,282 @@ +"""Bounded text-only hosted Qwen3.5-9B inference; credentials never enter artifacts.""" +from copy import deepcopy +import json +import math +import os +from pathlib import Path +import re +import time +import urllib.error +import urllib.request + +from artifacts import BUDGET, GENERATION, MODEL_REVISION, fingerprint, put, read, sha + +MODEL = 'Qwen/Qwen3.5-9B' +TOKENIZER_FILES = { + 'chat_template.jinja': 'a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715', + 'config.json': 'd0883072e01861ed0b2d47be3c16c36a8e81c224c7ffaa310c6558fb3f932b05', + 'merges.txt': 'a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d', + 'tokenizer.json': '5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42', + 'tokenizer_config.json': '316230d6a809701f4db5ea8f8fc862bc3a6f3229c937c174e674ff3ca0a64ac8', + 'vocab.json': 'ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003', +} +CREDENTIAL_FILE = Path(os.environ.get('VEHICLE_NONTD_CREDENTIAL_FILE', str(Path.home() / '.config/vehicle-nontd/credentials.md'))) +CREDENTIAL_HEADING = '汽车 NONT-D SiliconFlow API' + + +class APIError(RuntimeError): + """Only fixed error codes are public; never include server error bodies.""" + + +class APIBudgetExceeded(APIError): + pass + + +def default_config(tokenizer): + return {'schema': 'vehicle-nontd-api-v2', 'provider': 'siliconflow', + 'base_url': 'https://api.siliconflow.cn/v1', 'model': MODEL, + 'provider_precision': 'not_verified', 'provider_revision': 'not_exposed', + 'tokenizer_path': str(Path(tokenizer).resolve()), + 'pricing_cny_per_million': {'input': 1.5, 'output': 12.}, + 'limits': {'max_http_requests': 32, 'max_estimated_cny': 1., + 'min_interval_seconds': 30., 'request_timeout_seconds': 90., + 'max_wall_seconds': 1800.}} + + +def load_config(path): + config = read(path) + expected = set(default_config('.')) + if set(config) != expected or config['schema'] != 'vehicle-nontd-api-v2': + raise ValueError('API profile fields differ; credentials are not profile fields') + if config['provider'] != 'siliconflow' or config['model'] != MODEL: + raise ValueError('this profile supports only the named hosted Qwen3.5-9B service') + # Credentials must never be forwarded to an unreviewed host or redirect target. + if config['base_url'] != 'https://api.siliconflow.cn/v1': + raise ValueError('API endpoint differs from the reviewed provider') + if config['provider_precision'] != 'not_verified' or config['provider_revision'] != 'not_exposed': + raise ValueError('do not claim hosted FP16 or a verified HF weight revision') + limits = config['limits'] + if set(limits) != set(default_config('.')['limits']): + raise ValueError('invalid API limit fields') + for value in [*limits.values(), *config['pricing_cny_per_million'].values()]: + if type(value) not in (int, float) or not math.isfinite(value) or value <= 0: + raise ValueError('API limits and prices must be finite and positive') + if type(limits['max_http_requests']) is not int: + raise ValueError('request limit must be an integer') + if set(config['pricing_cny_per_million']) != {'input', 'output'}: + raise ValueError('invalid price fields') + return config + + +def api_identity(config): + public = {k: deepcopy(v) for k, v in config.items() if k != 'tokenizer_path'} + public.update(reference_tokenizer={'repo_id': MODEL, 'revision': MODEL_REVISION, + 'files': TOKENIZER_FILES}, + hosted_weight_hash_verified=False, local_fp16_equivalence_claim=False) + return public + + +def api_generation(): + return {'seed_requested': None, 'seed_support': 'not_documented_by_provider', 'temperature': 0., + 'context_cap': GENERATION['context_cap'], 'max_new_tokens': GENERATION['max_new_tokens'], + 'enable_thinking': False, 'provider_precision': 'not_verified', + 'deterministic_output_guaranteed': False} + + +class ReferenceCounter: + """Offline official template/tokenizer, not proof of the provider's exact template.""" + def __init__(self, folder): + folder = Path(folder) + for name, expected in TOKENIZER_FILES.items(): + if (folder / name).is_symlink() or sha(folder / name) != expected: + raise ValueError('reference tokenizer identity differs') + from tokenizers import Tokenizer + from jinja2.sandbox import ImmutableSandboxedEnvironment + self.tokenizer = Tokenizer.from_file(str(folder / 'tokenizer.json')) + env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True) + def invalid_template(message): + raise ValueError('reference chat template rejected messages') + env.globals['raise_exception'] = invalid_template + self.template = env.from_string((folder / 'chat_template.jinja').read_text(encoding='utf-8')) + self.settings = read(folder / 'tokenizer_config.json') + + def __call__(self, messages): + prompt = self.template.render(messages=messages, add_generation_prompt=True, + enable_thinking=False, **{k: self.settings.get(k) + for k in ('bos_token', 'eos_token', 'pad_token')}) + return len(self.tokenizer.encode(prompt, add_special_tokens=False).ids) + + +def credential_from_text(text, heading=CREDENTIAL_HEADING): + lines = text.splitlines() + selected = [] + active = False + matches = 0 + for line in lines: + if re.match(r'^#{1,6}\s', line): + active = line.lstrip('#').strip() == heading + matches += int(active) + elif active: + selected.append(line) + keys = [m.group(1).strip() for line in selected + if (m := re.fullmatch(r'\s*API Key\s*[:：]\s*(\S+)\s*', line))] + if matches != 1 or len(keys) != 1 or len(keys[0]) < 16 or any(c in keys[0] for c in '<>`'): + raise ValueError('one API Key line is required in the dedicated credential section') + return keys[0] + + +def read_credential(): + return credential_from_text(CREDENTIAL_FILE.read_text(encoding='utf-8')) + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise APIError('redirect_refused') + + +def http_post(url, key, body, timeout): + request = urllib.request.Request(url, data=json.dumps(body, ensure_ascii=False).encode('utf-8'), + headers={'Authorization': 'Bearer ' + key, + 'Content-Type': 'application/json'}, method='POST') + try: + with urllib.request.build_opener(NoRedirect()).open(request, timeout=timeout) as response: + data = response.read(8 * 1024 * 1024 + 1) + if len(data) > 8 * 1024 * 1024: + raise APIError('response_too_large') + return json.loads(data) + except urllib.error.HTTPError as exc: + code = exc.code + exc.close() + raise APIError('http_' + str(code)) from None + except APIError: + raise + except Exception: + raise APIError('transport_or_json_error') from None + + +class Budget: + """Reserve worst-case cost durably before every HTTP attempt, with no automatic retries.""" + def __init__(self, path, config, *, clock=time.time): + self.path, self.config, self.clock = Path(path), config, clock + if self.path.exists(): + self.state = read(self.path) + if self.state['config_sha256'] != fingerprint(api_identity(config)): + raise ValueError('API budget belongs to a different profile') + else: + self.state = {'schema': 1, 'config_sha256': fingerprint(api_identity(config)), + 'started_at_unix': clock(), 'attempts': []} + put(self.path, self.state) + if [a['number'] for a in self.state['attempts']] != list(range(1, len(self.state['attempts']) + 1)): + raise ValueError('API budget attempt sequence differs') + + @property + def per_request_upper_cny(self): + p = self.config['pricing_cny_per_million'] + return ((GENERATION['context_cap'] - GENERATION['max_new_tokens']) * p['input'] + + GENERATION['max_new_tokens'] * p['output']) / 1_000_000 + + def check(self, count=1): + limits = self.config['limits'] + if self.clock() - self.state['started_at_unix'] >= limits['max_wall_seconds']: + raise APIBudgetExceeded('wall_budget_exhausted') + if len(self.state['attempts']) + count > limits['max_http_requests']: + raise APIBudgetExceeded('request_budget_exhausted') + cost = sum(a['reserved_cny'] for a in self.state['attempts']) + if cost + count * self.per_request_upper_cny > limits['max_estimated_cny'] + 1e-12: + raise APIBudgetExceeded('estimated_cost_budget_exhausted') + + def reserve(self, body): + self.check() + entry = {'number': len(self.state['attempts']) + 1, 'reserved_cny': self.per_request_upper_cny, + 'request_sha256': fingerprint(body), 'state': 'outcome_unknown', + 'sent_at_unix': self.clock()} + self.state['attempts'].append(entry) + put(self.path, self.state, replace=True) + return entry + + def finish(self, entry, outcome, usage=None): + entry.update(state=outcome, usage=usage) + put(self.path, self.state, replace=True) + + +class APIBackend: + def __init__(self, config, folder, *, key=None, counter=None, transport=http_post, + clock=time.time, sleep=time.sleep, budget=None): + self.config = config + self.counter = counter or ReferenceCounter(config['tokenizer_path']) + self.transport, self.clock, self.sleep = transport, clock, sleep + self.key = key if key is not None else read_credential() + self.budget = budget if budget is not None else Budget(Path(folder) / 'api-budget.json', config, clock=clock) + self.last_generation_metadata = {} + + def before_source(self): + # Leave remaining sources pending instead of starting one without its full call allowance. + self.budget.check(BUDGET['max_planner_attempts'] + BUDGET['max_skill_steps']) + + def __call__(self, messages): + self.last_generation_metadata = {'input_tokens': None, 'output_tokens': None, + 'truncation_at_budget': False, 'error': None} + entry = None + try: + if not isinstance(messages, list) or not messages or any( + set(m) != {'role', 'content'} or m['role'] not in ('system', 'user', 'assistant') + or not isinstance(m['content'], str) for m in messages): + raise APIError('text_message_contract') + tokens = self.counter(messages) + self.last_generation_metadata['reference_input_tokens'] = tokens + if tokens + GENERATION['max_new_tokens'] > GENERATION['context_cap']: + raise APIError('reference_context_budget_exceeded') + body = {'model': MODEL, 'messages': deepcopy(messages), 'temperature': 0., + 'max_tokens': GENERATION['max_new_tokens'], + 'stream': False, 'enable_thinking': False} + attempts = self.budget.state['attempts'] + if attempts: + remaining = (attempts[-1]['sent_at_unix'] + self.config['limits']['min_interval_seconds'] + - self.clock()) + if remaining > 0: + self.sleep(remaining) + reserve_started = time.monotonic() + entry = self.budget.reserve(body) + self.last_generation_metadata['rate_and_budget_wait_seconds'] = time.monotonic() - reserve_started + self.last_generation_metadata['api_attempt'] = entry['number'] + transport_started = time.monotonic() + try: + response = self.transport(self.config['base_url'] + '/chat/completions', self.key, + body, self.config['limits']['request_timeout_seconds']) + finally: + self.last_generation_metadata['transport_seconds'] = time.monotonic() - transport_started + usage = response.get('usage', {}) + if response.get('model') != MODEL or not isinstance(usage, dict): + raise APIError('response_model_or_usage_mismatch') + inp, out = usage.get('prompt_tokens'), usage.get('completion_tokens') + if type(inp) is not int or type(out) is not int or inp < 1 or out < 0: + raise APIError('missing_usage_counts') + self.last_generation_metadata.update(input_tokens=inp, output_tokens=out, + provider_model=response['model'], provider_precision=self.config['provider_precision'], + tokenizer_comparison='local_reference_only_provider_template_not_verified') + for key in ('id', 'system_fingerprint'): + value = response.get(key) + if isinstance(value, str) and re.fullmatch(r'[A-Za-z0-9_.:-]{1,256}', value) and self.key not in value: + self.last_generation_metadata['provider_' + key] = value + choices = response.get('choices') + if not isinstance(choices, list) or len(choices) != 1: + raise APIError('response_choice_contract') + message = choices[0].get('message', {}) + content, reason = message.get('content'), choices[0].get('finish_reason') + self.last_generation_metadata.update(finish_reason=reason, + truncation_at_budget=reason == 'length' or out >= GENERATION['max_new_tokens']) + if inp + GENERATION['max_new_tokens'] > GENERATION['context_cap'] or out > GENERATION['max_new_tokens']: + raise APIError('provider_token_budget_exceeded') + if (not isinstance(content, str) or not content.strip() or reason not in ('stop', 'length') + or message.get('reasoning') or message.get('reasoning_content') or '' in content + or self.key in content): + raise APIError('response_content_or_thinking_contract') + self.budget.finish(entry, 'response_received', {'prompt_tokens': inp, 'completion_tokens': out}) + return content + except Exception as exc: + # The runtime stores str(exc); do not let HTTP errors, headers or response bodies escape. + code = str(exc) if isinstance(exc, APIError) else 'api_contract_error' + self.last_generation_metadata['error'] = code + if entry is not None: + self.budget.finish(entry, code) + raise APIError(code) from None diff --git a/research/vehicle_program_v1/code/api_check.py b/research/vehicle_program_v1/code/api_check.py new file mode 100644 index 0000000..87acf86 --- /dev/null +++ b/research/vehicle_program_v1/code/api_check.py @@ -0,0 +1,69 @@ +"""Offline reference-tokenizer check; --live explicitly performs two synthetic API requests.""" +import argparse +from copy import deepcopy +import json +from pathlib import Path + +from api_backend import APIBackend, ReferenceCounter, api_identity, load_config +from artifacts import GENERATION, put, require_private_output + + +def synthetic_messages(counter): + short = [{'role': 'system', 'content': 'Return only the JSON object {"ok": true}.'}, + {'role': 'user', 'content': 'This is a synthetic connectivity check. Return the requested JSON.'}] + low, high = 1, GENERATION['context_cap'] + target = GENERATION['context_cap'] - GENERATION['max_new_tokens'] - 64 + while low < high: + mid = (low + high + 1) // 2 + probe = [short[0], {'role': 'user', 'content': 'padding ' * mid + '\nReturn the requested JSON.'}] + if counter(probe) <= target: + low = mid + else: + high = mid - 1 + long = [short[0], {'role': 'user', 'content': 'padding ' * low + '\nReturn the requested JSON.'}] + if not 32000 <= counter(long) <= target: + raise ValueError('synthetic boundary prompt did not reach the target context') + return [short, long] + + +def check(config_path, output, *, live=False): + config = load_config(config_path) + counter = ReferenceCounter(config['tokenizer_path']) + prompts = synthetic_messages(counter) + output = Path(output) + require_private_output(output) + output.mkdir(parents=True, exist_ok=False) + report = {'scope': 'synthetic only; no vehicle inputs or gold', 'live': live, + 'service': api_identity(config), 'calls': [], 'http_requests': 0} + put(output / 'check-started.json', {'live': live, 'service': api_identity(config)}) + backend = None + try: + if live: + backend = APIBackend(config, output / 'backend', counter=counter) + for messages in prompts: + item = {'reference_input_tokens': counter(messages), 'capacity_checked_locally': True} + if backend: + text = backend(messages) + item.update(generation=deepcopy(backend.last_generation_metadata), response_json_ok=json.loads(text) == {'ok': True}) + if not item['response_json_ok']: + raise ValueError('synthetic JSON instruction failed') + report['calls'].append(item) + report['passed'] = True + except Exception as exc: + report.update(passed=False, error_type=type(exc).__name__) + finally: + if backend: + report['http_requests'] = len(backend.budget.state['attempts']) + put(output / 'check.json', report) + return report + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--config', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--live', action='store_true') + args = p.parse_args() + result = check(args.config, args.output, live=args.live) + print(json.dumps({'passed': result['passed'], 'live': result['live'], 'http_requests': result['http_requests']})) + raise SystemExit(0 if result['passed'] else 2) diff --git a/research/vehicle_program_v1/code/api_discover.py b/research/vehicle_program_v1/code/api_discover.py new file mode 100644 index 0000000..2119b83 --- /dev/null +++ b/research/vehicle_program_v1/code/api_discover.py @@ -0,0 +1,48 @@ +"""Read the authenticated SiliconFlow model list; no generation or private sample upload.""" +import argparse +import hashlib +import json +from pathlib import Path +import urllib.error +import urllib.request + +from api_backend import APIError, CREDENTIAL_FILE, NoRedirect, credential_from_text +from artifacts import put, require_private_output + + +def discover(output): + output = Path(output) + require_private_output(output) + key = credential_from_text(CREDENTIAL_FILE.read_text(encoding='utf-8'), '汽车 NONT-D SiliconFlow API') + report = {'provider': 'siliconflow', 'base_url': 'https://api.siliconflow.cn/v1', + 'operation': 'GET /models', 'generation_requests': 0, + 'private_samples_sent': 0, 'exact_model': 'Qwen/Qwen3.5-9B'} + request = urllib.request.Request('https://api.siliconflow.cn/v1/models', + headers={'Authorization': 'Bearer ' + key}) + try: + with urllib.request.build_opener(NoRedirect()).open(request, timeout=30) as response: + raw = response.read(8 * 1024 * 1024 + 1) + if len(raw) > 8 * 1024 * 1024: + raise APIError('response_too_large') + body = json.loads(raw) + ids = sorted({r['id'] for r in body['data'] if isinstance(r.get('id'), str)}) + if any(key in ident for ident in ids): + raise APIError('unsafe_model_list_content') + report.update(http_status=200, model_count=len(ids), + public_model_ids_sha256=hashlib.sha256(json.dumps(ids).encode()).hexdigest(), + qwen35_models=[s for s in ids if 'Qwen3.5' in s], + exact_model_listed=report['exact_model'] in ids) + except urllib.error.HTTPError as exc: + report.update(http_status=exc.code, error='http_' + str(exc.code)) + exc.close() + except Exception: + report.update(http_status=None, error='model_list_transport_or_contract_error') + put(output, report) + return report + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--output', type=Path, required=True) + args = p.parse_args() + print(json.dumps(discover(args.output))) diff --git a/research/vehicle_program_v1/code/api_offline_audit.py b/research/vehicle_program_v1/code/api_offline_audit.py new file mode 100644 index 0000000..10b490f --- /dev/null +++ b/research/vehicle_program_v1/code/api_offline_audit.py @@ -0,0 +1,69 @@ +"""Audit API preparation with local tokenizer and scripted Val traces; never opens a network.""" +import argparse +from collections.abc import Mapping +import json +from pathlib import Path +from unittest.mock import patch + +from api_backend import ReferenceCounter, load_config +from api_check import synthetic_messages +from artifacts import load_study, put, sha +from replay import replay +from runner import run + + +def audit(package, output): + package, output = Path(package), Path(output) + config = load_config(package / 'api-profile.json') + study, _ = load_study(package) + counter = ReferenceCounter(config['tokenizer_path']) + prompts = synthetic_messages(counter) + from transformers import AutoTokenizer + tok = AutoTokenizer.from_pretrained(config['tokenizer_path'], local_files_only=True, trust_remote_code=False) + comparisons = [] + for messages in prompts: + encoded = tok.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, enable_thinking=False) + ids = encoded['input_ids'] if isinstance(encoded, Mapping) else encoded + if ids and isinstance(ids[0], list): + ids = ids[0] + comparisons.append({'reference_tokens': counter(messages), 'transformers_tokens': len(ids)}) + if any(v['reference_tokens'] != v['transformers_tokens'] for v in comparisons): + raise ValueError('reference counter differs from official AutoTokenizer') + def no_network(*args, **kwargs): + raise AssertionError('offline audit attempted network access') + original_open = Path.open + def no_inference_gold(path, *args, **kwargs): + if path.name in ('gold.json', '敏感信息交接.md'): + raise AssertionError('offline inference opened gold or credentials') + return original_open(path, *args, **kwargs) + with patch('urllib.request.OpenerDirector.open', no_network), patch.object(Path, 'open', no_inference_gold): + first = run(package, output, max_samples=7) + first_hashes = {str(i): sha(output / f'sources/{i:04d}/receipt.json') for i in range(1, 8)} + finished = run(package, output, resume=True) + hashes = {str(i): sha(output / f'sources/{i:04d}/receipt.json') for i in range(1, 8)} + replayed = replay(package, output) + lengths = [] + for call in sorted((output / 'sources').glob('*/calls/*.json')): + data = json.loads(call.read_text(encoding='utf-8')) + lengths.append(counter(data['messages'])) + report = {'scope': 'offline synthetic transport tests and scripted Val paths; not real model accuracy', + 'http_requests': 0, 'key_read': False, 'gold_opened_by_inference': False, + 'reference_tokenizer_vs_transformers': comparisons, + 'first_batch_committed': first['committed_count'], 'resume_receipts_unchanged': hashes == first_hashes, + 'scripted_run': finished, 'replay': replayed, + 'scripted_path_capacity': {'requests': len(lengths), 'min_input_tokens': min(lengths), + 'max_input_tokens': max(lengths), 'over_32256': sum(n > 32256 for n in lengths), + 'not_exhaustive_for_real_model_routes': True}, + 'expected_count': len(study['rows']), 'quality_claim': False} + if not report['resume_receipts_unchanged'] or finished['pending_count'] or not replayed['pass']: + raise ValueError('offline receipt/replay validation failed') + put(output.parent / 'api-offline-audit.json', report) + return report + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--package', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + print(json.dumps(audit(args.package, args.output))) diff --git a/research/vehicle_program_v1/code/api_parallel.py b/research/vehicle_program_v1/code/api_parallel.py new file mode 100644 index 0000000..8316c2b --- /dev/null +++ b/research/vehicle_program_v1/code/api_parallel.py @@ -0,0 +1,276 @@ +"""Concurrent independent sources with one durable token, request and money ledger.""" +import argparse +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait +from copy import deepcopy +import json +import math +from pathlib import Path +import threading +import time + +from api_backend import (APIBackend, APIBudgetExceeded, APIError, Budget, ReferenceCounter, + api_generation, api_identity, load_config, read_credential) +from artifacts import (BUDGET, fingerprint, load_controller, load_study, put, read, + require_private_output) +from runner import Recorder, environment, exclusive_run, members, read_receipts, write_summary + + +def scheduler_config(): + return {'workers': 3, 'tokens_per_minute': 72000, 'requests_per_minute': 900, + 'token_margin': 128, 'window_seconds': 60., 'window_guard_seconds': .1} + + +def validate_scheduler(config): + if set(config) != set(scheduler_config()): + raise ValueError('scheduler fields differ') + for name, value in config.items(): + if type(value) not in (int, float) or not math.isfinite(value) or value <= 0: + raise ValueError('scheduler limits must be positive and finite') + if name in ('workers', 'tokens_per_minute', 'requests_per_minute', 'token_margin') and type(value) is not int: + raise ValueError('scheduler counts must be integers') + if config['workers'] > 8 or config['window_seconds'] != 60.: + raise ValueError('unsupported scheduler concurrency/window') + + +class SharedBudget(Budget): + """Serialize reservations, settle successful usage, retain uncertain maximum charges. + + TPM uses reference input plus margin and maximum output for the entire window; + it is not refunded after short responses. Unknown outcomes retain full cost. + """ + def __init__(self, path, config, scheduler, counter, *, clock=time.time, sleep=time.sleep, + started_at_unix=None): + validate_scheduler(scheduler) + self.path, self.config, self.scheduler = Path(path), config, scheduler + self.counter, self.clock, self.sleep = counter, clock, sleep + self.lock = threading.RLock() + self.stopped = False + ident = fingerprint({'api': api_identity(config), 'scheduler': scheduler}) + if self.path.exists(): + self.state = read(self.path) + if self.state['config_sha256'] != ident: + raise ValueError('shared API budget profile differs') + else: + self.state = {'schema': 2, 'config_sha256': ident, + 'started_at_unix': clock() if started_at_unix is None else started_at_unix, 'attempts': []} + put(self.path, self.state) + if started_at_unix is not None and self.state['started_at_unix'] != started_at_unix: + raise ValueError('shared API budget wall origin differs') + if [a['number'] for a in self.state['attempts']] != list(range(1, len(self.state['attempts']) + 1)): + raise ValueError('API budget attempt sequence differs') + # An uncommitted HTTP outcome must never be retried by restarting the run. + self.stopped = any(a['state'] != 'response_received' for a in self.state['attempts']) + + def accounted_cny(self): + return sum(a.get('usage_cost_cny', a['reserved_cny']) for a in self.state['attempts']) + + def check(self, count=1): + with self.lock: + if self.stopped: + raise APIBudgetExceeded('shared_backend_stopped') + limits = self.config['limits'] + if self.clock() - self.state['started_at_unix'] >= limits['max_wall_seconds']: + raise APIBudgetExceeded('wall_budget_exhausted') + if len(self.state['attempts']) + count > limits['max_http_requests']: + raise APIBudgetExceeded('request_budget_exhausted') + if self.accounted_cny() + count * self.per_request_upper_cny > limits['max_estimated_cny'] + 1e-12: + raise APIBudgetExceeded('estimated_cost_budget_exhausted') + + def reservation_delay(self, now): + """Additional admission delay, evaluated under the reservation lock each iteration.""" + return 0. + + def reserve(self, body): + with self.lock: + tokens = self.counter(body['messages']) + body['max_tokens'] + self.scheduler['token_margin'] + if tokens > self.scheduler['tokens_per_minute']: + raise APIBudgetExceeded('request_exceeds_tpm_window') + while True: + with self.lock: + self.check() + now = self.clock() + extra_delay = self.reservation_delay(now) + window = self.scheduler['window_seconds'] + self.scheduler['window_guard_seconds'] + recent = [a for a in self.state['attempts'] if now - a['sent_at_unix'] < window] + rate_blocked = (len(recent) >= self.scheduler['requests_per_minute'] or + sum(a['rate_tokens'] for a in recent) + tokens > self.scheduler['tokens_per_minute']) + if extra_delay <= 0 and not rate_blocked: + entry = {'number': len(self.state['attempts']) + 1, + 'reserved_cny': self.per_request_upper_cny, + 'request_sha256': fingerprint(body), 'state': 'outcome_unknown', + 'sent_at_unix': now, 'rate_tokens': tokens} + self.state['attempts'].append(entry) + put(self.path, self.state, replace=True) + return entry + rate_delay = min((a['sent_at_unix'] + window - now for a in recent), default=0.) if rate_blocked else 0. + delay = max(extra_delay, rate_delay) + self.sleep(min(max(delay, .001), .5)) + + def finish(self, entry, outcome, usage=None): + with self.lock: + entry.update(state=outcome, usage=usage, finished_at_unix=self.clock()) + if outcome == 'response_received': + price = self.config['pricing_cny_per_million'] + cost = (usage['prompt_tokens'] * price['input'] + usage['completion_tokens'] * price['output']) / 1e6 + if cost > entry['reserved_cny'] + 1e-12: + self.stopped = True + raise APIError('usage_exceeds_cost_reservation') + entry['usage_cost_cny'] = cost + # A template difference beyond the margin invalidates the local rate estimate. + if usage['prompt_tokens'] + 512 > entry['rate_tokens']: + entry['rate_tokens'] = usage['prompt_tokens'] + 512 + self.stopped = True + entry['state'] = 'provider_token_estimate_exceeded' + put(self.path, self.state, replace=True) + raise APIError('provider_token_estimate_exceeded') + else: + self.stopped = True + put(self.path, self.state, replace=True) + + +def source_receipt(builder, study, row, folder, generator, *, kind='hosted_qwen35_9b_api'): + folder.mkdir(parents=True) + put(folder / 'started.json', {'id': row['id'], 'arm': 'NONT-D'}) + recorded = Recorder(generator, folder / 'calls') + started = time.monotonic() + receipt = {'id': row['id'], 'status': 'runner_error', 'terminated': False, 'stop_reason': 'runner_error'} + try: + controller = builder(recorded, study['registry'], study['rows'], BUDGET, skill_pool='F66') + if len(controller.library) != 66: + raise ValueError('F66 library size differs') + trace = controller.run({'source_id': row['id'], 'domain': row['domain'], + 'field_name': row['metadata']['field_name']}).to_dict() + trace['execution_mode'] = kind + put(folder / 'trace.json', trace) + receipt.update(status='backend_error' if recorded.failed else 'completed', + terminated=trace['terminated'] and not recorded.failed, stop_reason=trace['stop_reason']) + except Exception as exc: + receipt['error_type'] = type(exc).__name__ + receipt.update(seconds=time.monotonic() - started, model_calls=len(recorded.calls), files=members(folder), + input_tokens=sum(c['generation'].get('input_tokens') or 0 for c in recorded.calls), + output_tokens=sum(c['generation'].get('output_tokens') or 0 for c in recorded.calls), + token_counts_measured=bool(recorded.calls) and all( + c['generation'].get('input_tokens') is not None and c['generation'].get('output_tokens') is not None + for c in recorded.calls)) + put(folder / 'receipt.json', receipt) + return receipt + + +def run_parallel(package, output, api_config, scheduler, *, positions=None, resume=False, + counter=None, key=None, transport=None): + validate_scheduler(scheduler) + study, manifest = load_study(package) + ids = study['expected_ids'] + selected = positions if positions is not None else list(range(1, len(ids) + 1)) + if (not selected or any(type(n) is not int or n < 1 or n > len(ids) for n in selected) + or len(set(selected)) != len(selected) or selected != sorted(selected)): + raise ValueError('selected positions must be ordered, unique and within study') + config = load_config(api_config) + if config['limits']['min_interval_seconds'] > .001: + raise ValueError('parallel scheduler replaces fixed request intervals') + output = Path(output) + require_private_output(output) + kind = 'hosted_qwen35_9b_api' + identity = {'schema': 2, 'arm': 'NONT-D', 'backend': kind, + 'study_sha256': manifest['study_sha256'], 'code_sha256': fingerprint(manifest['code']), + 'model_manifest_sha256': None, 'generation': api_generation(), 'budget': BUDGET, + 'expected_ids': ids, 'environment': environment(), 'api_service': api_identity(config), + 'scheduler': scheduler, 'selected_positions': selected} + if output.exists() and not resume: + raise FileExistsError('run exists; explicitly resume same identity') + output.mkdir(parents=True, exist_ok=resume) + with exclusive_run(output): + if (output / 'run.json').exists(): + if read(output / 'run.json') != identity: + raise ValueError('run identity differs') + else: + if set(p.name for p in output.iterdir()) != {'.run.lock'}: + raise ValueError('unidentified run artifacts') + put(output / 'run.json', identity) + receipts = read_receipts(output, ids) + todo = [] + for n in selected: + row, folder = study['rows'][n - 1], output / 'sources' / f'{n:04d}' + if row['id'] in receipts: + continue + if folder.exists(): + receipt = {'id': row['id'], 'status': 'interrupted', 'terminated': False, + 'stop_reason': 'interrupted_before_commit', 'seconds': None, + 'model_calls': len(list((folder / 'calls').glob('*.json'))), 'files': members(folder)} + put(folder / 'receipt.json', receipt) + receipts[row['id']] = receipt + else: + todo.append((row, folder)) + halted, budget = None, None + if todo and any(r['status'] != 'completed' for r in receipts.values()): + halted = 'existing_source_error' + elif todo: + if not (output / 'backend/api-budget.json').exists() and any((output / 'sources').glob('*/calls/*.json')): + raise ValueError('API budget missing for existing requests') + counter = counter or ReferenceCounter(config['tokenizer_path']) + key = key if key is not None else read_credential() + budget = SharedBudget(output / 'backend/api-budget.json', config, scheduler, counter) + builder = load_controller() + def work(row, folder): + kwargs = {'transport': transport} if transport is not None else {} + generator = APIBackend(config, output / 'backend', key=key, counter=counter, budget=budget, **kwargs) + generator.before_source() + return source_receipt(builder, study, row, folder, generator) + pending = iter(todo) + with ThreadPoolExecutor(max_workers=scheduler['workers']) as pool: + active = {} + def submit(): + item = next(pending, None) + if item is not None: + active[pool.submit(work, *item)] = item[0]['id'] + for _ in range(min(scheduler['workers'], len(todo))): + submit() + while active: + done, _ = wait(active, return_when=FIRST_COMPLETED) + for future in done: + ident = active.pop(future) + try: + receipt = future.result() + receipts[ident] = receipt + if receipt['status'] != 'completed': + halted = receipt['status'] + except Exception as exc: + halted = type(exc).__name__ + # Windows may reject replacing an open ledger. Its readers must + # share the writer lock, including progress summaries. + with budget.lock: + write_summary(output, ids, receipts, kind, halted=halted) + if not halted: + for _ in done: + submit() + else: + with budget.lock: + budget.stopped = True + if budget is not None: + with budget.lock: + result = write_summary(output, ids, receipts, kind, halted=halted) + else: + result = write_summary(output, ids, receipts, kind, halted=halted) + path = output / 'backend/api-budget.json' + attempts = read(path)['attempts'] if path.exists() else [] + result.update(selected_count=len(selected), selected_committed_count=sum(ids[n - 1] in receipts for n in selected), + usage_estimate_plus_uncertain_reserves_cny=sum(a.get('usage_cost_cny', a['reserved_cny']) for a in attempts), + selected_complete=all(ids[n - 1] in receipts for n in selected)) + put(output / 'summary.json', result, replace=True) + return result + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--package', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--api-config', type=Path, required=True) + p.add_argument('--scheduler', type=Path, required=True) + p.add_argument('--start-position', type=int, default=1) + p.add_argument('--count', type=int) + p.add_argument('--resume', action='store_true') + a = p.parse_args() + positions = list(range(a.start_position, a.start_position + a.count)) if a.count is not None else None + result = run_parallel(a.package, a.output, a.api_config, read(a.scheduler), positions=positions, resume=a.resume) + print(json.dumps(result)) + raise SystemExit(0 if result['selected_complete'] and not result['halted'] else 2) diff --git a/research/vehicle_program_v1/code/api_prepare.py b/research/vehicle_program_v1/code/api_prepare.py new file mode 100644 index 0000000..301e8b2 --- /dev/null +++ b/research/vehicle_program_v1/code/api_prepare.py @@ -0,0 +1,31 @@ +"""Prepare a new immutable vehicle package and key-free API profile, without network access.""" +import argparse +from pathlib import Path +import json + +from api_backend import ReferenceCounter, default_config +from artifacts import put +from prepare import prepare + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--source', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--tokenizer', type=Path, required=True) + args = p.parse_args() + counter = ReferenceCounter(args.tokenizer) + report = prepare(args.source, args.output) + put(args.output / 'api-profile.json', default_config(args.tokenizer)) + put(args.output / 'api-preparation.json', { + 'offline_only': True, 'http_requests': 0, 'key_read': False, + 'reference_tokenizer_verified': True, 'synthetic_reference_tokens': counter([ + {'role': 'system', 'content': 'Return only JSON.'}, + {'role': 'user', 'content': 'Return {"ok": true}.'}]), + 'hosted_precision': 'not_verified', 'hosted_weight_hash_verified': False, + 'full_validation_authorized_by_this_script': False}) + print(json.dumps(report, ensure_ascii=False)) + + +if __name__ == '__main__': + main() diff --git a/research/vehicle_program_v1/code/artifacts.py b/research/vehicle_program_v1/code/artifacts.py new file mode 100644 index 0000000..55a50bd --- /dev/null +++ b/research/vehicle_program_v1/code/artifacts.py @@ -0,0 +1,109 @@ +"""Local immutable artifacts; no model, network or label access on import.""" +import hashlib +import json +import os +from pathlib import Path +import time + +ROOT = Path(__file__).resolve().parent +REVISION = '84a9a462dcb828332f980ef121f40e1281ce1794' +TEAM_COMMIT = '9473b2c2a330b7f48dc0c1831bda11e36d428379' +MODEL_REVISION = 'c202236235762e1c871ad0ccb60c8ee5ba337b9a' +BUDGET = dict(max_programs=3, max_program_steps=10, max_skill_steps=12, + max_planner_attempts=4, max_identical_calls=1) +GENERATION = dict(seed=20260928, context_cap=32768, max_new_tokens=512, + enable_thinking=False, do_sample=False, dtype='float16') + + +def read(path): + return json.loads(Path(path).read_text(encoding='utf-8')) + + +def sha(path): + h = hashlib.sha256() + with Path(path).open('rb') as f: + for block in iter(lambda: f.read(1024 * 1024), b''): + h.update(block) + return h.hexdigest() + + +def fingerprint(value): + return hashlib.sha256(json.dumps(value, ensure_ascii=False, sort_keys=True, + allow_nan=False).encode('utf-8')).hexdigest() + + +def require_private_output(path): + resolved = Path(path).resolve() + roots = [ROOT] + [parent for parent in ROOT.parents if (parent / '.git').exists()] + if any(resolved.is_relative_to(root) for root in roots): + raise ValueError('private data/artifacts must stay outside the code and Git repository') + + +def put(path, value, *, replace=False): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists() and not replace: + raise FileExistsError(path) + temporary = path.with_name(path.name + '.tmp') + with temporary.open('x', encoding='utf-8') as f: + json.dump(value, f, ensure_ascii=False, indent=2, allow_nan=False) + f.write('\n') + f.flush() + os.fsync(f.fileno()) + # A brief status reader on Windows can deny replace. Retry the file commit, + # never the HTTP request that produced the evidence. + for attempt in range(101): + try: + temporary.replace(path) + break + except PermissionError: + if attempt == 100: + raise + time.sleep(.01) + + +def code_inventory(): + return {p.relative_to(ROOT).as_posix(): sha(p) for p in sorted(ROOT.rglob('*')) + if p.is_file() and p.suffix in ('.py', '.json', '.md', '.ps1') + and '__pycache__' not in p.parts} + + +def verify_vendor(): + source = read(ROOT / 'upstream.json') + if source['commit'] != TEAM_COMMIT: + raise ValueError('team commit differs') + actual = {p.relative_to(ROOT).as_posix(): sha(p) + for p in sorted((ROOT / 'upstream').rglob('*')) if p.is_file() + and '__pycache__' not in p.parts} + if actual != source['files']: + raise ValueError('vendored runtime changed') + + +def load_study(package): + package = Path(package) + manifest = read(package / 'manifest.json') + if manifest['code'] != code_inventory(): + raise ValueError('experiment code changed; prepare a new package') + verify_vendor() + if sha(package / 'inference/study.json') != manifest['study_sha256']: + raise ValueError('inference study identity changed') + study = read(package / 'inference/study.json') + if study['budget'] != BUDGET or study['skill_pool'] != 'F66' or study['generation'] != GENERATION: + raise ValueError('frozen F66 budget differs') + ids = [r['id'] for r in study['rows']] + if ids != study['expected_ids'] or len(ids) != len(set(ids)) or not ids: + raise ValueError('study identity/order differs') + for row in study['rows']: + if set(row) != {'id', 'domain', 'metadata'} or not set(row['metadata']) <= {'field_name', 'field_description'}: + raise ValueError('study metadata allowlist violated') + return study, manifest + + +def load_controller(): + import sys + runtime = ROOT / 'upstream/runtime' + sys.path.insert(0, str(runtime)) + import program_loop + if not Path(program_loop.__file__).resolve().is_relative_to(runtime): + raise ValueError('a different program_loop is already imported') + return program_loop.build_program_controller diff --git a/research/vehicle_program_v1/code/backends.py b/research/vehicle_program_v1/code/backends.py new file mode 100644 index 0000000..cf63ff4 --- /dev/null +++ b/research/vehicle_program_v1/code/backends.py @@ -0,0 +1,40 @@ +"""Scripted CPU fixture and opt-in offline local base model; no adapters.""" +import json +from pathlib import Path +import sys +from artifacts import ROOT, MODEL_REVISION, read + + +class ScriptedBackend: + kind = 'scripted_cpu_test' + + def __call__(self, messages): + head = messages[0]['content'].splitlines()[0] + user = json.loads(messages[-1]['content']) + if head.startswith('You are the Global Program Planner'): + names = ['GET_FIELD_METADATA', 'INTERPRET_FIELD_NAME', 'SELECT_CATEGORY_TOPK', + 'INFER_CATEGORY_FROM_TOPK', 'INFER_GRADING_LEVEL', 'FINALIZE_CLASSIFICATION'] + return json.dumps({'program': [{'tool': n, 'arguments': {}} for n in names]}) + skill = head.removeprefix('Execute only the selected Skill ').removesuffix('.') + if skill == 'INTERPRET_FIELD_NAME': + ctx = user['grounding_context'] + desc = ctx.get('field_description') + status = 'observed_description' if desc else 'unknown' + return json.dumps({'summary': status, 'data': {'field_semantics': { + 'schema': 'grounded-source-v1', 'status': status, + 'field_name': ctx['field_name'], 'description': desc, 'source_consistency': 'not_adjudicated'}}}) + if skill == 'SELECT_CATEGORY_TOPK': + return json.dumps({'option_ids': [r['option_id'] for r in user['category_options'][:5]], 'summary': 'scripted'}) + if skill == 'INFER_CATEGORY_FROM_TOPK': + return json.dumps({'option_id': user['candidate_details'][0]['option_id'], 'summary': 'scripted'}) + raise ValueError('unexpected scripted role') + + +def local_backend(model, folder): + """The unchanged team FP16 greedy backend loads only an already verified local model.""" + manifest = read(Path(model) / 'model-manifest.json') + if manifest['repo_id'] != 'Qwen/Qwen3.5-9B' or manifest['revision'] != MODEL_REVISION: + raise ValueError('base model revision differs') + sys.path.insert(0, str(ROOT / 'upstream/backend')) + from local_smoke import Backend + return Backend(model, folder) diff --git a/research/vehicle_program_v1/code/bundle.py b/research/vehicle_program_v1/code/bundle.py new file mode 100644 index 0000000..83597b9 --- /dev/null +++ b/research/vehicle_program_v1/code/bundle.py @@ -0,0 +1,28 @@ +"""Build a controlled inference-only archive; evaluator gold stays local.""" +import argparse +from pathlib import Path +import zipfile +from artifacts import ROOT, code_inventory, load_study, require_private_output, sha + + +def bundle(package, output): + _, manifest = load_study(package) + package, output = Path(package), Path(output) + require_private_output(output) + output.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(output, 'x', compression=zipfile.ZIP_DEFLATED) as archive: + for name in sorted(code_inventory()): + archive.write(ROOT / name, 'code/' + name) + for name in ('manifest.json', 'inference/study.json'): + archive.write(package / name, 'package/' + name) + return {'archive_sha256': sha(output), 'bytes': output.stat().st_size, + 'study_sha256': manifest['study_sha256'], 'evaluator_gold_included': False} + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--package', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + a = p.parse_args() + import json + print(json.dumps(bundle(a.package, a.output))) diff --git a/research/vehicle_program_v1/code/night_ops.py b/research/vehicle_program_v1/code/night_ops.py new file mode 100644 index 0000000..ba53be2 --- /dev/null +++ b/research/vehicle_program_v1/code/night_ops.py @@ -0,0 +1,83 @@ +"""Offline preflight, compact progress, graceful stop and explicit resume checks.""" +import argparse +import json +from pathlib import Path +import time + +from api_backend import CREDENTIAL_FILE, ReferenceCounter, load_config +from artifacts import read +from night_runner import public_status, validate_plan, TRANSIENT +from night_supervisor import on_ac_power +from night_resume import validate_resume_ledger + + +def supervisor_alive(plan): + import psutil + p = Path(plan['home']) / 'supervisor.json' + if not p.exists(): + return False + meta = read(p) + try: + process = psutil.Process(meta['supervisor_pid']) + return (abs(process.create_time() - meta['supervisor_created_unix']) < .01 + and str(Path(__file__).with_name('night_supervisor.py')) in process.cmdline()) + except (psutil.NoSuchProcess, psutil.AccessDenied): + return False + + +def preflight(plan, *, resume=False): + study, manifest, seed = validate_plan(plan) + if supervisor_alive(plan): + raise RuntimeError('night_already_running') + output = Path(plan['output']) + if output.exists() and not resume: + raise RuntimeError('existing_run_use_explicit_resume') + if resume: + start = validate_resume_ledger(plan, study, seed, load_config(Path(plan['package']) / 'night-api-profile.json')) + wall = min(plan['supervisor_wall_seconds'], read(Path(plan['package']) / 'night-api-profile.json')['limits']['max_wall_seconds']) + if time.time() >= start + wall: + raise RuntimeError('wall_budget_exhausted') + if plan['mode'] == 'api': + if not CREDENTIAL_FILE.is_file(): + raise RuntimeError('credential_source_missing') + if not on_ac_power(): + raise RuntimeError('ac_power_required') + ReferenceCounter(read(Path(plan['package']) / 'night-api-profile.json')['tokenizer_path']) + return {'ready': True, 'mode': plan['mode'], 'expected_sources': len(study['rows']), 'preserved_sources': len(seed), + 'pending_sources_before_start': len(study['rows']) - len(seed), 'network_requests': 0, + 'credential_value_read': False, 'scheduled_trigger': None} + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--plan', type=Path, required=True) + p.add_argument('--action', choices=('check', 'status', 'stop', 'resume-check'), required=True) + a = p.parse_args(); plan = read(a.plan) + if a.action in ('check', 'resume-check'): + result = preflight(plan, resume=a.action == 'resume-check') + if a.action == 'resume-check': + stop = Path(plan['stop_file']) + if stop.exists(): + stop.rename(stop.with_name('STOP.' + str(time.time_ns()))) + elif a.action == 'stop': + stop = Path(plan['stop_file']); stop.parent.mkdir(parents=True, exist_ok=True); stop.touch(exist_ok=True) + result = {'stop_requested': True, 'force_kill': False, 'inflight_requests': 'allowed to finish or reach timeout'} + else: + result = public_status(plan['output']) + result['supervisor_alive'] = supervisor_alive(plan) + p = Path(plan['home']) / 'supervisor.json' + if p.exists(): + result['supervisor'] = read(p) + failure = Path(plan['home']) / 'supervisor-failure.json' + if failure.exists(): + result['supervisor_failure'] = read(failure) + print(json.dumps(result)) + + +if __name__ == '__main__': + try: + main() + except Exception as exc: + print(json.dumps({'ready': False, 'error_type': type(exc).__name__, + 'reason': str(exc) if isinstance(exc, RuntimeError) else 'validation_failed'})) + raise SystemExit(2) diff --git a/research/vehicle_program_v1/code/night_prepare.py b/research/vehicle_program_v1/code/night_prepare.py new file mode 100644 index 0000000..ae260e3 --- /dev/null +++ b/research/vehicle_program_v1/code/night_prepare.py @@ -0,0 +1,89 @@ +"""Prepare an offline night plan, preserving all pilot receipts and their charges.""" +import argparse +from copy import deepcopy +import json +import os +from pathlib import Path +import shutil +import sys + +from api_backend import api_generation, default_config, MODEL +from api_parallel import scheduler_config +from artifacts import BUDGET, code_inventory, fingerprint, put, read, sha +from prepare import prepare +from runner import read_receipts + + +def prepare_night(source, package, home, donors, *, mode='api'): + package, home = Path(package), Path(home) + prepare(source, package) + study = read(package / 'inference/study.json') + provenance = {'schema': 1, 'receipts': {}, 'donors': [], 'usage_plus_uncertain_reserves_cny': 0.} + for donor_package, donor_run in donors: + donor_package, donor_run = Path(donor_package), Path(donor_run) + manifest = read(donor_package / 'manifest.json') + identity = read(donor_run / 'run.json') + if (identity['expected_ids'] != study['expected_ids'] or identity['budget'] != BUDGET + or identity['study_sha256'] != sha(package / 'inference/study.json') + or identity['generation'] != api_generation() + or identity['code_sha256'] != fingerprint(manifest['code']) + or identity['api_service']['model'] != MODEL + or identity['api_service']['base_url'] != 'https://api.siliconflow.cn/v1'): + raise ValueError('pilot_identity_differs') + if {k: v for k, v in manifest['code'].items() if k.startswith('upstream/')} != { + k: v for k, v in code_inventory().items() if k.startswith('upstream/')}: + raise ValueError('pilot_controller_changed') + receipts = read_receipts(donor_run, study['expected_ids']) + if set(receipts) & set(provenance['receipts']): + raise ValueError('duplicate_pilot_source') + attempts = read(donor_run / 'backend/api-budget.json')['attempts'] + price = identity['api_service']['pricing_cny_per_million'] + carried = sum((a['usage']['prompt_tokens'] * price['input'] + a['usage']['completion_tokens'] * price['output']) / 1e6 + if a['state'] == 'response_received' else a['reserved_cny'] for a in attempts) + for n, ident in enumerate(study['expected_ids'], 1): + if ident in receipts: + src = donor_run / f'sources/{n:04d}' + shutil.copytree(src, package / f'seed/sources/{n:04d}') + provenance['receipts'][ident] = {'position': n, 'receipt_sha256': sha(src / 'receipt.json'), + 'origin_run_sha256': sha(donor_run / 'run.json'), + 'status': receipts[ident]['status']} + provenance['donors'].append({'package': str(donor_package), 'run': str(donor_run), + 'manifest_sha256': sha(donor_package / 'manifest.json'), + 'run_sha256': sha(donor_run / 'run.json'), + 'api_budget_sha256': sha(donor_run / 'backend/api-budget.json'), + 'source_count': len(receipts), 'http_attempts': len(attempts), + 'carried_usage_plus_uncertain_reserves_cny': carried}) + provenance['usage_plus_uncertain_reserves_cny'] += carried + put(package / 'seed-provenance.json', provenance) + tokenizer = Path(os.environ.get('VEHICLE_NONTD_TOKENIZER', str(Path.home() / '.cache/vehicle-nontd/official-tokenizer'))) + config = default_config(tokenizer) + config['limits'].update(min_interval_seconds=.000001, max_http_requests=(len(study['rows']) - len(provenance['receipts'])) * 16, + max_estimated_cny=70., max_wall_seconds=50400., request_timeout_seconds=180.) + put(package / 'night-api-profile.json', config) + scheduler = scheduler_config(); scheduler['workers'] = 2 + home.mkdir(parents=True, exist_ok=False) + python = Path(sys.executable) + plan = {'schema': 'vehicle-nontd-night-v1', 'mode': mode, 'package': str(package.resolve()), + 'package_manifest_sha256': sha(package / 'manifest.json'), + 'home': str(home.resolve()), 'output': str((home / 'run').resolve()), 'stop_file': str((home / 'STOP').resolve()), + 'files': {name: sha(package / name) for name in ('night-api-profile.json', 'seed-provenance.json')}, + 'python': str(python), 'pythonw': str(python.with_name('pythonw.exe')), 'scheduler': scheduler, + 'supervisor_wall_seconds': 50400., 'manual_start_only': True, 'scheduled_trigger': None, + 'task_name': 'VeRL-NONTD-Night-' + fingerprint(str(home.resolve()))[:10], + 'policy': {'heartbeat_seconds': 5., 'transient_cooldown_seconds': 30., + 'rate_limit_cooldown_seconds': 120., 'consecutive_source_error_limit': 3, + 'shutdown_grace_seconds': 210.}} + put(home / 'plan.json', plan) + return {'prepared': True, 'model_calls': 0, 'preserved_sources': len(provenance['receipts']), + 'pending_sources': len(study['rows']) - len(provenance['receipts']), + 'carried_usage_plus_uncertain_reserves_cny': provenance['usage_plus_uncertain_reserves_cny']} + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--source', type=Path, required=True) + p.add_argument('--package', type=Path, required=True) + p.add_argument('--home', type=Path, required=True) + p.add_argument('--donors', type=Path, required=True) + a = p.parse_args() + print(json.dumps(prepare_night(a.source, a.package, a.home, read(a.donors)['donors']))) diff --git a/research/vehicle_program_v1/code/night_resume.py b/research/vehicle_program_v1/code/night_resume.py new file mode 100644 index 0000000..cacd407 --- /dev/null +++ b/research/vehicle_program_v1/code/night_resume.py @@ -0,0 +1,111 @@ +"""Fail-closed reconciliation of night HTTP evidence before any resumed network access.""" +import math +from pathlib import Path + +from api_backend import MODEL, api_identity +from artifacts import GENERATION, fingerprint, read, sha +from runner import environment, read_receipts + +TRANSIENT = {'transport_or_json_error', 'http_408', 'http_429', 'http_500', 'http_502', 'http_503', 'http_504'} + + +def finite_number(value): + return type(value) in (int, float) and math.isfinite(value) + + +def effective_cooldown(ledger, policy): + """Conservatively reconstruct admission delay, including failures drained after stop.""" + deadline = ledger.get('cooldown_until_unix', 0.) + if not finite_number(deadline) or deadline < 0: + raise RuntimeError('resume_cooldown_invalid') + for attempt in ledger['attempts']: + if attempt['state'] in TRANSIENT: + finished = attempt.get('finished_at_unix') + if not finite_number(finished) or finished <= 0: + raise RuntimeError('resume_reservation_invalid') + delay = policy['rate_limit_cooldown_seconds'] if attempt['state'] == 'http_429' else policy['transient_cooldown_seconds'] + deadline = max(deadline, finished + delay) + return deadline + + +def validate_resume_ledger(plan, study, seed, config): + output = Path(plan['output']) + path = output / 'backend/api-budget.json' + if not path.is_file(): + raise RuntimeError('resume_budget_missing_requires_review') + ledger = read(path) + attempts = ledger['attempts'] + if any(a['state'] not in TRANSIENT | {'response_received'} for a in attempts): + raise RuntimeError('uncertain_or_fatal_attempt_requires_review') + if not (output / 'run.json').is_file(): + raise RuntimeError('resume_run_identity_missing') + identity = read(output / 'run.json') + manifest = read(Path(plan['package']) / 'manifest.json') + start = identity.get('wall_started_at_unix') + if (identity.get('schema') != 4 or not finite_number(start) or start <= 0 + or ledger.get('started_at_unix') != start + or identity.get('night_plan_sha256') != fingerprint(plan) + or identity.get('study_sha256') != manifest['study_sha256'] + or identity.get('code_sha256') != fingerprint(manifest['code']) + or identity.get('expected_ids') != study['expected_ids'] + or identity.get('environment') != environment() + or identity.get('api_service') != api_identity(config) + or ledger.get('config_sha256') != fingerprint({'api': api_identity(config), 'scheduler': plan['scheduler']})): + raise RuntimeError('resume_budget_identity_or_wall_origin_differs') + if [a['number'] for a in attempts] != list(range(1, len(attempts) + 1)): + raise RuntimeError('resume_budget_sequence_differs') + effective_cooldown(ledger, plan['policy']) + price = config['pricing_cny_per_million'] + upper = ((GENERATION['context_cap'] - GENERATION['max_new_tokens']) * price['input'] + + GENERATION['max_new_tokens'] * price['output']) / 1e6 + receipts = read_receipts(output, study['expected_ids']) + seen = set() + for position, ident in enumerate(study['expected_ids'], 1): + folder = output / f'sources/{position:04d}' + if ident in seed: + if ident not in receipts or sha(folder / 'receipt.json') != sha(Path(plan['package']) / f'seed/sources/{position:04d}/receipt.json'): + raise RuntimeError('resume_seed_receipt_differs') + continue + if not folder.exists(): + continue + if ident not in receipts or receipts[ident]['status'] == 'interrupted': + raise RuntimeError('interrupted_source_requires_review') + for call_path in sorted((folder / 'calls').glob('*.json')): + call = read(call_path) + generation = call['generation'] + number = generation.get('api_attempt') + if number is None: + if call['error_type'] is None or call['raw_response'] is not None: + raise RuntimeError('resume_call_missing_attempt') + continue # A local pre-send rejection incurred no HTTP charge. + if type(number) is not int or number < 1 or number > len(attempts) or number in seen: + raise RuntimeError('resume_call_attempt_missing_or_duplicate') + seen.add(number) + entry = attempts[number - 1] + body = {'model': MODEL, 'messages': call['messages'], 'temperature': 0., + 'max_tokens': GENERATION['max_new_tokens'], 'stream': False, 'enable_thinking': False} + outcome = generation.get('error') if call['error_type'] else 'response_received' + if entry['request_sha256'] != fingerprint(body) or entry['state'] != outcome: + raise RuntimeError('resume_request_or_outcome_differs') + if (not finite_number(entry['reserved_cny']) or abs(entry['reserved_cny'] - upper) > 1e-12 + or not finite_number(entry['sent_at_unix']) or entry['sent_at_unix'] < start + or not finite_number(entry.get('finished_at_unix')) or entry['finished_at_unix'] < entry['sent_at_unix'] + or type(entry.get('rate_tokens')) is not int or entry['rate_tokens'] <= 0): + raise RuntimeError('resume_reservation_invalid') + if outcome == 'response_received': + inp, out = generation.get('input_tokens'), generation.get('output_tokens') + if (type(inp) is not int or type(out) is not int or inp < 1 or out < 0 + or inp + GENERATION['max_new_tokens'] > GENERATION['context_cap'] + or out > GENERATION['max_new_tokens']): + raise RuntimeError('resume_usage_invalid') + usage = {'prompt_tokens': inp, 'completion_tokens': out} + cost = (inp * price['input'] + out * price['output']) / 1e6 + saved_cost = entry.get('usage_cost_cny') + if (entry.get('usage') != usage or not finite_number(saved_cost) + or abs(saved_cost - cost) > 1e-12 or cost > upper + 1e-12): + raise RuntimeError('resume_usage_or_cost_differs') + elif 'usage_cost_cny' in entry: + raise RuntimeError('resume_uncertain_charge_released') + if seen != set(range(1, len(attempts) + 1)): + raise RuntimeError('resume_unmatched_budget_attempt_requires_review') + return start diff --git a/research/vehicle_program_v1/code/night_runner.py b/research/vehicle_program_v1/code/night_runner.py new file mode 100644 index 0000000..00fe9ee --- /dev/null +++ b/research/vehicle_program_v1/code/night_runner.py @@ -0,0 +1,248 @@ +"""Unattended, bounded evaluation; preserve old failures and never reroll requests.""" +import argparse +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait +from copy import deepcopy +import json +from pathlib import Path +import shutil +import threading +import time + +from api_backend import APIBackend, APIBudgetExceeded, APIError, ReferenceCounter, api_generation, api_identity, load_config, read_credential +from api_parallel import SharedBudget, source_receipt, validate_scheduler +from artifacts import BUDGET, fingerprint, load_controller, load_study, put, read, sha, require_private_output +from backends import ScriptedBackend +from runner import environment, exclusive_run, members, read_receipts, write_summary +from night_resume import TRANSIENT, effective_cooldown, validate_resume_ledger + + +class NightBudget(SharedBudget): + def __init__(self, path, config, scheduler, counter, *, policy, stop_file, carry_cny=0., **kwargs): + self.policy, self.stop_file, self.carry_cny = policy, Path(stop_file), carry_cny + self.cooldown_until = 0. + super().__init__(path, config, scheduler, counter, **kwargs) + self.cooldown_until = effective_cooldown(self.state, policy) + # Known failed requests stay charged and are not retried. Only fresh sources continue. + self.stopped = any(a['state'] not in TRANSIENT | {'response_received'} for a in self.state['attempts']) + + def accounted_cny(self): + return self.carry_cny + super().accounted_cny() + + def check(self, count=1): + if self.stop_file.exists(): + raise APIBudgetExceeded('user_stop_requested') + super().check(count) + + def reservation_delay(self, now): + return max(0., self.cooldown_until - now) + + def finish(self, entry, outcome, usage=None): + with self.lock: + was_stopped = self.stopped + if outcome in TRANSIENT and not was_stopped: + delay = self.policy['rate_limit_cooldown_seconds'] if outcome == 'http_429' else self.policy['transient_cooldown_seconds'] + self.cooldown_until = max(self.cooldown_until, self.clock() + delay) + self.state['cooldown_until_unix'] = self.cooldown_until + super().finish(entry, outcome, usage) + if outcome in TRANSIENT and not was_stopped: + self.stopped = False + + +def validate_plan(plan): + if plan.get('schema') != 'vehicle-nontd-night-v1' or plan.get('mode') not in ('api', 'scripted_test'): + raise ValueError('night_plan_schema') + package = Path(plan['package']) + home = Path(plan['home']).resolve() + require_private_output(home) + if Path(plan['output']).resolve() != home / 'run' or Path(plan['stop_file']).resolve() != home / 'STOP': + raise ValueError('night_output_or_stop_path_differs') + study, manifest = load_study(package) + for name, expected in plan['files'].items(): + if sha(package / name) != expected: + raise ValueError('night_package_file_changed') + if sha(package / 'manifest.json') != plan['package_manifest_sha256']: + raise ValueError('night_package_manifest_changed') + validate_scheduler(plan['scheduler']) + provenance = read(package / 'seed-provenance.json') + seed = read_receipts(package / 'seed', study['expected_ids']) + if set(seed) != set(provenance['receipts']): + raise ValueError('night_seed_members_changed') + for n, ident in enumerate(study['expected_ids'], 1): + if ident in seed and sha(package / f'seed/sources/{n:04d}/receipt.json') != provenance['receipts'][ident]['receipt_sha256']: + raise ValueError('night_seed_receipt_changed') + return study, manifest, seed + + +def public_status(output): + """Small status file only; never prompts, response content, IDs or credentials.""" + p = Path(output) / 'night-status.json' + return read(p) if p.exists() else {'state': 'not_started'} + + +def run_night(plan, *, resume=False, transport=None, counter=None, key=None): + study, manifest, seed = validate_plan(plan) + ids, output = study['expected_ids'], Path(plan['output']) + config = load_config(Path(plan['package']) / 'night-api-profile.json') + carry = read(Path(plan['package']) / 'seed-provenance.json')['usage_plus_uncertain_reserves_cny'] + kind = 'hosted_qwen35_9b_api' if plan['mode'] == 'api' else 'scripted_cpu_test' + identity = {'schema': 4, 'arm': 'NONT-D', 'backend': kind, 'study_sha256': manifest['study_sha256'], + 'code_sha256': fingerprint(manifest['code']), 'expected_ids': ids, 'generation': api_generation(), + 'budget': BUDGET, 'environment': environment(), 'api_service': api_identity(config), + 'night_plan_sha256': fingerprint(plan), 'scheduler': plan['scheduler'], + 'seed_provenance_sha256': sha(Path(plan['package']) / 'seed-provenance.json'), + 'carried_usage_plus_uncertain_reserves_cny': carry, + 'comparison_scope': 'preserved pilot prefix plus night continuation; scheduler/service profiles disclosed separately'} + if output.exists() and not resume: + raise FileExistsError('night_output_exists') + if resume and not (output / 'run.json').is_file(): + raise RuntimeError('resume_run_identity_missing') + output.mkdir(parents=True, exist_ok=resume) + with exclusive_run(output): + if (output / 'run.json').exists(): + identity['wall_started_at_unix'] = validate_resume_ledger(plan, study, seed, config) + if read(output / 'run.json') != identity: + raise ValueError('night_run_identity_changed') + else: + if set(p.name for p in output.iterdir()) != {'.run.lock'}: + raise ValueError('night_output_unidentified') + identity['wall_started_at_unix'] = time.time() + put(output / 'run.json', identity) + for n, ident in enumerate(ids, 1): + if ident in seed: + shutil.copytree(Path(plan['package']) / f'seed/sources/{n:04d}', output / f'sources/{n:04d}') + receipts = read_receipts(output, ids) + halted = None + for n, ident in enumerate(ids, 1): + folder = output / f'sources/{n:04d}' + if ident not in receipts and folder.exists(): + # Preserve the partial evidence; manual review is required, never reroll it. + r = {'id': ident, 'status': 'interrupted', 'terminated': False, 'stop_reason': 'interrupted_before_commit', + 'seconds': None, 'model_calls': len(list((folder / 'calls').glob('*.json'))), 'files': members(folder)} + put(folder / 'receipt.json', r); receipts[ident] = r + halted = 'interrupted_source_requires_review' + if any(r['status'] == 'interrupted' for r in receipts.values()): + halted = 'interrupted_source_requires_review' + remaining = [(row, output / f'sources/{n:04d}') for n, row in enumerate(study['rows'], 1) if row['id'] not in receipts] + config_folder = output / 'backend' + if remaining and not halted: + counter = counter or (ReferenceCounter(config['tokenizer_path']) if plan['mode'] == 'api' else lambda messages: 100) + key = key if key is not None else (read_credential() if plan['mode'] == 'api' else 'fictional-night-test-only') + else: + counter = counter or (lambda messages: 100) + budget = NightBudget(config_folder / 'api-budget.json', config, plan['scheduler'], counter, + policy=plan['policy'], stop_file=plan['stop_file'], carry_cny=carry, + started_at_unix=identity['wall_started_at_unix']) + if budget.stopped: + halted = 'uncertain_or_fatal_attempt_requires_review' + builder = load_controller() + active = {} + newly_finished, consecutive_errors = 0, 0 + started = time.monotonic() + + def status(state): + with budget.lock: + summary = write_summary(output, ids, receipts, kind, halted=halted) + attempts = budget.state['attempts'] + value = {'state': state, 'updated_unix': time.time(), 'halted': halted, + 'expected_sources': len(ids), 'preserved_sources': len(seed), + 'committed_sources': len(receipts), 'pending_sources': len(ids) - len(receipts), + 'newly_finished_sources': newly_finished, 'active_sources': len(active), + 'source_errors': sum(r['status'] != 'completed' for r in receipts.values()), + 'legal_terminations': sum(r['terminated'] for r in receipts.values()), + 'new_http_attempts': len(attempts), 'cost_cap_cny': config['limits']['max_estimated_cny'], + 'usage_plus_uncertain_reserves_cny': budget.accounted_cny(), + 'uncertain_new_requests': sum(a['state'] != 'response_received' for a in attempts), + 'carried_usage_plus_uncertain_reserves_cny': carry, + 'wall_started_at_unix': identity['wall_started_at_unix'], + 'wall_deadline_unix': identity['wall_started_at_unix'] + config['limits']['max_wall_seconds'], + 'elapsed_this_process_seconds': time.monotonic() - started, 'invoice': False} + put(output / 'night-status.json', value, replace=True) + return value + + def work(row, folder): + kwargs = {'transport': transport} if transport else {} + if plan['mode'] == 'scripted_test' and transport is None: + def scripted(url, secret, body, timeout): + if plan.get('test_delay_seconds'): + time.sleep(plan['test_delay_seconds']) + return {'model': config['model'], 'usage': {'prompt_tokens': 100, 'completion_tokens': 10}, + 'choices': [{'message': {'content': ScriptedBackend()(body['messages'])}, 'finish_reason': 'stop'}]} + kwargs = {'transport': scripted} + generator = APIBackend(config, config_folder, key=key, counter=counter, budget=budget, **kwargs) + generator.before_source() + return source_receipt(builder, study, row, folder, generator, kind=kind) + + status('running' if remaining and not halted else 'stopped') + if remaining and not halted: + pending = iter(remaining) + with ThreadPoolExecutor(max_workers=plan['scheduler']['workers']) as pool: + def submit(): + nonlocal halted + if halted: + return + try: + budget.check(BUDGET['max_planner_attempts'] + BUDGET['max_skill_steps']) + except APIError as exc: + halted = str(exc); return + item = next(pending, None) + if item is not None: + active[pool.submit(work, *item)] = item[0]['id'] + for _ in range(min(plan['scheduler']['workers'], len(remaining))): + submit() + while active: + done, _ = wait(active, timeout=plan['policy']['heartbeat_seconds'], return_when=FIRST_COMPLETED) + for future in done: + ident = active.pop(future) + try: + receipt = future.result(); receipts[ident] = receipt; newly_finished += 1 + if receipt['status'] == 'runner_error': + halted = 'runner_error' + elif receipt['status'] == 'backend_error': + folder = output / f'sources/{ids.index(ident) + 1:04d}' + error_codes = [read(p)['generation'].get('error') for p in (folder / 'calls').glob('*.json') + if read(p)['error_type'] is not None] + controls = {'user_stop_requested', 'wall_budget_exhausted', + 'estimated_cost_budget_exhausted', 'request_budget_exhausted'} + control_errors = [code for code in error_codes if code in controls] + if control_errors: + halted = control_errors[0] + elif any(code not in TRANSIENT for code in error_codes): + halted = 'fatal_backend_error' + consecutive_errors += 1 + if consecutive_errors >= plan['policy']['consecutive_source_error_limit']: + halted = 'consecutive_source_errors' + else: + consecutive_errors = 0 + except APIError as exc: + halted = str(exc) + except Exception: + halted = 'worker_exception' + with budget.lock: + fatal_stop = budget.stopped + if fatal_stop: + halted = halted or 'fatal_backend_error' + if Path(plan['stop_file']).exists(): + halted = halted or 'user_stop_requested' + status('stopping' if halted else 'running') + if not halted: + for _ in done: + submit() + if halted: + with budget.lock: + budget.stopped = True + result = status('complete' if len(receipts) == len(ids) and not halted else 'stopped') + return result + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--plan', type=Path, required=True) + p.add_argument('--resume', action='store_true') + a = p.parse_args() + try: + result = run_night(read(a.plan), resume=a.resume) + print(json.dumps(result), flush=True) + raise SystemExit(0 if result['state'] == 'complete' else 2) + except Exception as exc: + print(json.dumps({'state': 'startup_failed', 'error_type': type(exc).__name__}), flush=True) + raise SystemExit(3) diff --git a/research/vehicle_program_v1/code/night_supervisor.py b/research/vehicle_program_v1/code/night_supervisor.py new file mode 100644 index 0000000..b5c6cef --- /dev/null +++ b/research/vehicle_program_v1/code/night_supervisor.py @@ -0,0 +1,145 @@ +"""Independent task host: sleep inhibition, owned worker, bounded wall time, final audit.""" +import argparse +from contextlib import contextmanager +import ctypes +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +from artifacts import put, read, sha +from night_runner import validate_plan +from runner import exclusive_run +from api_backend import load_config +from night_resume import validate_resume_ledger + + +@contextmanager +def keep_awake(enabled=True): + """Keep the system awake only while running; allow the display to turn off.""" + if enabled and os.name == 'nt': + fn = ctypes.windll.kernel32.SetThreadExecutionState + fn.argtypes = [ctypes.c_uint32] + fn.restype = ctypes.c_uint32 + if not fn(0x80000001): + raise RuntimeError('sleep_inhibition_failed') + try: + yield + finally: + fn(0x80000000) + else: + yield + + +def on_ac_power(): + if os.name != 'nt': + return True + class Power(ctypes.Structure): + _fields_ = [('ac', ctypes.c_ubyte), ('flags', ctypes.c_ubyte), ('percent', ctypes.c_ubyte), + ('reserved', ctypes.c_ubyte), ('remaining', ctypes.c_ulong), ('full', ctypes.c_ulong)] + value = Power() + if not ctypes.windll.kernel32.GetSystemPowerStatus(ctypes.byref(value)): + raise RuntimeError('power_status_unavailable') + return value.ac == 1 + + +@contextmanager +def owned_worker(command, *, cwd, stdout, stderr, flags, stop_file, grace): + process = subprocess.Popen(command, cwd=cwd, stdout=stdout, stderr=stderr, + stdin=subprocess.DEVNULL, creationflags=flags) + try: + yield process + finally: + if process.poll() is None: + Path(stop_file).touch(exist_ok=True) + try: + process.wait(timeout=grace) + except subprocess.TimeoutExpired: + process.terminate(); process.wait(timeout=30) + + +def supervise(plan_path, *, resume=False): + plan_path = Path(plan_path).resolve() + plan = read(plan_path) + study, manifest, seed = validate_plan(plan) + config = load_config(Path(plan['package']) / 'night-api-profile.json') + original_start = validate_resume_ledger(plan, study, seed, config) if resume else time.time() + deadline = original_start + min(plan['supervisor_wall_seconds'], config['limits']['max_wall_seconds']) + if time.time() >= deadline: + raise RuntimeError('wall_budget_exhausted') + home = Path(plan['home']); home.mkdir(parents=True, exist_ok=True) + if plan['mode'] == 'api' and not on_ac_power(): + raise RuntimeError('ac_power_required') + if Path(plan['stop_file']).exists(): + raise RuntimeError('stop_marker_present') + if plan['mode'] == 'api': + from api_discover import discover + report = discover(home / ('connectivity-' + str(time.time_ns()) + '.json')) + if not report.get('exact_model_listed') or report.get('http_status') != 200: + raise RuntimeError('provider_preflight_failed') + (home / 'supervisor-lock').mkdir(exist_ok=True) + with exclusive_run(home / 'supervisor-lock'), keep_awake(plan['mode'] == 'api'): + import psutil + me = psutil.Process() + started = time.time() + meta = {'state': 'starting', 'supervisor_pid': os.getpid(), 'supervisor_created_unix': me.create_time(), + 'plan_sha256': sha(plan_path), 'started_unix': started, 'wall_deadline_unix': deadline, + 'system_sleep_inhibited': plan['mode'] == 'api', 'display_kept_on': False} + put(home / 'supervisor.json', meta, replace=True) + command = [sys.executable.replace('pythonw.exe', 'python.exe'), '-u', + str(Path(__file__).with_name('night_runner.py')), '--plan', str(plan_path)] + if resume: + command.append('--resume') + flags = subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0 + timed_out = False + with (home / 'worker.stdout.log').open('ab') as stdout, (home / 'worker.stderr.log').open('ab') as stderr, \ + owned_worker(command, cwd=Path(__file__).parent, stdout=stdout, stderr=stderr, + flags=flags, stop_file=plan['stop_file'], + grace=plan['policy']['shutdown_grace_seconds']) as process: + meta.update(state='running', worker_pid=process.pid, + worker_created_unix=psutil.Process(process.pid).create_time()) + put(home / 'supervisor.json', meta, replace=True) + while process.poll() is None: + if time.time() >= deadline: + timed_out = True + stop = Path(plan['stop_file']); stop.parent.mkdir(parents=True, exist_ok=True) + stop.touch(exist_ok=True) + try: + process.wait(timeout=plan['policy']['shutdown_grace_seconds']) + except subprocess.TimeoutExpired: + # This is the Popen handle for our own child, never an arbitrary PID. + process.terminate(); process.wait(timeout=30) + break + time.sleep(1.) + meta.update(state='stopped' if timed_out or process.returncode else 'complete', + worker_exit_code=process.returncode, finished_unix=time.time(), + supervisor_wall_stop=timed_out, system_sleep_inhibited=False) + # Audits run only after generation exits. gold is never read by the worker. + if (Path(plan['output']) / 'run.json').exists(): + try: + from replay import replay + from score import score_run + put(home / 'replay.json', replay(plan['package'], plan['output']), replace=True) + put(home / 'score.json', score_run(plan['package'], plan['output']), replace=True) + meta['post_run_audit'] = 'pass' + except Exception as exc: + meta['post_run_audit'] = type(exc).__name__ + put(home / 'supervisor.json', meta, replace=True) + return meta + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--plan', type=Path, required=True) + p.add_argument('--resume', action='store_true') + args = p.parse_args() + try: + result = supervise(args.plan, resume=args.resume) + raise SystemExit(0 if result['state'] == 'complete' and result.get('post_run_audit') == 'pass' else 2) + except Exception as exc: + # Never write exception text or HTTP bodies into a launcher log. + home = Path(read(args.plan)['home']) + put(home / 'supervisor-failure.json', {'error_type': type(exc).__name__, 'time_unix': time.time()}, replace=True) + raise SystemExit(3) diff --git a/research/vehicle_program_v1/code/pinned-inputs.json b/research/vehicle_program_v1/code/pinned-inputs.json new file mode 100644 index 0000000..0f5279e --- /dev/null +++ b/research/vehicle_program_v1/code/pinned-inputs.json @@ -0,0 +1,12 @@ +{ + "repository": "DocAILab/DCG", + "revision": "84a9a462dcb828332f980ef121f40e1281ce1794", + "download_manifest_sha256": "6f727ae2c3649b17ec9796c1804c9d14ece7720cd2b4c0a135e0b3fe0825ba88", + "files": { + "DCG_VEHICLE/DCG_VEHICLE_CORPUS.json": "4e0a24e1f407543eb946ef6ae5552bd213a22bd6a494ee174a2968e38b809373", + "DCG_VEHICLE/DCG_VEHICLE_SFT.json": "2ccb4e719fec17be93cf9228349760b22c0d87a906cca859252578ed6a94f356", + "DCG_VEHICLE/DCG_VEHICLE_TEST.json": "9f67060671ae0b5eec7fdb95a3a8b27dfde06cd615288fdb50a5cce27740a336", + "DCG_VEHICLE/DCG_VEHICLE_TRAIN.json": "da7c61f1d85e29c424ac0cc4b783a5a9eb3b0cb88f0f137a83b06084789eaead", + "DCG_VEHICLE/DCG_VEHICLE_VAL.json": "847a847c3b8a0aec3dda7654bcca25647683aa4454375c0a0eae1651ef6122f7" + } +} diff --git a/research/vehicle_program_v1/code/prepare.py b/research/vehicle_program_v1/code/prepare.py new file mode 100644 index 0000000..8e0498e --- /dev/null +++ b/research/vehicle_program_v1/code/prepare.py @@ -0,0 +1,106 @@ +"""Freeze the vehicle Val study and mechanically isolate evaluator-only labels.""" +import argparse +from collections import Counter +from itertools import combinations +from pathlib import Path +from artifacts import ROOT, REVISION, BUDGET, GENERATION, code_inventory, put, read, require_private_output, sha, verify_vendor + +SIZES = {'SFT': 1000, 'TRAIN': 1105, 'VAL': 397, 'TEST': 395} + + +def project(row): + metadata = row['metadata'] + if not isinstance(row.get('id'), str) or not row['id'] or not isinstance(row.get('domain'), str) or not row['domain'].strip(): + raise ValueError('missing source identity/domain') + if not isinstance(metadata.get('field_name'), str) or not metadata['field_name'].strip(): + raise ValueError('missing field name') + result = {'field_name': metadata['field_name']} + description = metadata.get('field_description') + if isinstance(description, str) and description.strip(): + result['field_description'] = description + return {'id': row['id'], 'domain': row['domain'], 'metadata': result} + + +def registry_from(corpus): + registry = {} + for entry in corpus: + ident = entry['id'] + if not isinstance(ident, str) or not ident or ident in registry: + raise ValueError('invalid/duplicate standard ID') + required = ('category_root_level', 'category_leaf_level', 'sensitivity_level', 'data_description_and_example', 'reference_standard') + if any(not isinstance(entry[k], str) or not entry[k].strip() for k in required): + raise ValueError('standard missing required text') + registry[ident] = {'standard_id': ident, 'path': [entry['category_root_level'], entry['category_leaf_level']], + 'name': entry['category_leaf_level'], 'grading_level': entry['sensitivity_level'], + 'description': entry['data_description_and_example'] + '\n' + entry['reference_standard']} + if not registry: + raise ValueError('empty standard corpus') + return registry + + +def gold_for(rows, registry): + answers = {} + for row in rows: + path = [row['classification']['category_root_level'], row['classification']['category_leaf_level']] + grade = row['grading']['sensitivity_level'] + compatible = [key for key, e in registry.items() if e['path'] == path and e['grading_level'] == grade] + if not compatible: + raise ValueError('gold has no compatible standard') + answers[row['id']] = {'path': path, 'grading_level': grade, 'compatible_standard_ids': compatible} + return answers + + +def prepare(source, output): + source, output = Path(source), Path(output) + require_private_output(output) + verify_vendor() + pinned = read(ROOT / 'pinned-inputs.json') + if pinned['revision'] != REVISION: + raise ValueError('data revision differs') + for name, expected in pinned['files'].items(): + path = source / name + if path.is_symlink() or sha(path) != expected: + raise ValueError('pinned input hash differs: ' + name) + splits = {key: read(source / f'DCG_VEHICLE/DCG_VEHICLE_{key}.json') for key in SIZES} + ids = {} + for key, rows in splits.items(): + ids[key] = [r['id'] for r in rows] + if len(rows) != SIZES[key] or len(set(ids[key])) != len(rows): + raise ValueError('split size/unique IDs differ: ' + key) + overlaps = {a + '/' + b: len(set(ids[a]) & set(ids[b])) for a, b in combinations(ids, 2)} + if any(overlaps.values()): + raise ValueError('source split overlap') + corpus = read(source / 'DCG_VEHICLE/DCG_VEHICLE_CORPUS.json') + registry = registry_from(corpus) + rows = [project(r) for r in splits['VAL']] + answers = gold_for(splits['VAL'], registry) + study = {'schema': 'vehicle-nontd-v1', 'dataset_revision': REVISION, 'scope': 'development Val397, not blind Test', + 'skill_pool': 'F66', 'budget': BUDGET, 'generation': GENERATION, + 'expected_ids': ids['VAL'], 'rows': rows, 'registry': registry, + 'candidate_unit': 'standard_entry; multiple entries may share category and grade', + 'initial_visible_fields': ['domain', 'field_name'], + 'tool_accessible_fields': ['field_name', 'field_description']} + output.mkdir(parents=True, exist_ok=False) + put(output / 'inference/study.json', study) + put(output / 'evaluation/gold.json', {'answers': answers}) + report = {'split_sizes': {k: len(v) for k, v in splits.items()}, 'id_overlaps': overlaps, + 'standard_entries': len(registry), 'leaf_paths': len({tuple(e['path']) for e in registry.values()}), + 'multistandard_leaf_paths': sum(n > 1 for n in Counter(tuple(e['path']) for e in registry.values()).values()), + 'gold_compatible_entry_counts': dict(Counter(len(v['compatible_standard_ids']) for v in answers.values())), + 'val_rows': len(rows), 'field_description_present': sum('field_description' in r['metadata'] for r in rows), + 'table_or_database_context': False, 'real_model_run': False, 'training_run': False} + put(output / 'preparation-report.json', report) + put(output / 'manifest.json', {'schema': 1, 'repository': 'DocAILab/DCG', 'revision': REVISION, + 'source_files': pinned['files'], 'code': code_inventory(), + 'study_sha256': sha(output / 'inference/study.json'), + 'gold_sha256': sha(output / 'evaluation/gold.json')}) + return report + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--source', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + args = p.parse_args() + import json + print(json.dumps(prepare(args.source, args.output), ensure_ascii=False)) diff --git a/research/vehicle_program_v1/code/replay.py b/research/vehicle_program_v1/code/replay.py new file mode 100644 index 0000000..1b69586 --- /dev/null +++ b/research/vehicle_program_v1/code/replay.py @@ -0,0 +1,50 @@ +"""Replay completed traces against saved messages and responses without a model.""" +import argparse +from copy import deepcopy +from pathlib import Path +from artifacts import BUDGET, load_controller, load_study, read +from runner import read_receipts + + +def replay(package, output): + study, manifest = load_study(package) + identity = read(Path(output) / 'run.json') + if identity['study_sha256'] != manifest['study_sha256']: + raise ValueError('replay study identity differs') + receipts = read_receipts(output, study['expected_ids']) + builder = load_controller() + count = 0 + for n, row in enumerate(study['rows'], 1): + if receipts.get(row['id'], {}).get('status') != 'completed': + continue + folder = Path(output) / 'sources' / f'{n:04d}' + records = [read(p) for p in sorted((folder / 'calls').glob('*.json'))] + position = 0 + def generate(messages): + nonlocal position + if position >= len(records) or messages != records[position]['messages']: + raise AssertionError('replay message/order differs') + call = records[position] + position += 1 + if call['error_type'] is not None: + raise AssertionError('completed source contains backend failure') + generate.last_generation_metadata = deepcopy(call['generation']) + return call['raw_response'] + controller = builder(generate, study['registry'], study['rows'], BUDGET, skill_pool='F66') + trace = controller.run({'source_id': row['id'], 'domain': row['domain'], + 'field_name': row['metadata']['field_name']}).to_dict() + trace['execution_mode'] = identity['backend'] + if position != len(records) or trace != read(folder / 'trace.json'): + raise ValueError('full controller replay differs') + count += 1 + return {'completed_sources_replayed': count, 'receipt_count': len(receipts), + 'expected_count': len(study['rows']), 'model_calls': 0, 'pass': True} + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--package', type=Path, required=True) + p.add_argument('--run', type=Path, required=True) + a = p.parse_args() + import json + print(json.dumps(replay(a.package, a.run))) diff --git a/research/vehicle_program_v1/code/runner.py b/research/vehicle_program_v1/code/runner.py new file mode 100644 index 0000000..1e7d42c --- /dev/null +++ b/research/vehicle_program_v1/code/runner.py @@ -0,0 +1,249 @@ +"""Full Val inference, durable per-source receipts, identity-checked crash recovery.""" +import argparse +from contextlib import contextmanager +from copy import deepcopy +from importlib.metadata import PackageNotFoundError, version +import os +from pathlib import Path +import platform +import time +from artifacts import BUDGET, GENERATION, MODEL_REVISION, fingerprint, load_controller, load_study, put, read, require_private_output, sha +from backends import ScriptedBackend, local_backend + + +@contextmanager +def exclusive_run(output): + """OS releases the advisory lock on crashes; never signal another process.""" + path = Path(output) / '.run.lock' + with path.open('a+b') as f: + f.seek(0, 2) + if f.tell() == 0: + f.write(b'0') + f.flush() + f.seek(0) + if os.name == 'nt': + import msvcrt + msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + try: + # Interrupted atomic writes are never committed evidence. + for tmp in Path(output).rglob('*.json.tmp'): + tmp.unlink() + yield + finally: + if os.name == 'nt': + f.seek(0) + msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, 1) + else: + fcntl.flock(f.fileno(), fcntl.LOCK_UN) + + +def members(folder): + return {p.relative_to(folder).as_posix(): sha(p) for p in sorted(folder.rglob('*')) + if p.is_file() and p.name != 'receipt.json' and not p.name.endswith('.tmp')} + + +def read_receipts(output, ids): + result = {} + samples = Path(output) / 'sources' + expected = {f'{n:04d}': ident for n, ident in enumerate(ids, 1)} + for folder in sorted(samples.iterdir()) if samples.exists() else []: + if not folder.is_dir() or folder.name not in expected: + raise ValueError('unknown source artifact') + receipt_path = folder / 'receipt.json' + if not receipt_path.exists(): + continue + receipt = read(receipt_path) + ident = expected[folder.name] + if receipt['id'] != ident or receipt['files'] != members(folder): + raise ValueError('saved source receipt identity/hash differs') + if receipt['status'] not in ('completed', 'runner_error', 'backend_error', 'interrupted'): + raise ValueError('invalid committed status') + result[ident] = receipt + return result + + +class Recorder: + def __init__(self, generate, folder): + self.generate = generate + self.folder = Path(folder) + self.calls = [] + self.failed = False + self.last_generation_metadata = {} + + def __call__(self, messages): + if self.failed: + raise RuntimeError('backend failed earlier in this source; no further generation') + record = {'messages': deepcopy(messages), 'raw_response': None, 'error_type': None} + started = time.monotonic() + try: + record['raw_response'] = self.generate(messages) + return record['raw_response'] + except Exception as exc: + self.failed = True + record['error_type'] = type(exc).__name__ + raise + finally: + self.last_generation_metadata = deepcopy(getattr(self.generate, 'last_generation_metadata', {})) + record.update(generation=self.last_generation_metadata, seconds=time.monotonic() - started) + self.calls.append(record) + put(self.folder / f'{len(self.calls):04d}.json', record) + + +def environment(): + packages = {} + for name in ('torch', 'transformers', 'accelerate', 'safetensors', 'tokenizers', 'jinja2'): + try: + packages[name] = version(name) + except PackageNotFoundError: + packages[name] = None + return {'python': platform.python_version(), 'platform': platform.platform(), 'packages': packages} + + +def run(package, output, *, backend='mock', model=None, api_config=None, resume=False, max_samples=None, factory=None): + study, manifest = load_study(package) + ids = study['expected_ids'] + output = Path(output) + require_private_output(output) + if max_samples is not None and max_samples < 1: + raise ValueError('max_samples must be positive') + if backend not in ('mock', 'local', 'api') or (backend == 'local' and not model) or (backend == 'api' and not api_config): + raise ValueError('select mock, local with --model, or api with --api-config') + kind = {'mock': 'scripted_cpu_test', 'local': 'local_official_base', 'api': 'hosted_qwen35_9b_api'}[backend] + config, service = None, None + generation = GENERATION + if backend == 'api': + from api_backend import api_generation, api_identity, load_config + config = load_config(api_config) + service, generation = api_identity(config), api_generation() + model_identity = None + if backend == 'local': + m = read(Path(model) / 'model-manifest.json') + if m.get('repo_id') != 'Qwen/Qwen3.5-9B' or m.get('revision') != MODEL_REVISION: + raise ValueError('base model revision differs') + model_identity = sha(Path(model) / 'model-manifest.json') + identity = {'schema': 1, 'arm': 'NONT-D', 'backend': kind, + 'study_sha256': manifest['study_sha256'], 'code_sha256': fingerprint(manifest['code']), + 'model_manifest_sha256': model_identity, 'generation': generation, 'budget': BUDGET, + 'expected_ids': ids, 'environment': environment()} + if service is not None: + identity['api_service'] = service + if output.exists() and not resume: + raise FileExistsError('run exists; explicitly resume the same immutable identity') + output.mkdir(parents=True, exist_ok=resume) + with exclusive_run(output): + if (output / 'run.json').exists(): + if read(output / 'run.json') != identity: + raise ValueError('run identity differs') + else: + if set(p.name for p in output.iterdir()) != {'.run.lock'}: + raise ValueError('unidentified run artifacts') + put(output / 'run.json', identity) + receipts = read_receipts(output, ids) + builder = load_controller() + generator = None + newly_finished = 0 + halted = None + try: + for n, row in enumerate(study['rows'], 1): + if row['id'] in receipts: + continue + folder = output / 'sources' / f'{n:04d}' + if folder.exists(): + # Preserve an interrupted attempt, including any partial model I/O. Never reroll it. + receipt = {'id': row['id'], 'status': 'interrupted', 'terminated': False, + 'stop_reason': 'interrupted_before_commit', 'seconds': None, + 'model_calls': len(list((folder / 'calls').glob('*.json'))), 'files': members(folder)} + put(folder / 'receipt.json', receipt) + else: + if generator is None: + if factory: + generator = factory() + elif backend == 'api': + from api_backend import APIBackend + if not (output / 'backend/api-budget.json').exists() and any((output / 'sources').glob('*/calls/*.json')): + raise ValueError('API budget missing for existing requests') + generator = APIBackend(config, output / 'backend') + else: + generator = ScriptedBackend() if backend == 'mock' else local_backend(model, output / 'backend') + if hasattr(generator, 'before_source'): + generator.before_source() + folder.mkdir(parents=True) + put(folder / 'started.json', {'id': row['id'], 'arm': 'NONT-D'}) + recorded = Recorder(generator, folder / 'calls') + started = time.monotonic() + receipt = {'id': row['id'], 'status': 'runner_error', 'terminated': False, + 'stop_reason': 'runner_error'} + try: + controller = builder(recorded, study['registry'], study['rows'], BUDGET, skill_pool='F66') + if len(controller.library) != 66: + raise ValueError('F66 library size differs') + trace = controller.run({'source_id': row['id'], 'domain': row['domain'], + 'field_name': row['metadata']['field_name']}).to_dict() + trace['execution_mode'] = kind + put(folder / 'trace.json', trace) + receipt.update(status='backend_error' if recorded.failed else 'completed', + terminated=trace['terminated'] and not recorded.failed, + stop_reason=trace['stop_reason']) + except Exception as exc: + receipt['error_type'] = type(exc).__name__ + receipt.update(seconds=time.monotonic() - started, model_calls=len(recorded.calls), files=members(folder)) + receipt.update(input_tokens=sum(c['generation'].get('input_tokens') or 0 for c in recorded.calls), + output_tokens=sum(c['generation'].get('output_tokens') or 0 for c in recorded.calls), + token_counts_measured=bool(recorded.calls) and all( + c['generation'].get('input_tokens') is not None and c['generation'].get('output_tokens') is not None + for c in recorded.calls)) + put(folder / 'receipt.json', receipt) + receipts[row['id']] = receipt + newly_finished += 1 + write_summary(output, ids, receipts, kind) + if receipt['status'] in ('runner_error', 'backend_error'): + halted = receipt['status'] + break + if max_samples is not None and newly_finished >= max_samples: + break + except Exception as exc: + halted = type(exc).__name__ + result = write_summary(output, ids, receipts, kind, halted=halted) + return result + + +def write_summary(output, ids, receipts, kind, *, halted=None): + summary = {'backend': kind, 'quality_claim': False if kind == 'scripted_cpu_test' else 'pending_local_scoring', + 'expected_count': len(ids), 'committed_count': len(receipts), 'pending_count': len(ids) - len(receipts), + 'terminated_count': sum(r['terminated'] for r in receipts.values()), + 'error_count': sum(r['status'] != 'completed' for r in receipts.values()), + 'model_calls': sum(r['model_calls'] for r in receipts.values()), 'halted': halted, + 'input_tokens': sum(r.get('input_tokens') or 0 for r in receipts.values()), + 'output_tokens': sum(r.get('output_tokens') or 0 for r in receipts.values()), + 'token_counts_measured': bool(receipts) and all(r.get('token_counts_measured') for r in receipts.values()), + 'status': 'complete' if len(receipts) == len(ids) else 'incomplete'} + put(Path(output) / 'summary.json', summary, replace=True) + budget_path = Path(output) / 'backend/api-budget.json' + if kind == 'hosted_qwen35_9b_api' and budget_path.exists(): + attempts = read(budget_path)['attempts'] + summary.update(api_http_attempts=len(attempts), + api_reserved_cost_upper_cny=sum(a['reserved_cny'] for a in attempts), + api_uncertain_or_failed_attempts=sum(a['state'] != 'response_received' for a in attempts), + api_cost_is_invoice=False) + put(Path(output) / 'summary.json', summary, replace=True) + return summary + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--package', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--backend', choices=('mock', 'local', 'api'), default='mock') + p.add_argument('--model', type=Path) + p.add_argument('--api-config', type=Path) + p.add_argument('--resume', action='store_true') + p.add_argument('--max-samples', type=int) + a = p.parse_args() + import json + result = run(a.package, a.output, backend=a.backend, model=a.model, api_config=a.api_config, + resume=a.resume, max_samples=a.max_samples) + print(json.dumps(result)) + raise SystemExit(0 if result['status'] == 'complete' and not result['halted'] else 2) diff --git a/research/vehicle_program_v1/code/score.py b/research/vehicle_program_v1/code/score.py new file mode 100644 index 0000000..b5ae7a0 --- /dev/null +++ b/research/vehicle_program_v1/code/score.py @@ -0,0 +1,102 @@ +"""Evaluator-only label access; missing/failed sources stay in the full denominator.""" +import argparse +from collections import Counter +from collections.abc import Mapping +from pathlib import Path +from artifacts import fingerprint, load_study, put, read, sha +from runner import read_receipts + +SEARCH = {'SELECT_CATEGORY_TOPK', 'SEARCH_STANDARD', 'SEARCH_STANDARD_BY_FIELD', + 'SEARCH_STANDARD_BY_CONTEXT', 'SEARCH_STANDARD_BY_TABLE', 'SEARCH_STANDARD_BY_DATABASE', 'RESEARCH_STANDARD'} + + +def macro(golds, preds, labels): + values = [] + for label in labels: + tp = sum(g == label and p == label for g, p in zip(golds, preds)) + fp = sum(g != label and p == label for g, p in zip(golds, preds)) + fn = sum(g == label and p != label for g, p in zip(golds, preds)) + values.append(2 * tp / (2 * tp + fp + fn) if 2 * tp + fp + fn else 0.) + return sum(values) / len(values) if values else 0. + + +def score_traces(ids, traces, answers, registry): + if not ids or len(set(ids)) != len(ids) or set(ids) != set(answers) or not set(traces) <= set(ids): + raise ValueError('score IDs differ') + counts = Counter() + details = {} + golds, preds = [], [] + for ident in ids: + trace = traces.get(ident) or {} + final = trace.get('final_result') or {} + final = final if isinstance(final, Mapping) else {} + key = final.get('category') + entry = registry.get(key) if isinstance(key, str) else None + legal = bool(trace.get('terminated') is True and trace.get('stop_reason') == 'validated_terminal' + and entry and final.get('grading_level') == entry['grading_level']) + gold = answers[ident] + category = bool(legal and entry['path'] == gold['path']) + grade = bool(legal and final['grading_level'] == gold['grading_level']) + history = [] + for program in trace.get('programs', []): + for step in program.get('steps', []): + candidates = (step.get('data') or {}).get('standard_candidates') + if step.get('call', {}).get('tool') in SEARCH and step.get('status') == 'success' and isinstance(candidates, list): + keys = [r.get('category') or r.get('category_id') for r in candidates if isinstance(r, dict)] + if len(keys) == 5 and len(set(keys)) == 5 and set(keys) <= set(registry): + history.append(keys) + hits = [any(registry[k]['path'] == gold['path'] for k in group) for group in history] + entry_hits = [bool(set(group) & set(gold['compatible_standard_ids'])) for group in history] + row = {'terminated': legal, 'category_correct': category, 'grading_correct': grade, + 'joint_correct': category and grade, 'top5_first_hit': bool(hits and hits[0]), + 'top5_final_hit': bool(hits and hits[-1]), 'top5_any_hit': any(hits), + 'top5_compatible_entry_any_hit': any(entry_hits)} + counts.update({k: int(v) for k, v in row.items()}) + details[ident] = row + golds.append(tuple(gold['path'])) + preds.append(tuple(entry['path']) if legal else None) + n = len(ids) + return {'denominator': n, 'missing_or_failed_traces': n - len(traces), + **{key: counts[key] for key in ('terminated', 'category_correct', 'grading_correct', 'joint_correct', + 'top5_first_hit', 'top5_final_hit', 'top5_any_hit', 'top5_compatible_entry_any_hit')}, + 'category_accuracy': counts['category_correct'] / n, 'grading_accuracy': counts['grading_correct'] / n, + 'joint_accuracy': counts['joint_correct'] / n, 'termination_rate': counts['terminated'] / n, + 'macro_f1_supported': macro(golds, preds, set(golds)), + 'macro_f1_all_leaf_paths': macro(golds, preds, {tuple(e['path']) for e in registry.values()}), + 'supported_leaf_paths': len(set(golds)), 'registry_leaf_paths': len({tuple(e['path']) for e in registry.values()}), + 'per_id': details} + + +def score_run(package, output): + study, manifest = load_study(package) + if sha(Path(package) / 'evaluation/gold.json') != manifest['gold_sha256']: + raise ValueError('gold identity changed') + identity = read(Path(output) / 'run.json') + if identity['study_sha256'] != manifest['study_sha256'] or identity['code_sha256'] != fingerprint(manifest['code']) or identity['expected_ids'] != study['expected_ids']: + raise ValueError('scoring run identity differs') + receipts = read_receipts(output, study['expected_ids']) + traces = {} + for n, ident in enumerate(study['expected_ids'], 1): + if ident in receipts and receipts[ident]['status'] == 'completed': + traces[ident] = read(Path(output) / 'sources' / f'{n:04d}' / 'trace.json') + result = score_traces(study['expected_ids'], traces, read(Path(package) / 'evaluation/gold.json')['answers'], study['registry']) + result.update(backend=identity['backend'], quality_claim=identity['backend'] in ('local_official_base', 'hosted_qwen35_9b_api'), + complete=len(receipts) == len(study['expected_ids']), + scope='development validation; grading derives from the chosen standard entry', + run_manifest_sha256=sha(Path(output) / 'run.json')) + if 'api_service' in identity: + result['api_service'] = identity['api_service'] + result['comparison_scope'] = 'hosted API baseline; not verified equivalent to pinned local FP16 weights' + return result + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--package', type=Path, required=True) + p.add_argument('--run', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + a = p.parse_args() + result = score_run(a.package, a.run) + put(a.output, result) + import json + print(json.dumps({k: v for k, v in result.items() if k != 'per_id'})) diff --git a/research/vehicle_program_v1/code/tests/test_api.py b/research/vehicle_program_v1/code/tests/test_api.py new file mode 100644 index 0000000..8591898 --- /dev/null +++ b/research/vehicle_program_v1/code/tests/test_api.py @@ -0,0 +1,190 @@ +from copy import deepcopy +import io +import json +from pathlib import Path +import sys +import tempfile +import unittest +import urllib.error +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from api_backend import (APIBackend, APIBudgetExceeded, APIError, Budget, MODEL, + NoRedirect, api_identity, credential_from_text, default_config, + http_post, load_config) +from artifacts import put, read +from backends import ScriptedBackend +from replay import replay +from runner import run +from test_vehicle import fixture + +SECRET = 'fictional-secret-for-unit-tests-only' +MESSAGES = [{'role': 'system', 'content': 'Return JSON.'}, {'role': 'user', 'content': 'synthetic'}] + + +def response(content='{"ok":true}', *, inp=100, out=10, reason='stop'): + return {'model': MODEL, 'usage': {'prompt_tokens': inp, 'completion_tokens': out}, + 'choices': [{'message': {'content': content}, 'finish_reason': reason}]} + + +class APITests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.config = default_config(self.root / 'fictional-tokenizer') + self.config['limits']['min_interval_seconds'] = .00001 + self.config['limits']['max_estimated_cny'] = 10. + + def backend(self, transport=None, counter=lambda m: 100, **kwargs): + return APIBackend(self.config, self.root / 'backend', key=SECRET, + counter=counter, transport=transport or (lambda *a: response()), **kwargs) + + def test_no_key_in_profile_identity_and_wrong_model_or_host_refused(self): + self.assertNotIn(SECRET, json.dumps(api_identity(self.config))) + for key, value in [('model', 'qwen3.5-flash'), ('base_url', 'https://example.com/v1'), + ('provider_precision', 'FP16'), ('api_key', SECRET)]: + config = deepcopy(self.config); config[key] = value + path = self.root / (key + '.json'); put(path, config) + with self.assertRaises(ValueError): + load_config(path) + + def test_contract_sent_to_same_9b_without_json_mode_or_gold(self): + calls = [] + def transport(url, key, body, timeout): + calls.append(deepcopy(body)) + self.assertEqual(url, 'https://api.siliconflow.cn/v1/chat/completions') + self.assertEqual(key, SECRET) + return response() + backend = self.backend(transport) + self.assertEqual(backend(MESSAGES), '{"ok":true}') + self.assertEqual(calls[0]['model'], MODEL) + self.assertIs(calls[0]['enable_thinking'], False) + self.assertNotIn('seed', calls[0]) + self.assertNotIn('chat_template_kwargs', calls[0]) + self.assertEqual(calls[0]['max_tokens'], 512) + self.assertEqual(calls[0]['temperature'], 0.) + self.assertNotIn('response_format', calls[0]) + self.assertNotIn(SECRET, (self.root / 'backend/api-budget.json').read_text()) + + def test_overlong_reference_context_prevents_any_http(self): + def forbidden(*args): + raise AssertionError('network called for overlong prompt') + backend = self.backend(forbidden, counter=lambda m: 32257) + with self.assertRaisesRegex(APIError, 'reference_context_budget_exceeded'): + backend(MESSAGES) + self.assertEqual(backend.budget.state['attempts'], []) + + def test_durable_unknown_attempt_is_counted_and_never_retried(self): + calls = [] + def fail(*args): + calls.append(1) + raise TimeoutError(SECRET) + backend = self.backend(fail) + with self.assertRaisesRegex(APIError, '^api_contract_error$'): + backend(MESSAGES) + self.assertEqual(len(calls), 1) + resumed = self.backend() + self.assertEqual(len(resumed.budget.state['attempts']), 1) + self.assertNotIn(SECRET, (self.root / 'backend/api-budget.json').read_text()) + self.assertNotIn(SECRET, json.dumps(backend.last_generation_metadata)) + + def test_actual_http_error_body_is_not_propagated(self): + class FakeOpener: + def open(self, *args, **kwargs): + raise urllib.error.HTTPError('https://api.together.ai', 401, SECRET, {}, io.BytesIO(SECRET.encode())) + with patch('urllib.request.build_opener', return_value=FakeOpener()): + with self.assertRaisesRegex(APIError, '^http_401$'): + http_post('https://api.together.ai/v1/chat/completions', SECRET, {}, 1) + + def test_redirect_does_not_forward_bearer(self): + with self.assertRaisesRegex(APIError, '^redirect_refused$'): + NoRedirect().redirect_request(None, None, 307, '', {}, 'https://example.com') + + def test_wrong_model_thinking_missing_usage_and_overbudget_usage_stop(self): + bad = [] + value = response(); value['model'] = 'Qwen/Qwen3.5-27B'; bad.append(value) + value = response(); value['choices'][0]['message']['reasoning_content'] = 'hidden'; bad.append(value) + value = response(); value['usage'] = {}; bad.append(value) + bad.append(response(inp=32257)); bad.append(response(out=513)); bad.append(response(SECRET)) + for i, value in enumerate(bad): + backend = APIBackend(self.config, self.root / f'bad-{i}', key=SECRET, counter=lambda m: 100, + transport=lambda *a, v=value: v) + with self.assertRaises(APIError): + backend(MESSAGES) + self.assertTrue(backend.last_generation_metadata['error']) + + def test_length_finish_reason_is_preserved_for_controller(self): + backend = self.backend(lambda *a: response('{"unfinished":', out=512, reason='length')) + self.assertEqual(backend(MESSAGES), '{"unfinished":') + self.assertTrue(backend.last_generation_metadata['truncation_at_budget']) + + def test_request_cost_and_wall_limits_include_restart(self): + for kind in ('requests', 'cost', 'wall'): + config = deepcopy(self.config) + if kind == 'requests': config['limits']['max_http_requests'] = 1 + if kind == 'cost': config['limits']['max_estimated_cny'] = .0001 + clock = [10.] + budget = Budget(self.root / f'{kind}.json', config, clock=lambda: clock[0]) + if kind == 'requests': budget.reserve({'synthetic': 1}) + if kind == 'wall': clock[0] += 1801 + resumed = Budget(self.root / f'{kind}.json', config, clock=lambda: clock[0]) + with self.assertRaises(APIBudgetExceeded): resumed.reserve({'synthetic': 2}) + + def test_credential_loader_requires_unique_dedicated_section(self): + text = '## Other\nAPI Key: other-key-must-not-be-used\n## 汽车 NONT-D SiliconFlow API\nAPI Key: ' + SECRET + self.assertEqual(credential_from_text(text), SECRET) + for value in ('## Other\nAPI Key: ' + SECRET, text + '\n' + text, + '## 汽车 NONT-D SiliconFlow API\nAPI Key: <尚未填写>'): + with self.assertRaises(ValueError): credential_from_text(value) + + def test_api_integration_gold_isolation_resume_and_full_replay(self): + package, output = self.root / 'package', self.root / 'run' + fixture(package) + config_path = self.root / 'profile.json'; put(config_path, self.config) + scripted = ScriptedBackend() + observed = [] + def transport(url, key, body, timeout): + observed.append(body) + return response(scripted(body['messages'])) + def factory(): + return APIBackend(self.config, output / 'backend', key=SECRET, + counter=lambda m: 100, transport=transport) + original_open = Path.open + def guard(path, *args, **kwargs): + if path.name == 'gold.json': raise AssertionError('inference read gold') + return original_open(path, *args, **kwargs) + with patch.object(Path, 'open', guard): + first = run(package, output, backend='api', api_config=config_path, factory=factory, max_samples=1) + self.assertEqual(first['committed_count'], 1) + complete = run(package, output, backend='api', api_config=config_path, factory=factory, resume=True) + self.assertEqual(complete['committed_count'], 3) + self.assertEqual(complete['api_http_attempts'], 12) + self.assertEqual(replay(package, output)['completed_sources_replayed'], 3) + for path in output.rglob('*.json'): + self.assertNotIn(SECRET, path.read_text(encoding='utf-8')) + self.assertNotIn('SECRET_GOLD', json.dumps(observed)) + def forbidden(): raise AssertionError('completed run initialized API') + run(package, output, backend='api', api_config=config_path, factory=forbidden, resume=True) + changed = deepcopy(self.config); changed['limits']['max_estimated_cny'] = 2. + put(self.root / 'changed.json', changed) + with self.assertRaisesRegex(ValueError, 'run identity differs'): + run(package, output, backend='api', api_config=self.root / 'changed.json', resume=True) + + def test_source_is_left_pending_when_its_call_allowance_does_not_fit(self): + package, output = self.root / 'package', self.root / 'run' + fixture(package) + self.config['limits']['max_http_requests'] = 15 + path = self.root / 'profile.json'; put(path, self.config) + result = run(package, output, backend='api', api_config=path, + factory=lambda: APIBackend(self.config, output / 'backend', key=SECRET, + counter=lambda m: 100, transport=lambda *a: response())) + self.assertEqual(result['halted'], 'APIBudgetExceeded') + self.assertEqual(result['pending_count'], 3) + self.assertEqual(result['committed_count'], 0) + self.assertEqual(result['api_http_attempts'], 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/research/vehicle_program_v1/code/tests/test_night.py b/research/vehicle_program_v1/code/tests/test_night.py new file mode 100644 index 0000000..075c807 --- /dev/null +++ b/research/vehicle_program_v1/code/tests/test_night.py @@ -0,0 +1,173 @@ +from copy import deepcopy +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from api_backend import APIBudgetExceeded, APIError, default_config +from api_parallel import scheduler_config +from artifacts import put, read, sha +from backends import ScriptedBackend +from night_ops import preflight +from night_runner import NightBudget, run_night +from night_supervisor import keep_awake +from replay import replay +from runner import run +from test_api import MESSAGES, response +from test_vehicle import fixture + + +def night_fixture(root, *, count=6, seed_count=0): + package, home = root / 'package', root / 'home' + study = fixture(package, count=count) + config = default_config(root / 'fictional-tokenizer') + config['limits'].update(min_interval_seconds=.000001, max_http_requests=count * 16, + max_estimated_cny=10., max_wall_seconds=600.) + put(package / 'night-api-profile.json', config) + provenance = {'receipts': {}, 'usage_plus_uncertain_reserves_cny': .1 if seed_count else 0.} + if seed_count: + donor = root / 'donor' + run(package, donor, max_samples=seed_count) + import shutil + for n, ident in enumerate(study['expected_ids'][:seed_count], 1): + src = donor / f'sources/{n:04d}' + if n == 1: + receipt = read(src / 'receipt.json'); receipt.update(status='backend_error', terminated=False) + put(src / 'receipt.json', receipt, replace=True) + shutil.copytree(src, package / f'seed/sources/{n:04d}') + provenance['receipts'][ident] = {'receipt_sha256': sha(src / 'receipt.json')} + put(package / 'seed-provenance.json', provenance) + home.mkdir() + scheduler = scheduler_config(); scheduler.update(workers=2, tokens_per_minute=100000000, requests_per_minute=100000) + plan = {'schema': 'vehicle-nontd-night-v1', 'mode': 'scripted_test', 'package': str(package), + 'package_manifest_sha256': sha(package / 'manifest.json'), + 'home': str(home), 'output': str(home / 'run'), 'stop_file': str(home / 'STOP'), + 'files': {n: sha(package / n) for n in ('night-api-profile.json', 'seed-provenance.json')}, + 'scheduler': scheduler, 'supervisor_wall_seconds': 30., + 'policy': {'heartbeat_seconds': .01, 'transient_cooldown_seconds': .001, + 'rate_limit_cooldown_seconds': .001, 'consecutive_source_error_limit': 3, + 'shutdown_grace_seconds': 1.}} + return plan + + +class NightTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + + def test_preserve_seed_errors_continue_fresh_sources_and_replay(self): + plan = night_fixture(self.root, count=8, seed_count=2) + before = {p.name + p.parent.name: sha(p) for p in (self.root / 'package/seed').glob('sources/*/receipt.json')} + original_open = Path.open + def guard(path, *a, **kw): + if path.name in ('gold.json', '敏感信息交接.md'): raise AssertionError('inference read gold or credential') + return original_open(path, *a, **kw) + with patch.object(Path, 'open', guard): + result = run_night(plan) + self.assertEqual(result['state'], 'complete') + self.assertEqual(result['newly_finished_sources'], 6) + self.assertEqual(result['new_http_attempts'], 24) + self.assertEqual(result['source_errors'], 1) + self.assertEqual(result['carried_usage_plus_uncertain_reserves_cny'], .1) + after = {p.name + p.parent.name: sha(p) for p in Path(plan['output']).glob('sources/000[12]/receipt.json')} + self.assertEqual(before, after) + self.assertEqual(replay(plan['package'], plan['output'])['completed_sources_replayed'], 7) + previous = {p.as_posix(): sha(p) for p in Path(plan['output']).glob('sources/*/receipt.json')} + resumed = run_night(plan, resume=True) + self.assertEqual(resumed['new_http_attempts'], 24) + self.assertEqual(previous, {p.as_posix(): sha(p) for p in Path(plan['output']).glob('sources/*/receipt.json')}) + + def test_one_transport_failure_is_not_retried_and_new_sources_continue(self): + plan = night_fixture(self.root, count=5) + calls = [0] + def transport(url, key, body, timeout): + calls[0] += 1 + if calls[0] == 1: raise APIError('transport_or_json_error') + return response(ScriptedBackend()(body['messages'])) + import time + def slow_error_commit(path, value, **kwargs): + if Path(path).name == 'api-budget.json' and any(a['state'] == 'transport_or_json_error' for a in value['attempts']): + time.sleep(.005) + return put(path, value, **kwargs) + with patch('api_parallel.put', slow_error_commit): + result = run_night(plan, transport=transport) + self.assertEqual(result['state'], 'complete') + self.assertEqual(result['source_errors'], 1) + self.assertEqual(calls[0], 17) + self.assertEqual(result['uncertain_new_requests'], 1) + self.assertGreaterEqual(result['usage_plus_uncertain_reserves_cny'], .054528) + + def test_three_consecutive_source_errors_trip_circuit(self): + plan = night_fixture(self.root, count=9) + def failure(*args): raise APIError('transport_or_json_error') + result = run_night(plan, transport=failure) + self.assertEqual(result['state'], 'stopped') + self.assertGreaterEqual(result['pending_sources'], 5) + self.assertIn(result['halted'], ('consecutive_source_errors', 'fatal_backend_error')) + self.assertLessEqual(result['new_http_attempts'], 4) + + def test_auth_error_stops_all_new_requests(self): + plan = night_fixture(self.root, count=8) + def failure(*args): raise APIError('http_401') + result = run_night(plan, transport=failure) + self.assertEqual(result['state'], 'stopped') + self.assertEqual(result['halted'], 'fatal_backend_error') + self.assertLessEqual(result['new_http_attempts'], 2) + + def test_stop_marker_blocks_http_and_retains_pending_sources(self): + plan = night_fixture(self.root) + Path(plan['stop_file']).touch() + result = run_night(plan) + self.assertEqual(result['new_http_attempts'], 0) + self.assertEqual(result['pending_sources'], 6) + self.assertEqual(result['halted'], 'user_stop_requested') + + def test_money_cap_includes_carried_pilot_spend(self): + plan = night_fixture(self.root, seed_count=1) + profile = Path(plan['package']) / 'night-api-profile.json' + config = read(profile); config['limits']['max_estimated_cny'] = .95 + put(profile, config, replace=True); plan['files']['night-api-profile.json'] = sha(profile) + result = run_night(plan) + self.assertEqual(result['new_http_attempts'], 0) + self.assertEqual(result['halted'], 'estimated_cost_budget_exhausted') + + def test_unknown_durable_request_blocks_restart_and_preflight(self): + plan = night_fixture(self.root) + config = read(Path(plan['package']) / 'night-api-profile.json') + ledger = NightBudget(self.root / 'budget.json', config, plan['scheduler'], lambda m: 100, + policy=plan['policy'], stop_file=plan['stop_file']) + ledger.reserve({'messages': MESSAGES, 'max_tokens': 512}) + restored = NightBudget(self.root / 'budget.json', config, plan['scheduler'], lambda m: 100, + policy=plan['policy'], stop_file=plan['stop_file']) + with self.assertRaises(APIBudgetExceeded): restored.check() + Path(plan['output']).mkdir() + put(Path(plan['output']) / 'backend/api-budget.json', read(self.root / 'budget.json')) + with self.assertRaisesRegex(RuntimeError, 'uncertain_or_fatal_attempt_requires_review'): + preflight(plan, resume=True) + + def test_sleep_guard_releases_system_requirement_without_display_flag(self): + import os + if os.name != 'nt': self.skipTest('Windows sleep API') + calls = [] + def fake(flags): calls.append(flags); return 1 + with patch('night_supervisor.ctypes.windll.kernel32.SetThreadExecutionState', fake): + with self.assertRaises(RuntimeError): + with keep_awake(): raise RuntimeError('simulated worker exception') + self.assertEqual(calls, [0x80000001, 0x80000000]) + + def test_atomic_windows_commit_retry_preserves_evidence(self): + original = Path.replace; calls = [0] + def transient(path, target): + calls[0] += 1 + if calls[0] < 4: raise PermissionError('simulated status reader') + return original(path, target) + with patch.object(Path, 'replace', transient), patch('artifacts.time.sleep'): + put(self.root / 'committed.json', {'synthetic': True}) + self.assertEqual(read(self.root / 'committed.json'), {'synthetic': True}) + self.assertEqual(calls[0], 4) + + +if __name__ == '__main__': unittest.main() diff --git a/research/vehicle_program_v1/code/tests/test_night_recovery.py b/research/vehicle_program_v1/code/tests/test_night_recovery.py new file mode 100644 index 0000000..3d41390 --- /dev/null +++ b/research/vehicle_program_v1/code/tests/test_night_recovery.py @@ -0,0 +1,180 @@ +"""Adversarial recovery and admission tests: no real HTTP or credentials.""" +from copy import deepcopy +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from artifacts import put, read, sha +from api_backend import APIBudgetExceeded, default_config +from api_parallel import scheduler_config +from backends import ScriptedBackend +from night_ops import preflight +from night_runner import NightBudget, run_night +from night_supervisor import supervise +from night_resume import effective_cooldown +from test_api import MESSAGES, response +from test_night import night_fixture + + +class RecoveryTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.network = patch('urllib.request.OpenerDirector.open', side_effect=AssertionError('real network forbidden')) + self.network.start(); self.addCleanup(self.network.stop) + + def paused_run(self, root): + root.mkdir(exist_ok=True) + plan = night_fixture(root, count=3) + plan['scheduler']['workers'] = 1 + calls = [0] + def transport(url, key, body, timeout): + calls[0] += 1 + if calls[0] == 4: + Path(plan['stop_file']).touch() + return response(ScriptedBackend()(body['messages'])) + first = run_night(plan, transport=transport) + self.assertEqual(first['committed_sources'], 1) + Path(plan['stop_file']).unlink() + return plan, Path(plan['output']) / 'backend/api-budget.json' + + def assert_rejected(self, plan): + with patch('night_runner.read_credential', side_effect=AssertionError('credential read')), \ + patch('night_runner.APIBackend', side_effect=AssertionError('backend initialized')): + with self.assertRaises(RuntimeError): preflight(plan, resume=True) + with self.assertRaises(RuntimeError): run_night(plan, resume=True) + + def test_missing_ledger_rejected_without_recreating_or_requesting(self): + plan, ledger = self.paused_run(self.root) + before = sha(Path(plan['output']) / 'run.json') + ledger.rename(self.root / 'preserved-ledger.json') + self.assert_rejected(plan) + self.assertFalse(ledger.exists()) + self.assertEqual(before, sha(Path(plan['output']) / 'run.json')) + + def test_resume_without_existing_run_never_initializes(self): + plan = night_fixture(self.root) + with self.assertRaises(RuntimeError): preflight(plan, resume=True) + with self.assertRaisesRegex(RuntimeError, 'resume_run_identity_missing'): run_night(plan, resume=True) + self.assertFalse(Path(plan['output']).exists()) + + def test_rollback_hash_usage_charge_and_origin_corruption_rejected(self): + def mutate(value, kind): + if kind == 'empty': value['attempts'] = [] + elif kind == 'truncated': value['attempts'].pop() + elif kind == 'hash': value['attempts'][0]['request_sha256'] = '0' * 64 + elif kind == 'usage': value['attempts'][0]['usage']['prompt_tokens'] += 1 + elif kind == 'cost': value['attempts'][0]['usage_cost_cny'] = 0. + elif kind == 'origin': value['started_at_unix'] += 1 + elif kind == 'unmatched': + extra = deepcopy(value['attempts'][0]); extra['number'] = 5; value['attempts'].append(extra) + for kind in ('empty', 'truncated', 'hash', 'usage', 'cost', 'origin', 'unmatched'): + with self.subTest(kind=kind): + plan, ledger = self.paused_run(self.root / kind) + state = read(ledger); mutate(state, kind); put(ledger, state, replace=True) + self.assert_rejected(plan) + + def test_normal_resume_preserves_cost_origin_and_receipt(self): + plan, ledger = self.paused_run(self.root) + original = read(ledger) + receipt = Path(plan['output']) / 'sources/0001/receipt.json' + digest = sha(receipt) + self.assertTrue(preflight(plan, resume=True)['ready']) + result = run_night(plan, resume=True) + after = read(ledger) + self.assertEqual(result['state'], 'complete') + self.assertEqual(after['started_at_unix'], original['started_at_unix']) + self.assertEqual(after['attempts'][:4], original['attempts']) + self.assertEqual(len(after['attempts']), 12) + self.assertAlmostEqual(result['usage_plus_uncertain_reserves_cny'], .00324) + self.assertEqual(digest, sha(receipt)) + + def test_expired_resume_rejected_before_supervisor_network_or_worker(self): + plan, ledger = self.paused_run(self.root) + plan_path = self.root / 'plan.json'; put(plan_path, plan) + origin = read(ledger)['started_at_unix'] + with patch('time.time', return_value=origin + 601), \ + patch('night_supervisor.subprocess.Popen', side_effect=AssertionError('worker started')): + with self.assertRaisesRegex(RuntimeError, 'wall_budget_exhausted'): preflight(plan, resume=True) + with self.assertRaisesRegex(RuntimeError, 'wall_budget_exhausted'): supervise(plan_path, resume=True) + + def test_cooldown_added_during_tpm_wait_is_honored(self): + for outcome, delay in [('http_429', 120.), ('http_503', 30.)]: + with self.subTest(outcome=outcome): + config = default_config(self.root) + config['limits'].update(max_estimated_cny=10., max_wall_seconds=600.) + scheduler = scheduler_config(); scheduler['tokens_per_minute'] = 1000 + now = [1000.]; holder = {}; triggered = [False] + def sleep(seconds): + # Inject a new cooldown just before the token window opens. + if not triggered[0] and now[0] >= 1059.: + triggered[0] = True + holder['budget'].finish(holder['first'], outcome) + now[0] += seconds + budget = NightBudget(self.root / (outcome + '.json'), config, scheduler, lambda m: 100, + policy={'rate_limit_cooldown_seconds': 120., 'transient_cooldown_seconds': 30.}, + stop_file=self.root / 'STOP', clock=lambda: now[0], sleep=sleep) + holder['budget'] = budget + body = {'messages': MESSAGES, 'max_tokens': 512} + holder['first'] = budget.reserve(body) + second = budget.reserve(body) + self.assertTrue(triggered[0]) + self.assertGreaterEqual(second['sent_at_unix'], 1059. + delay) + self.assertGreaterEqual(second['sent_at_unix'], budget.cooldown_until) + + def test_resume_preserves_unexpired_cooldown_and_stop_interrupts_wait(self): + config = default_config(self.root); config['limits'].update(max_estimated_cny=10.) + scheduler = scheduler_config(); now = [1000.] + policy = {'rate_limit_cooldown_seconds': 120., 'transient_cooldown_seconds': 30.} + body = {'messages': MESSAGES, 'max_tokens': 512} + def sleep(seconds): now[0] += seconds + kwargs = dict(policy=policy, stop_file=self.root / 'STOP', clock=lambda: now[0], sleep=sleep) + budget = NightBudget(self.root / 'ledger.json', config, scheduler, lambda m: 100, **kwargs) + entry = budget.reserve(body); budget.finish(entry, 'http_429') + restored = NightBudget(budget.path, config, scheduler, lambda m: 100, **kwargs) + self.assertEqual(restored.cooldown_until, 1120.) + self.assertGreaterEqual(restored.reserve(body)['sent_at_unix'], 1120.) + restored.cooldown_until = now[0] + 120. + def stop(seconds): Path(kwargs['stop_file']).touch(); now[0] += seconds + restored.sleep = stop + with self.assertRaisesRegex(APIBudgetExceeded, 'user_stop_requested'): restored.reserve(body) + self.assertEqual(len(restored.state['attempts']), 2) + + def test_failure_drained_after_circuit_stop_restores_conservative_cooldown(self): + config = default_config(self.root); config['limits'].update(max_estimated_cny=10.) + scheduler = scheduler_config(); now = [1000.] + policy = {'rate_limit_cooldown_seconds': 120., 'transient_cooldown_seconds': 30.} + body = {'messages': MESSAGES, 'max_tokens': 512} + def sleep(seconds): now[0] += seconds + kwargs = dict(policy=policy, stop_file=self.root / 'STOP', clock=lambda: now[0], sleep=sleep) + budget = NightBudget(self.root / 'ledger.json', config, scheduler, lambda m: 100, **kwargs) + first, tail = budget.reserve(body), budget.reserve(body) + budget.finish(first, 'http_503') + budget.stopped = True + now[0] = 1008.478 + budget.finish(tail, 'transport_or_json_error') + original = read(budget.path) + self.assertEqual(original['cooldown_until_unix'], 1030.) + restored = NightBudget(budget.path, config, scheduler, lambda m: 100, **kwargs) + self.assertEqual(read(budget.path), original) + self.assertAlmostEqual(restored.cooldown_until, 1038.478) + next_entry = restored.reserve(body) + self.assertGreaterEqual(next_entry['sent_at_unix'], 1038.478) + + def test_effective_cooldown_uses_longer_recorded_or_429_tail_deadline(self): + policy = {'rate_limit_cooldown_seconds': 120., 'transient_cooldown_seconds': 30.} + ledger = {'cooldown_until_unix': 1050., 'attempts': [{'state': 'http_429', 'finished_at_unix': 1000.}]} + self.assertEqual(effective_cooldown(ledger, policy), 1120.) + ledger['cooldown_until_unix'] = 1200. + self.assertEqual(effective_cooldown(ledger, policy), 1200.) + for invalid in (float('nan'), -1, '1000'): + with self.subTest(invalid=invalid): + ledger['attempts'][0]['finished_at_unix'] = invalid + with self.assertRaises(RuntimeError): effective_cooldown(ledger, policy) + + +if __name__ == '__main__': unittest.main() diff --git a/research/vehicle_program_v1/code/tests/test_parallel.py b/research/vehicle_program_v1/code/tests/test_parallel.py new file mode 100644 index 0000000..50336f5 --- /dev/null +++ b/research/vehicle_program_v1/code/tests/test_parallel.py @@ -0,0 +1,167 @@ +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +import json +from pathlib import Path +import sys +import tempfile +import threading +import time +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from api_backend import APIBudgetExceeded, APIError, default_config +from api_parallel import SharedBudget, run_parallel, scheduler_config +from artifacts import put, read, sha +from backends import ScriptedBackend +from replay import replay +from runner import run +from test_api import MESSAGES, SECRET, response +from test_vehicle import fixture + + +class ParallelTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.config = default_config(self.root / 'tokenizer') + self.config['limits'].update(min_interval_seconds=.000001, max_estimated_cny=4., max_http_requests=100) + self.scheduler = scheduler_config() + self.body = {'messages': MESSAGES, 'max_tokens': 512} + + def budget(self, counter=lambda m: 100, **kw): + return SharedBudget(self.root / 'budget.json', self.config, self.scheduler, counter, **kw) + + def test_sliding_token_window_uses_request_size_and_survives_restart(self): + self.scheduler['tokens_per_minute'] = 2000 + now = [100.] + clock = lambda: now[0] + def sleep(seconds): now[0] += seconds + budget = self.budget(clock=clock, sleep=sleep) + for _ in range(2): + entry = budget.reserve(self.body) + budget.finish(entry, 'response_received', {'prompt_tokens': 100, 'completion_tokens': 1}) + self.assertEqual(now[0], 100.) + resumed = SharedBudget(budget.path, self.config, self.scheduler, lambda m: 100, clock=clock, sleep=sleep) + third = resumed.reserve(self.body) + self.assertGreaterEqual(third['sent_at_unix'] - 100., 60.1 - 1e-8) + self.assertEqual(len(read(budget.path)['attempts']), 3) + + def test_success_releases_money_reservation_but_unknown_retains_it(self): + budget = self.budget() + entry = budget.reserve(self.body) + self.assertAlmostEqual(budget.accounted_cny(), .054528) + budget.finish(entry, 'response_received', {'prompt_tokens': 100, 'completion_tokens': 10}) + self.assertAlmostEqual(budget.accounted_cny(), .00027) + budget.reserve(self.body) + self.assertAlmostEqual(budget.accounted_cny(), .054798) + restarted = self.budget() + with self.assertRaisesRegex(APIBudgetExceeded, 'shared_backend_stopped'): + restarted.reserve(self.body) + + def test_concurrent_money_reservations_cannot_overspend(self): + self.config['limits']['max_estimated_cny'] = .11 + budget = self.budget() + def reserve(): + try: return budget.reserve(self.body)['number'] + except APIBudgetExceeded: return None + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(lambda _: reserve(), range(20))) + self.assertEqual(sorted(n for n in results if n is not None), [1, 2]) + self.assertLessEqual(budget.accounted_cny(), .11) + + def test_error_stops_all_workers_without_retry_and_keeps_reserve(self): + budget = self.budget() + entry = budget.reserve(self.body) + budget.finish(entry, 'http_429') + with self.assertRaises(APIBudgetExceeded): budget.reserve(self.body) + self.assertAlmostEqual(budget.accounted_cny(), .054528) + + def test_provider_token_estimate_excess_halts(self): + budget = self.budget() + entry = budget.reserve(self.body) + with self.assertRaisesRegex(APIError, 'provider_token_estimate_exceeded'): + budget.finish(entry, 'response_received', {'prompt_tokens': 1000, 'completion_tokens': 1}) + self.assertTrue(budget.stopped) + + def test_parallel_requests_equal_serial_and_resume_replays_without_gold(self): + package, output, serial = self.root / 'package', self.root / 'run', self.root / 'serial' + fixture(package, count=6) + profile = self.root / 'profile.json'; put(profile, self.config) + scripted = ScriptedBackend() + lock = threading.Lock() + active, peak = [0], [0] + def transport(url, key, body, timeout): + with lock: + active[0] += 1; peak[0] = max(peak[0], active[0]) + try: + time.sleep(.01) + return response(scripted(body['messages'])) + finally: + with lock: active[0] -= 1 + original_open = Path.open + def guard(path, *args, **kwargs): + if path.name == 'gold.json': raise AssertionError('inference read gold') + return original_open(path, *args, **kwargs) + with patch.object(Path, 'open', guard): + result = run_parallel(package, output, profile, self.scheduler, counter=lambda m: 100, + key=SECRET, transport=transport) + run(package, serial, backend='mock') + self.assertIsNone(result['halted']) + self.assertEqual(result['committed_count'], 6) + self.assertEqual(result['model_calls'], 24) + self.assertEqual(peak[0], 3) + self.assertEqual(replay(package, output)['completed_sources_replayed'], 6) + before = {p.as_posix(): sha(p) for p in output.glob('sources/*/receipt.json')} + def forbidden(*args): raise AssertionError('completed run called API') + run_parallel(package, output, profile, self.scheduler, resume=True, counter=lambda m: 100, + key=SECRET, transport=forbidden) + self.assertEqual(before, {p.as_posix(): sha(p) for p in output.glob('sources/*/receipt.json')}) + for folder in output.glob('sources/*'): + actual = [read(p)['messages'] for p in sorted((folder / 'calls').glob('*.json'))] + expected = [read(p)['messages'] for p in sorted((serial / 'sources' / folder.name / 'calls').glob('*.json'))] + self.assertEqual(actual, expected) + for path in output.rglob('*.json'): + self.assertNotIn(SECRET, path.read_text(encoding='utf-8')) + self.assertNotIn('SECRET_GOLD', json.dumps([read(p) for p in output.glob('sources/*/calls/*.json')] )) + changed = deepcopy(self.scheduler); changed['workers'] = 2 + with self.assertRaisesRegex(ValueError, 'run identity differs'): + run_parallel(package, output, profile, changed, resume=True, key=SECRET, counter=lambda m: 100) + + def test_explicit_selection_and_wall_budget_leave_unstarted_rows_pending(self): + package, output = self.root / 'package', self.root / 'run' + fixture(package, count=3) + profile = self.root / 'profile.json'; put(profile, self.config) + result = run_parallel(package, output, profile, self.scheduler, positions=[2], counter=lambda m: 100, + key=SECRET, transport=lambda url,key,body,timeout: response(ScriptedBackend()(body['messages']))) + self.assertEqual(result['committed_count'], 1) + self.assertTrue((output / 'sources/0002/receipt.json').exists()) + self.assertFalse((output / 'sources/0001').exists()) + self.assertEqual(result['pending_count'], 2) + now = [100.] + budget = self.budget(clock=lambda: now[0]) + now[0] += 1801 + with self.assertRaisesRegex(APIBudgetExceeded, 'wall_budget_exhausted'): budget.reserve(self.body) + + def test_progress_reads_ledger_under_its_writer_lock_on_windows(self): + package, output = self.root / 'package', self.root / 'run' + fixture(package, count=9) + profile = self.root / 'profile.json'; put(profile, self.config) + captured = [] + def budget_factory(*args, **kwargs): + budget = SharedBudget(*args, **kwargs); captured.append(budget); return budget + def guarded_read(path): + if Path(path).name == 'api-budget.json': + self.assertTrue(captured[0].lock._is_owned(), 'ledger opened without its writer lock') + return read(path) + with patch('api_parallel.SharedBudget', budget_factory), patch('runner.read', guarded_read): + result = run_parallel(package, output, profile, self.scheduler, counter=lambda m: 100, key=SECRET, + transport=lambda url,key,body,timeout: response(ScriptedBackend()(body['messages']))) + self.assertEqual(result['committed_count'], 9) + self.assertIsNone(result['halted']) + + +if __name__ == '__main__': + unittest.main() diff --git a/research/vehicle_program_v1/code/tests/test_vehicle.py b/research/vehicle_program_v1/code/tests/test_vehicle.py new file mode 100644 index 0000000..82f7474 --- /dev/null +++ b/research/vehicle_program_v1/code/tests/test_vehicle.py @@ -0,0 +1,243 @@ +import json +from pathlib import Path +import sys +import tempfile +import subprocess +import unittest +import zipfile +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from artifacts import BUDGET, GENERATION, code_inventory, load_controller, load_study, put, read, sha +from backends import ScriptedBackend +from bundle import bundle +from prepare import project, registry_from, gold_for +from replay import replay +from runner import exclusive_run, run +from score import score_traces, score_run + + +def corpus(): + return [{'id': f'std-{i}', 'category_root_level': 'synthetic', + 'category_leaf_level': 'same' if i < 3 else f'leaf-{i}', + 'sensitivity_level': 'L2' if i == 1 else 'L1', + 'data_description_and_example': f'fictional standard {i}', + 'reference_standard': 'fictional reference'} for i in range(7)] + + +def raw(ident='fixture-1'): + return {'id': ident, 'domain': 'synthetic', 'metadata': { + 'field_name': 'OPAQUE', 'field_description': 'fictional field description', + 'table_name': 'FORBIDDEN_TABLE', 'gold': 'SECRET_GOLD'}, + 'classification': {'category_root_level': 'synthetic', 'category_leaf_level': 'same'}, + 'grading': {'sensitivity_level': 'L1'}, 'data_description_and_example': 'SECRET_GOLD', + 'reference_standard': 'SECRET_GOLD'} + + +def fixture(path, count=3): + rows = [raw(f'fixture-{n}') for n in range(count)] + registry = registry_from(corpus()) + study = {'expected_ids': [r['id'] for r in rows], 'rows': [project(r) for r in rows], + 'registry': registry, 'skill_pool': 'F66', 'budget': BUDGET, 'generation': GENERATION} + put(path / 'inference/study.json', study) + put(path / 'evaluation/gold.json', {'answers': gold_for(rows, registry)}) + put(path / 'manifest.json', {'code': code_inventory(), 'study_sha256': sha(path / 'inference/study.json'), + 'gold_sha256': sha(path / 'evaluation/gold.json')}) + return study + + +class VehicleTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.package = self.root / 'package' + self.study = fixture(self.package) + self.output = self.root / 'run' + + def test_one_to_many_registry_and_gold(self): + self.assertEqual(len(self.study['registry']), 7) + self.assertEqual(len(gold_for([raw()], self.study['registry'])['fixture-1']['compatible_standard_ids']), 2) + changed = raw(); changed['grading']['sensitivity_level'] = 'L2' + self.assertEqual(gold_for([changed], self.study['registry'])['fixture-1']['compatible_standard_ids'], ['std-1']) + + def test_gold_sentinel_does_not_change_actual_requests(self): + builder = load_controller() + transcripts = [] + for secret in ('SECRET_A', 'SECRET_B'): + source = raw(); source['classification'] = {'arbitrary': secret} + source['reference_standard'] = secret; source['metadata']['gold'] = secret + row = project(source) + messages = [] + def generate(value): + messages.append(value) + return ScriptedBackend()(value) + trace = builder(generate, self.study['registry'], [row], BUDGET, skill_pool='F66').run( + {'source_id': row['id'], 'domain': row['domain'], 'field_name': row['metadata']['field_name']}) + self.assertTrue(trace.terminated) + text = json.dumps(messages) + self.assertNotIn(secret, text) + self.assertNotIn('FORBIDDEN_TABLE', text) + transcripts.append(messages) + self.assertEqual(*transcripts) + + def test_metadata_lookup_does_not_invent_table_context(self): + load_controller() + from pool_metadata import PoolMetadata + from agent.orchestration.contracts import MetadataRecord + row = self.study['rows'][0] + lookup = PoolMetadata([row]) + record = MetadataRecord.from_mapping({'source_id': row['id'], 'domain': row['domain'], 'field_name': 'OPAQUE'}) + context = lookup.table_fields(record) + self.assertEqual(context['fields'], []) + self.assertEqual(context['missing_locators'], ['database_name', 'table_name']) + + def test_unknown_program_is_bounded_and_not_autofinalized(self): + calls = [] + def invalid(messages): + calls.append(messages) + return '{"program":[{"tool":"NONEXISTENT","arguments":{}}]}' + row = self.study['rows'][0] + trace = load_controller()(invalid, self.study['registry'], self.study['rows'], BUDGET, skill_pool='F66').run( + {'source_id': row['id'], 'domain': row['domain'], 'field_name': 'OPAQUE'}) + self.assertFalse(trace.terminated) + self.assertEqual(len(calls), BUDGET['max_planner_attempts']) + + def test_run_does_not_open_gold(self): + original = Path.open + def guard(path, *args, **kwargs): + if path.name == 'gold.json': + raise AssertionError('inference opened gold') + return original(path, *args, **kwargs) + with patch.object(Path, 'open', guard): + result = run(self.package, self.output) + self.assertEqual(result['committed_count'], 3) + self.assertEqual(result['terminated_count'], 3) + self.assertFalse(result['quality_claim']) + self.assertEqual(replay(self.package, self.output)['completed_sources_replayed'], 3) + + def test_resume_reuses_exact_artifacts_and_does_not_initialize_when_done(self): + first = run(self.package, self.output, max_samples=1) + self.assertEqual(first['pending_count'], 2) + original_sha = sha(self.output / 'sources/0001/receipt.json') + result = run(self.package, self.output, resume=True) + self.assertEqual(result['pending_count'], 0) + self.assertEqual(original_sha, sha(self.output / 'sources/0001/receipt.json')) + def forbidden(): + raise AssertionError('backend initialized for completed run') + self.assertEqual(run(self.package, self.output, resume=True, factory=forbidden)['committed_count'], 3) + + def test_interrupted_source_is_preserved_and_not_rerolled(self): + run(self.package, self.output, max_samples=1) + folder = self.output / 'sources/0002' + put(folder / 'started.json', {'partial': True}) + partial_sha = sha(folder / 'started.json') + run(self.package, self.output, resume=True) + self.assertEqual(read(folder / 'receipt.json')['status'], 'interrupted') + self.assertEqual(sha(folder / 'started.json'), partial_sha) + self.assertFalse((folder / 'trace.json').exists()) + score = score_run(self.package, self.output) + self.assertEqual(score['denominator'], 3) + self.assertFalse(score['per_id']['fixture-1']['terminated']) + + def test_saved_trace_tampering_rejected(self): + run(self.package, self.output, max_samples=1) + path = self.output / 'sources/0001/trace.json' + path.write_text('{}', encoding='utf-8') + with self.assertRaisesRegex(ValueError, 'receipt'): + run(self.package, self.output, resume=True) + + def test_changed_study_rejected_before_backend(self): + path = self.package / 'inference/study.json' + path.write_text('{}', encoding='utf-8') + with self.assertRaisesRegex(ValueError, 'study identity'): + run(self.package, self.output) + + def test_changed_run_identity_rejected(self): + run(self.package, self.output, max_samples=1) + path = self.output / 'run.json' + value = read(path); value['backend'] = 'local_official_base' + path.write_text(json.dumps(value), encoding='utf-8') + with self.assertRaisesRegex(ValueError, 'run identity'): + run(self.package, self.output, resume=True) + + def test_backend_failure_halts_and_remaining_sources_keep_denominator(self): + def failing(messages): + raise RuntimeError('synthetic backend failure') + result = run(self.package, self.output, factory=lambda: failing) + self.assertEqual(result['committed_count'], 1) + self.assertEqual(result['model_calls'], 1) + self.assertEqual(result['halted'], 'backend_error') + score = score_run(self.package, self.output) + self.assertEqual(score['denominator'], 3) + self.assertEqual(score['joint_accuracy'], 0.) + self.assertEqual(run(self.package, self.output, resume=True)['error_count'], 1) + + def test_score_checks_grade_against_selected_standard(self): + ids = self.study['expected_ids'] + answers = read(self.package / 'evaluation/gold.json')['answers'] + def trace(key, level): + return {'terminated': True, 'stop_reason': 'validated_terminal', + 'final_result': {'category': key, 'grading_level': level}} + result = score_traces(ids, {ids[0]: trace('std-2', 'L1'), ids[1]: trace('std-1', 'L2'), + ids[2]: trace('std-1', 'L1')}, answers, self.study['registry']) + self.assertEqual(result['category_correct'], 2) + self.assertEqual(result['grading_correct'], 1) + self.assertEqual(result['joint_correct'], 1) + self.assertEqual(result['terminated'], 2) + + def test_unlabelled_or_unknown_prediction_never_scores(self): + ids = self.study['expected_ids'] + unknown = {'terminated': True, 'stop_reason': 'validated_terminal', + 'final_result': {'category': 'UNKNOWN', 'grading_level': 'L1'}} + answers = read(self.package / 'evaluation/gold.json')['answers'] + result = score_traces(ids, {ids[0]: unknown}, answers, self.study['registry']) + self.assertEqual(result['joint_correct'], 0) + self.assertEqual(result['denominator'], 3) + self.assertEqual(result['missing_or_failed_traces'], 2) + + def test_run_lock_excludes_concurrent_writer(self): + self.output.mkdir() + with exclusive_run(self.output): + with self.assertRaises(OSError): + with exclusive_run(self.output): + pass + + def test_gold_hash_checked_only_by_scorer(self): + run(self.package, self.output) + (self.package / 'evaluation/gold.json').write_text('{}', encoding='utf-8') + with self.assertRaisesRegex(ValueError, 'gold identity'): + score_run(self.package, self.output) + + def test_inference_bundle_is_portable_and_excludes_gold(self): + archive = self.root / 'deployment.zip' + bundle(self.package, archive) + extracted = self.root / 'deployed' + with zipfile.ZipFile(archive) as z: + self.assertNotIn('package/evaluation/gold.json', z.namelist()) + self.assertEqual({n for n in z.namelist() if n.startswith('package/')}, + {'package/manifest.json', 'package/inference/study.json'}) + z.extractall(extracted) + completed = subprocess.run([sys.executable, '-B', str(extracted / 'code/runner.py'), + '--package', str(extracted / 'package'), '--output', str(self.root / 'deployed-run'), + '--backend', 'mock'], capture_output=True, text=True, timeout=30) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(json.loads(completed.stdout)['committed_count'], 3) + + def test_top5_keeps_standard_identity_and_macro_counts_unsupported_predictions(self): + ids = self.study['expected_ids'] + answers = read(self.package / 'evaluation/gold.json')['answers'] + trace = {'terminated': True, 'stop_reason': 'validated_terminal', + 'final_result': {'category': 'std-3', 'grading_level': 'L1'}, + 'programs': [{'steps': [{'status': 'success', 'call': {'tool': 'SELECT_CATEGORY_TOPK'}, + 'data': {'standard_candidates': [{'category': f'std-{i}'} for i in range(5)]}}]}]} + value = score_traces(ids, {ids[0]: trace}, answers, self.study['registry']) + self.assertEqual(value['top5_first_hit'], 1) + self.assertEqual(value['top5_compatible_entry_any_hit'], 1) + self.assertEqual(value['category_correct'], 0) + self.assertEqual(value['registry_leaf_paths'], 5) + + +if __name__ == '__main__': + unittest.main() diff --git a/research/vehicle_program_v1/code/upstream.json b/research/vehicle_program_v1/code/upstream.json new file mode 100644 index 0000000..b533da9 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream.json @@ -0,0 +1,124 @@ +{ + "repository": "https://github.com/DocAILab/DataClassifyGrading", + "commit": "9473b2c2a330b7f48dc0c1831bda11e36d428379", + "runtime_origin": "research/finance_program_v1/evaluation/runtime", + "files": { + "upstream/backend/download_model.py": "5e435d6f87d920d58ff6d305afe61be3a98fec8d12b6f79986072a9a208a8edd", + "upstream/backend/local_smoke.py": "6517f5bc9bbb8c5fa4bc7e9100d1c4cf65a7829b4d42543e837d5798ad81f8ef", + "upstream/runtime/program_loop/__init__.py": "8ebc1b9a3937a1848c199126edf97e5862741ede120a491e87a8473d6c0f7c46", + "upstream/runtime/program_loop/__main__.py": "935a1c1166b0c1ea35a82256345000bf2c73ded718d77773bc27a71ecce28f7d", + "upstream/runtime/program_loop/_vendor.py": "cfb187e38a0b8acd86e83476ee24f3aa02928c86fd3deb532f752ae8784bbf80", + "upstream/runtime/program_loop/catalog_context.py": "7e08a466d701eb8b67733629c7af4723f9baf1eedf16ba768a78ba4d6d0e10cb", + "upstream/runtime/program_loop/cli.py": "82d3841baa287c2d5e0cf519ae2532a4fd5a4d79e89d907e29ad029c17d59335", + "upstream/runtime/program_loop/contracts.py": "04f584b9acfb033fb1cbd38c8f08595f46bd3d4c57a55495fc282920bd9169a4", + "upstream/runtime/program_loop/controller.py": "a3a8ad54e30a0fb623cf9524bc766c4ee6d334de84648aaec77addda3f0c93e1", + "upstream/runtime/program_loop/execution.py": "960800db0c9f49ca7db61c9f20fa3967f6373d2df000da1a7eb253d150efa72e", + "upstream/runtime/program_loop/failure_policy.py": "92282fb6456a3df91deab50c141964abbad68741e948116ef0e4a3ad8057762f", + "upstream/runtime/program_loop/planner.py": "6edeab5d5164663a6e9f1ac5c31848836469f970b7accbeff937e033dc80e22b", + "upstream/runtime/program_loop/planning_contracts.py": "b0cd81f6c0aac6c0ef09bf460176522683f9612b908de06f0841abd16958e312", + "upstream/runtime/program_loop/prompt_encoding.py": "20862c2334a809ac06629058495eaaf4e94a601345dbcdbd5b6d573df07ebebc", + "upstream/runtime/program_loop/recovery_context.py": "0d67d47e51f55edc4bbffe1164327c7f74d7aff1b629dae6df3e5c9c201af2f7", + "upstream/runtime/program_loop/rejection_context.py": "fd71ad424ff4a201b01f4db7276f2d958fd94eba7aaf26deb16cc54dbcb9de1f", + "upstream/runtime/program_loop/validation.py": "c96220f85cc14375ce6171dc59c2ff5d46389017ca9549cb41ff00cda82b51eb", + "upstream/runtime/runtime_identity.py": "a183d837ec5f666ea8bd0c1965ff0004d63be74cd1c369505cd7e6ddcb2dccd1", + "upstream/runtime/tests/test_grounding.py": "8bc06990233ef500ea3b7cd572c65b0dd7ca14565c5269ec25f2a0ced649e716", + "upstream/runtime/tests/test_integration.py": "c38e9d6b4c0cae512637c050c951770d5bca35df519927ab2ce01905941946f9", + "upstream/runtime/vendor/common.py": "c79a02e987a169436796f438a415d7c309c0e0cf50bb447e493cd68cb9301fa4", + "upstream/runtime/vendor/comparison_output.py": "ed7b3bb552aeaff329ab4531bfa4fbd93772bf560475dc4d3fb848c12bee68e8", + "upstream/runtime/vendor/evidence_roles.py": "6b76fbc8696a70d9ed5723a4ba71edc4de540f29e883d5ee5f99d9f806536064", + "upstream/runtime/vendor/grounded_semantics.py": "4f2f764e2256ad1dda8716f42277e688fe710fbb607e310cdbe57995e43c254c", + "upstream/runtime/vendor/pool_availability.py": "c6fa8e23a678b4451ef78d301efb347507fb0513a9d18bfda76eeefc824d70aa", + "upstream/runtime/vendor/pool_bindings.py": "3c0c65e78f7bcdc5bb81003b455fd957dba5c4abdbfdaa20b8fddb3b155e07db", + "upstream/runtime/vendor/pool_catalog.py": "bd19db2c908d41120498eae4afc26e30726b60ddf7fbf834e77ca6251e8c8a82", + "upstream/runtime/vendor/pool_metadata.py": "0705318048785fd34edd970274145dec359269e5e30756a996eb156e168e3a8a", + "upstream/runtime/vendor/pool_runtime.py": "ae5f3927e4b4faa5a7808c9ae9975af9ae9ef7d7f2f347fd034f34cadf2e9f45", + "upstream/runtime/vendor/reference_skills_63.json": "a63b6ee85f983caf9f55d537fe85d24a283a860ccb0985c8a7fce8b1466e3caf", + "upstream/runtime/vendor/runtime/code/agent/__init__.py": "c2ee5b379bec6f6381e0651bee6078e38ad6ca294f497d40b6d27e583fd1492e", + "upstream/runtime/vendor/runtime/code/agent/orchestration/__init__.py": "64f470fa2d85450ebbd8819a66c63064bcdbf993c829d248c64315285e7aca73", + "upstream/runtime/vendor/runtime/code/agent/orchestration/agent_loop.py": "176bc72eebd17a7d2e45276654c49cbd5b1828306e11d2579faffe143eebcade", + "upstream/runtime/vendor/runtime/code/agent/orchestration/artifact_projection.py": "d71939b3e4b04b83d33f7aedbc3173cd44043b8a55d44c149c46f35d5aa3f7d8", + "upstream/runtime/vendor/runtime/code/agent/orchestration/catalog.py": "66ceb63dd02030cecb3965247a671da64c4d29724ece9a5eb52961a0647c8e6a", + "upstream/runtime/vendor/runtime/code/agent/orchestration/category_catalog.py": "baf7b0b234822060b48dded26f00d526654749f1384c3e23f9eb07ddc8fa0a58", + "upstream/runtime/vendor/runtime/code/agent/orchestration/contracts.py": "19ed69938bdc517be14625e0ef886945dc4483faeb6a8260fdfcacaffa83f233", + "upstream/runtime/vendor/runtime/code/agent/orchestration/metadata_lookup.py": "e338e0d804caa20218637074422b08f26b16fdbdd9faff39077a1cb73beb89fc", + "upstream/runtime/vendor/runtime/code/agent/orchestration/metrics.py": "fffb69bd3e8efa149e46e3413b4056457840b9cd8f6798fd5e0871cdb7e3a0b3", + "upstream/runtime/vendor/runtime/code/agent/orchestration/model_skill_executor.py": "6ad63e1de6cb77bbf6d3cc537f907b59e94147b7e58de885d0d0c43bdb31671b", + "upstream/runtime/vendor/runtime/code/agent/orchestration/planner.py": "5be67a873f8fa54c183c9fa381def425b73cc92181fab6079506eb2296fa53d8", + "upstream/runtime/vendor/runtime/code/agent/orchestration/retrieval.py": "fbe140936695f38c6bc6838c5a9a0100ca9e6f98b6e77984e587fdbea28b6bf4", + "upstream/runtime/vendor/runtime/code/agent/orchestration/runtime.py": "887e82115154f043801719b5e292895e3eeef0d25e277748805a0c1dabc39acc", + "upstream/runtime/vendor/runtime/code/agent/orchestration/skill_loader.py": "7528bada337e98b69ef61ccf558574f5940d26fcfc2106530f8c9d0d765964f6", + "upstream/runtime/vendor/runtime/code/agent/orchestration/table_field_lookup.py": "93319c4ee044439f9e25449cec220dfaededa1c6b799816486d19402b569dc22", + "upstream/runtime/vendor/runtime/code/agent/orchestration/topk_selection.py": "a58411d1acb421bd5e5c134f4ceef7ccce76851f684a397446904b7a17b1eb4c", + "upstream/runtime/vendor/runtime/code/agent/orchestration/validator.py": "15fd2b9faca5addf49f4fdf5d816eb2ecb3723c6b800da3fd591cda93b0bedcd", + "upstream/runtime/vendor/runtime/code/script/orchestration/__init__.py": "7c61249a27a9c7ad011f983e9857e3889ae54837d00faf2660fe2f72350b30d6", + "upstream/runtime/vendor/runtime/code/script/orchestration/run_agent_loop_batch.py": "6605815bf479e8fc50a3041af3c2430b3f121740bf38b8e6d497212d4425616d", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_ASSESS_CATEGORY_SUPPORT.md": "85ea64633013f69d582707acf4ab1b32201cf5ebbf7e0356041119d6c08c106a", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_CHECK_EVIDENCE_SUFFICIENCY.md": "f879c21470b9daeb773a80a256a07f7a6f43e38e8452347f128eca6391362f4a", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_COMPARE_CATEGORY_HYPOTHESES.md": "3fa31691df6bb353b4171a8c837da229549f8f4eff827bf502f81cb2318432a5", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_DECIDE_NEXT_EVIDENCE_SOURCE.md": "647a447ae584d92e67c17d2d7040ff48d14cb8d2b5039345d65eb072b5c14fd3", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FINALIZE_CLASSIFICATION.md": "cd66513c1ca4b6728bd217ae57a7ad7cd96257f61443a7af2ac1edd7e746f937", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_CONTEXT_EVIDENCE.md": "b0028a313fa3f81350f07093c5389c7ec0447539119e37930f1f05bceec539d1", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_DATABASE_EVIDENCE.md": "e27e9b2f57b5b1929d5286a10cc15233ab48ba40948fe520fde8853f66bc1e56", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_FIELD_EVIDENCE.md": "6dfef164297cb75a02f64ebbdafecb3f0519f1a5f1992c3bb9a98157b7a7ee54", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_METADATA_EVIDENCE.md": "e3a7e0de334bc39836c3a909edc79233075b0b73f7c30e879d4e087bc94aa6f2", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_STANDARD_EVIDENCE.md": "eddac25de1c3dda0baf876d29a7a32ae4fab980f2ae7ce892def82af3469087a", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_TABLE_EVIDENCE.md": "24c7eb0ea3381b640465e78e41eb2ff43495e7bb3eadcc652e5587c3bd167ab4", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_IDENTIFY_MISSING_EVIDENCE.md": "237695b8b5d6f9a7105902dc32a120297c364ee5f8d1c27f4ead212df110668f", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY.md": "032b758517d3a8535c29d20d8391bd8d27a2ad1d5c7e9f62246578076ddb5197", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY_FROM_TOPK.md": "7af18e11373d633fe4369338336c653b47238c214cb5f1937e0a5b30041f722f", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_GRADING_LEVEL.md": "276b5e1a8b080d9aecec762762939372f40e56c8fb4b1eb92964c53073b6d603", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_FIELD_CONTEXT_CONFLICT.md": "94e552edccadc03cb55a5ab8e54511a296971363bd4aafb30b3ab1385e5a18f3", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_METADATA_CONFLICT.md": "643472dc5b7e7cdb38f3483857acc94d539884efede9b9ed0f4cccc79a037d0b", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_STANDARD_CONFLICT.md": "be9b9c9c879b964acf4b5f7317dcf90c68833a1f98f6cb8ab9f8199460066b25", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CATEGORY_WITH_STANDARD.md": "deb07baab54b052a9323b475c912d7743115c96dd4b1dd0b7c4a80df3e3d0dfc", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CLASSIFICATION_GRADING_CONSISTENCY.md": "ff8ef17c1ce1408496c6634389ecba87f6d282d0264e4ac1ee9a8c2deb12bab1", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_WEIGH_EVIDENCE.md": "c928a03a7a8f2d8b1a28ffb01c100543000a62e4a388a0e60051a4b6815dd43f", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_ACQUIRE_MISSING_METADATA.md": "cc788f17f7c63e1f7974f220eca3becfb186d3df287a35d814a5387964d20ce3", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_CHECK_METADATA_AVAILABILITY.md": "fe27b6f518a98be4e88414d6180180954572f222fec37dfc4591e2b72a05ad5a", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_EXPAND_SELECTED_CONTEXT.md": "323b136804a9db277d6642b71de277ae3fcd7021910645475872b1b018d79a79", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_DATABASE_CONTEXT.md": "0008e4dc6b220757208cb656912ebb609c8230aaf62ce56a33f247dc5879f7cd", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_TABLE_CONTEXT.md": "09b1f136ca45244c603e98aba0bec6fad8997ca2e618eeaeaf7ac1c1de4fb711", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_DESCRIPTIONS.md": "0b5f7d9ced8902a3a7157d6781a3b3d6bc9fb9f2d2c55da99f31977ceb5609b8", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_TYPES.md": "61cfe7efeb0980d22e72c4069e70cb91d1a007c7352b528b600b7efbcfd313e5", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_VALUES.md": "780305d46253a0ab132f35feee6ba420347f351326827c4be781bd4224349504", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_DATABASE_DESCRIPTION.md": "37b2146b9cd669e1dc0cb0a54c0f6ee36dc0157ce1e4e753d96be08a6fdada21", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_DESCRIPTION.md": "d545962070cb10fa90ee1693261d48ee08df616cad4d098bfcd98643100fce7b", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_METADATA.md": "372ce861fcca6138cda43b693a8fc32f85c0ec380b0c275c3cfe613dbe25f629", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_DESCRIPTION.md": "912334b76f95c6f468cf4bab840ae0ec3a262639c59e545db63bfce866710ea0", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_FIELDS.md": "99c01c5869b511754feccfd335017c7f4290683fbcc46ed6355b95f775d605b0", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_SELECT_CONTEXT_FIELDS.md": "aacb11c41507d2c2743e56a7915c9dae9d6b4e2ff3ece6fec6b3d73babeb6683", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_DISAMBIGUATE_FIELD_SEMANTICS.md": "e497bddaf99da95503677a969926921b507b92ad542055dafaa88ee724a235d0", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_DATABASE_ABBREVIATION.md": "2338078f06cd5276b326747fa0b80790d3c638ba369f6579e4ddcff84882eb51", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_FIELD_ABBREVIATION.md": "fc8d4265d620c57c6d181e395aa08a0f19319f86437739037659b29016e15fad", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_TABLE_ABBREVIATION.md": "6ce033822d4d4f4ae7babba7bd0b44fa871476353b81fd86ac6742d266b3a426", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_CONTEXT_FIELDS.md": "297bf09515bbbc0a4b9e6f80fb77c7135b7484c5c428da0868d7a0d0732717cb", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_DESCRIPTION.md": "acc57509cff6330476522f8fc5a7b286b0ede3e477c9806000b54542f89070dc", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_NAME.md": "e427e888ac10d742784d390ad2bdce40d0aec7c9a7abd52b5f5497fc83b866fc", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DOMAIN.md": "6e1760ff387797cb4c171f404f744b6b6b87445969659bf8b37c8bca66de5375", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_DESCRIPTION.md": "fdbd8cf655fdc0ecddafd9ce7a73f4996d7b4698405e87d8a41e134c7d85adac", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_NAME.md": "861a87c3342531c281779ac572c118f92a4c4b9e13a43e5af3eb39620e5e1d39", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_TYPE.md": "93500ee995113eed8ee9d6d8887d6f65f2d4aa6ac52b12697fb8e82982c1b49d", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_VALUE.md": "c0dad806fde006487b38b9d56fc0b5fde6cc2e367b8a46d13adb5c4171ed19ad", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_DESCRIPTION.md": "901cef9276dd30ff1b2e35fb39bbe9138cf42bb4ded47b37fb3afe83c1945ac2", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_NAME.md": "bfd2506982a98615fa40d2a2760893f7634a270a663f65a84547e4a2ec7a4186", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_SYNTHESIZE_METADATA_SEMANTICS.md": "4d64debe0b8657d6c0960eb5e3eebad7aa5125e34e35f66deb38c050f665ec26", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_CONTEXT_TO_STANDARD.md": "debf2fb145c10526e4952ba7cb322d8a88370ae136d1ae6f0b74b7b9d60734b0", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DATABASE_TO_STANDARD.md": "5bdb03e45720da5ff53297fae7ce476e6fd46f3af6277e8be147a87033917630", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DESCRIPTION_TO_STANDARD.md": "ec00677a4433b6110f529eafc28d5c0272746f186121ee563ab86eb8a3a9a6ec", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_FIELD_TO_STANDARD.md": "19c1e2c28c4df124d9ef56501635c31443552cc3d103d727b927db78891eb53f", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_TABLE_TO_STANDARD.md": "7fa123b49aca5ec7f2525d3697e0108aaf3d85f61c966eb9492a19b657e2a300", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_VALUE_TO_STANDARD.md": "d71db5f6adcf269897e8c9b662cc31edc18b7a7727db7a241624e319a390588e", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_COMPARE_STANDARD_CANDIDATES.md": "93213f34a0d765acb7a4925097fe6691c1e27aed539baa72893abf94872ec39e", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_REFINE_STANDARD_QUERY.md": "51c1ead62e9d83c194ac1fc24c8f43b9879497477b79054e03384b6ded980420", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_RESEARCH_STANDARD.md": "05ccd5f271a060ed497242c494c34d8ce3cac22f03e6349d0d6edfe0a4b303ac", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD.md": "dcbd3841440dc7bf0ee22c02b9ba1014fff1fa6a68900f270bb8cbb40aa9074c", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_CONTEXT.md": "6c1c9ad8fd7036ddf2504e30bc4f920d9754c8c3de68f2ff5263f760630d839e", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_DATABASE.md": "fc38fe4bccc6637de6abe3b8434f7d069d42b6eabf1a8dc29b98cfcdd443a362", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_FIELD.md": "98862fb615e97f189420455219b484ef990e0f4768f7ba4e49983e237d3998f2", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_TABLE.md": "ae067ae1570ad6e24e5c43a08aa4057223923b45e2593525255419144847ebd0", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SELECT_CATEGORY_TOPK.md": "c0dac66df87884c2ecd0e6dbfe2af1027b522a1653972ffa4d79e8a957a9bb74", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SUMMARIZE_STANDARD_EVIDENCE.md": "ea3ee35fa9ddde244353f533a527bf25674ffb711218f74c8960862183bd664b", + "upstream/runtime/vendor/skill_reference.json": "752ed72d4ed6e987542197dfff40786ebd4487bcf6b1042cbf769d1fed2392b8", + "upstream/runtime/vendor/study.py": "d8c78d387e00b13762fb509d23e147210b0016931879c4af7089c593d6eddbb3" + } +} diff --git a/research/vehicle_program_v1/code/upstream/backend/download_model.py b/research/vehicle_program_v1/code/upstream/backend/download_model.py new file mode 100644 index 0000000..7a7e10f --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/backend/download_model.py @@ -0,0 +1,118 @@ +"""Pinned, token-free official checkpoint download; never executes remote code.""" +import hashlib +import json +from pathlib import Path +import re + +REPO = 'Qwen/Qwen3.5-9B' +MODEL_DIR = Path(__file__).resolve().parent / 'models' / 'Qwen3.5-9B' +ALLOWED = {'config.json', 'generation_config.json', 'tokenizer.json', 'tokenizer_config.json', + 'special_tokens_map.json', 'chat_template.json', 'chat_template.jinja', 'vocab.json', + 'merges.txt', 'added_tokens.json', 'model.safetensors.index.json'} + + +def write_json(path, value): + with Path(path).open('x', encoding='utf-8') as stream: + json.dump(value, stream, ensure_ascii=False, indent=2, allow_nan=False) + stream.write('\n') + + +def sha256(path): + digest = hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda: stream.read(1024 * 1024), b''): + digest.update(block) + return digest.hexdigest() + + +def allowed(name): + return isinstance(name, str) and (name in ALLOWED or re.fullmatch(r'model(?:\.safetensors)?(?:-\d+-of-\d+)?\.safetensors', name) is not None) + + +def validate_identity(identity): + if identity.get('repo_id') != REPO or not re.fullmatch('[0-9a-f]{40}', identity.get('revision', '')): + raise ValueError('invalid official model identity') + entries = identity.get('files', []) + names = [e.get('path') for e in entries] + if not entries or len(set(names)) != len(names) or not all(allowed(n) for n in names): + raise ValueError('invalid model identity file inventory') + if 'config.json' not in names or 'tokenizer.json' not in names or not any(n.endswith('.safetensors') for n in names): + raise ValueError('incomplete model identity') + if any(type(e.get('bytes')) is not int or e['bytes'] < 0 for e in entries): + raise ValueError('model identity requires expected file sizes') + if any(e.get('expected_sha256') is not None and not re.fullmatch('[0-9a-f]{64}', e['expected_sha256']) for e in entries): + raise ValueError('invalid model identity expected hash') + + +def download_model(model_dir=MODEL_DIR, *, api=None, snapshot=None): + model_dir = Path(model_dir) + model_dir.mkdir(parents=True, exist_ok=True) + manifest_path = model_dir / 'model-manifest.json' + if manifest_path.exists(): + raise FileExistsError(f'completed manifest already exists: {manifest_path}') + identity_path = model_dir / 'download-identity.json' + if api is None or snapshot is None: + from huggingface_hub import HfApi, snapshot_download + api = api or HfApi(token=False) + snapshot = snapshot or snapshot_download + if identity_path.exists(): + identity = json.loads(identity_path.read_text(encoding='utf-8')) + else: + if any(model_dir.iterdir()): + raise ValueError('unidentified existing checkpoint directory') + info = api.model_info(REPO, files_metadata=True, token=False) + identity = {'repo_id': REPO, 'revision': info.sha, + 'files': [{'path': f.rfilename, 'bytes': f.size, + 'expected_sha256': getattr(getattr(f, 'lfs', None), 'sha256', None)} + for f in info.siblings if allowed(f.rfilename)]} + validate_identity(identity) + write_json(identity_path, identity) + validate_identity(identity) + snapshot(repo_id=REPO, revision=identity['revision'], local_dir=str(model_dir), + allow_patterns=[e['path'] for e in identity['files']], token=False) + files = [] + for expected in identity['files']: + path = model_dir / expected['path'] + if path.is_symlink() or not path.is_file() or path.stat().st_size != expected['bytes']: + raise ValueError(f'download file size/path mismatch: {expected["path"]}') + actual_hash = sha256(path) + if expected.get('expected_sha256') and actual_hash != expected['expected_sha256']: + raise ValueError(f'download file trusted hash mismatch: {expected["path"]}') + files.append({**expected, 'sha256': actual_hash}) + manifest = {'repo_id': REPO, 'revision': identity['revision'], 'total_bytes': sum(e['bytes'] for e in files), + 'download_identity_sha256': sha256(identity_path), 'files': files} + expected = {e['path'] for e in files} | {'download-identity.json'} + actual = {p.relative_to(model_dir).as_posix() for p in model_dir.rglob('*') if p.is_file() and '.cache' not in p.relative_to(model_dir).parts} + if actual != expected: + raise ValueError('download file inventory mismatch') + write_json(manifest_path, manifest) + verify_model(model_dir) + return manifest + + +def verify_model(model_dir): + """Read-only complete inventory, byte and hash gate before model loading.""" + model_dir = Path(model_dir) + identity_path = model_dir / 'download-identity.json' + identity = json.loads(identity_path.read_text(encoding='utf-8')) + manifest = json.loads((model_dir / 'model-manifest.json').read_text(encoding='utf-8')) + validate_identity(identity) + validate_identity(manifest) + if manifest['revision'] != identity['revision'] or manifest['download_identity_sha256'] != sha256(identity_path): + raise ValueError('model identity hash/revision mismatch') + if [{k: v for k, v in e.items() if k != 'sha256'} for e in manifest['files']] != identity['files']: + raise ValueError('model identity inventory mismatch') + expected = {e['path'] for e in manifest['files']} | {'download-identity.json', 'model-manifest.json'} + actual = {p.relative_to(model_dir).as_posix() for p in model_dir.rglob('*') if p.is_file() and '.cache' not in p.relative_to(model_dir).parts} + if actual != expected: + raise ValueError('model file inventory mismatch') + for entry in manifest['files']: + path = model_dir / entry['path'] + if path.is_symlink() or path.stat().st_size != entry['bytes'] or sha256(path) != entry['sha256'] or (entry.get('expected_sha256') and entry['expected_sha256'] != entry['sha256']): + raise ValueError(f'model file hash/size mismatch: {entry["path"]}') + return manifest + + +if __name__ == '__main__': + result = download_model() + print(json.dumps({'repo_id': result['repo_id'], 'revision': result['revision'], 'total_bytes': result['total_bytes']})) diff --git a/research/vehicle_program_v1/code/upstream/backend/local_smoke.py b/research/vehicle_program_v1/code/upstream/backend/local_smoke.py new file mode 100644 index 0000000..1803754 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/backend/local_smoke.py @@ -0,0 +1,250 @@ +"""Three unlabelled bounded engineering smokes through the frozen Program core.""" +import argparse +from copy import deepcopy +from importlib.metadata import PackageNotFoundError, distribution +import json +import os +from pathlib import Path +import platform +import sys +import time + +from download_model import sha256, verify_model, write_json + +OFFLINE = {'local_files_only': True, 'trust_remote_code': False} +CONTEXT_CAP = 32768 +OUTPUT_CAP = 512 +SEED = 20260928 + + +def generation_metadata(result): + """Bounded, observed per-call fields shared by execution and CPU replay.""" + return {key: deepcopy(result[key]) for key in + ('input_tokens', 'output_tokens', 'truncation_at_budget', 'error') if key in result} + + +class ContextBudgetExceeded(ValueError): + def __init__(self, tokens): + self.input_tokens = tokens + super().__init__(f'context budget exceeded: {tokens}+{OUTPUT_CAP}>{CONTEXT_CAP}; no truncation') + + +def reserve_run(path): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.mkdir() # Atomic directory creation; must precede any CUDA allocation. + return path + + +def smoke_inputs(): + definitions = [('account_balance', '账户余额'), ('contact_phone', '客户联系手机号'), + ('invoice_amount', '发票金额'), ('customer_name', '客户姓名'), + ('order_id', '订单标识'), ('transaction_date', '交易日期')] + registry = {name: {'label': desc, 'description': desc, 'grading_level': 'L2'} for name, desc in definitions} + rows = [{'id': ident, 'domain': '金融', 'metadata': {'field_name': name, 'field_description': desc}} + for ident, name, desc in [('local-balance', 'account_balance', '账户当前余额'), + ('local-phone', 'customer_phone', '客户联系手机号'), + ('local-invoice', 'invoice_amount', '发票金额')]] + return registry, rows + + +def generation_options(config, tokenizer): + eos = getattr(config, 'eos_token_id', None) + if eos is None: + eos = getattr(tokenizer, 'eos_token_id', None) + if eos is None: + raise ValueError('actual model/tokenizer EOS is required') + pad = getattr(tokenizer, 'pad_token_id', None) + if pad is None: + pad = getattr(config, 'pad_token_id', None) + if pad is None: + pad = eos[0] if isinstance(eos, list) else eos + return {'do_sample': False, 'max_new_tokens': OUTPUT_CAP, 'max_time': 90.0, + 'eos_token_id': eos, 'pad_token_id': pad} + + +def encode_messages(tokenizer, messages): + prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=False) + encoded = tokenizer(prompt, return_tensors='pt', truncation=False) + tokens = encoded['input_ids'].shape[1] + if encoded['input_ids'].shape[0] != 1: + raise ValueError('batch size must be one') + if tokens + OUTPUT_CAP > CONTEXT_CAP: + raise ContextBudgetExceeded(tokens) + return encoded + + +def environment(model_path, *, include_comparison_explanation=True): + packages = {} + for package in ('torch', 'transformers', 'accelerate', 'huggingface_hub', 'safetensors'): + try: + packages[package] = distribution(package).metadata.get('Version') + except PackageNotFoundError: + packages[package] = None + root = Path(__file__).resolve().parent + from tools.freeze_revision import environment_identity + identity = environment_identity(root) + return {'include_comparison_explanation': include_comparison_explanation, + 'pid': os.getpid(), 'python': sys.version, 'executable': sys.executable, 'platform': platform.platform(), + 'packages': packages, 'model_path': str(Path(model_path).resolve()), **identity, + 'seed': SEED, 'context_cap': CONTEXT_CAP, 'max_new_tokens': OUTPUT_CAP, 'per_generate_max_time': 90.0, + 'whole_process_timeout_seconds': 900, 'whole_process_timeout_enforced_by': 'external coordinator', + 'inference': 'FP16 cuda:0 batch1 greedy enable_thinking=False'} + + +class Backend: + def __init__(self, model_path, call_dir): + self.call_dir = Path(call_dir) + self.call_dir.mkdir(parents=True, exist_ok=True) + self.calls = [] + self.failures = [] + self.identity = verify_model(model_path) + import torch + from transformers import AutoConfig, AutoModelForImageTextToText, AutoTokenizer + self.torch = torch + if not torch.cuda.is_available(): + raise RuntimeError('CUDA is required; no CPU/offload fallback') + torch.cuda.set_device(0) + free, total = torch.cuda.mem_get_info(0) + if free <= 0: + raise RuntimeError('cuda:0 has no free device memory') + torch.manual_seed(SEED) + torch.cuda.manual_seed_all(SEED) + config = AutoConfig.from_pretrained(str(model_path), **OFFLINE) + self.tokenizer = AutoTokenizer.from_pretrained(str(model_path), **OFFLINE) + if config.model_type != 'qwen3_5': + raise ValueError(f'unsupported official checkpoint config: {config.model_type}') + self.attention = 'sdpa' if hasattr(torch.nn.functional, 'scaled_dot_product_attention') else 'eager' + self.model = AutoModelForImageTextToText.from_pretrained(str(model_path), **OFFLINE, dtype=torch.float16, + device_map={'': 'cuda:0'}, attn_implementation=self.attention) + self.model.eval() + self.parameter_devices = sorted({str(p.device) for p in self.model.parameters()}) + self.parameter_dtypes = sorted({str(p.dtype) for p in self.model.parameters() if p.is_floating_point()}) + if self.parameter_devices != ['cuda:0'] or self.parameter_dtypes != ['torch.float16']: + raise RuntimeError(f'checkpoint must be entirely FP16 cuda:0: {self.parameter_devices}, {self.parameter_dtypes}') + self.options = generation_options(self.model.generation_config, self.tokenizer) + write_json(self.call_dir / 'backend.json', {'repo_id': self.identity.get('repo_id'), 'revision': self.identity['revision'], + 'model_class': type(self.model).__name__, 'config_type': config.model_type, + 'attention_implementation': self.attention, 'parameter_devices': self.parameter_devices, + 'parameter_dtypes': self.parameter_dtypes, 'cuda_device': torch.cuda.get_device_name(0), + 'torch_cuda_version': torch.version.cuda, 'free_bytes_before_load': free, 'total_device_bytes': total, + 'generation': self.options, 'offline_options': OFFLINE}) + + def __call__(self, messages): + return self.generate(messages) + + def generate(self, messages): + self.last_generation_metadata = {} + call = self.call_dir / f'call-{len(self.calls) + 1:04d}' + call.mkdir() + self.calls.append(str(call)) + started = time.perf_counter() + result = {'messages': deepcopy(messages), 'raw_text': None, 'input_tokens': None, 'output_tokens': None, + 'input_devices': [], 'parameter_devices': self.parameter_devices, 'parameter_dtypes': self.parameter_dtypes, + 'attention_implementation': self.attention, 'generation': self.options, + 'truncation_at_budget': False, 'error': None} + write_json(call / 'request.json', {'messages': messages, 'generation': self.options}) + try: + encoded = encode_messages(self.tokenizer, messages) + result['input_tokens'] = encoded['input_ids'].shape[1] + encoded = {key: value.to('cuda:0') for key, value in encoded.items()} + result['input_devices'] = sorted({str(value.device) for value in encoded.values()}) + if result['input_devices'] != ['cuda:0']: + raise RuntimeError(f'input tensors must be on cuda:0: {result["input_devices"]}') + self.torch.cuda.reset_peak_memory_stats(0) + self.torch.cuda.synchronize(0) + with self.torch.inference_mode(): + output = self.model.generate(**encoded, **self.options) + self.torch.cuda.synchronize(0) + generated = output[0, result['input_tokens']:] + result['output_tokens'] = len(generated) + result['truncation_at_budget'] = len(generated) >= OUTPUT_CAP + result['raw_text'] = self.tokenizer.decode(generated, skip_special_tokens=True, clean_up_tokenization_spaces=False) + return result['raw_text'] + except Exception as exc: + if isinstance(exc, ContextBudgetExceeded): + result['input_tokens'] = exc.input_tokens + result['error'] = f'{type(exc).__name__}: {exc}' + self.failures.append({'call': str(call), 'error': result['error']}) + raise + finally: + result['seconds'] = time.perf_counter() - started + result['memory'] = {} + result['telemetry_errors'] = {} + for name in ('memory_allocated', 'memory_reserved', 'max_memory_allocated'): + try: + result['memory'][name] = getattr(self.torch.cuda, name)(0) + except Exception as exc: + result['memory'][name] = None + result['telemetry_errors'][name] = f'{type(exc).__name__}: {exc}' + self.last_generation_metadata = generation_metadata(result) + write_json(call / 'result.json', result) + + +def main(argv=None, *, backend_factory=Backend): + parser = argparse.ArgumentParser(description='Three unlabelled local GPU engineering smokes; no accuracy metric') + parser.add_argument('--model', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--include-comparison-explanation', action=argparse.BooleanOptionalAction, default=True) + args = parser.parse_args(argv) + try: + output = reserve_run(args.output) + except Exception as exc: + print(f'{type(exc).__name__}: {exc}', file=sys.stderr) + return 2 + started = time.perf_counter() + summary = {'status': 'initialization_failure', 'terminated_count': 0, 'accepted_plans': 0, + 'planner_attempts': 0, 'skill_steps': 0, 'backend_errors': [], 'samples': []} + try: + write_json(output / 'environment.json', environment(args.model, include_comparison_explanation=args.include_comparison_explanation)) + (output / 'samples').mkdir() + backend = backend_factory(args.model, output / 'calls') + from program_loop import ProgramConfig, build_program_controller + config = ProgramConfig(max_programs=3, max_program_steps=10, max_skill_steps=12, + max_planner_attempts=4, max_identical_calls=1) + registry, rows = smoke_inputs() + write_json(output / 'inputs.json', {'registry': registry, 'rows': rows, 'budget': vars(config), + 'include_comparison_explanation': args.include_comparison_explanation}) + summary['status'] = 'completed' + def generate(messages): + generate.last_generation_metadata = {} + try: + return backend(messages) + except Exception as exc: + summary['backend_errors'].append({'error': f'{type(exc).__name__}: {exc}'}) + raise + finally: + generate.last_generation_metadata = generation_metadata(getattr(backend, 'last_generation_metadata', {})) + for row in rows: + controller = build_program_controller(generate, registry, rows, config, + include_comparison_explanation=args.include_comparison_explanation) + trace = controller.run({'source_id': row['id'], 'domain': row['domain'], 'field_name': row['metadata']['field_name']}) + trace.execution_mode = 'caller supplied backend I/O + real frozen executor; backend evidence in calls/' + trace.write_json(output / 'samples' / (row['id'] + '.json')) + summary['terminated_count'] += int(trace.terminated) + summary['accepted_plans'] += trace.budget['programs'] + summary['planner_attempts'] += trace.budget['planner_attempts'] + summary['skill_steps'] += trace.budget['skill_steps'] + summary['samples'].append({'id': row['id'], 'terminated': trace.terminated, 'stop_reason': trace.stop_reason, + 'budget': trace.budget, 'planner_errors': sum('error' in a for a in trace.planner_attempts), + 'execution_errors': sum(s['status'] != 'success' for p in trace.programs for s in p['steps'])}) + if summary['backend_errors']: + summary['status'] = 'backend_failure' + records = [json.loads(p.read_text(encoding='utf-8')) for p in (output / 'calls').glob('call-*/result.json')] + summary['model_calls'] = len(records) + summary['input_tokens'] = sum(r.get('input_tokens') or 0 for r in records) + summary['output_tokens'] = sum(r.get('output_tokens') or 0 for r in records) + summary['generation_seconds'] = sum(r.get('seconds') or 0 for r in records) + except Exception as exc: + if summary['status'] != 'initialization_failure': + summary['status'] = 'runner_failure' + summary['error'] = f'{type(exc).__name__}: {exc}' + finally: + summary['seconds'] = time.perf_counter() - started + write_json(output / 'summary.json', summary) + print(json.dumps(summary, ensure_ascii=True)) + return 0 if summary['status'] == 'completed' else 2 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/research/vehicle_program_v1/code/upstream/runtime/program_loop/__init__.py b/research/vehicle_program_v1/code/upstream/runtime/program_loop/__init__.py new file mode 100644 index 0000000..44a7fe3 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/program_loop/__init__.py @@ -0,0 +1,5 @@ +"""Public CPU-safe Program orchestration entry point.""" +from .contracts import ProgramConfig, ProgramTrace +from .controller import ProgramController, build_program_controller + +__all__ = ["ProgramConfig", "ProgramTrace", "ProgramController", "build_program_controller"] diff --git a/research/vehicle_program_v1/code/upstream/runtime/program_loop/__main__.py b/research/vehicle_program_v1/code/upstream/runtime/program_loop/__main__.py new file mode 100644 index 0000000..eb53e2f --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/program_loop/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/research/vehicle_program_v1/code/upstream/runtime/program_loop/_vendor.py b/research/vehicle_program_v1/code/upstream/runtime/program_loop/_vendor.py new file mode 100644 index 0000000..6222f14 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/program_loop/_vendor.py @@ -0,0 +1,9 @@ +"""Locate the immutable local snapshot without requiring model libraries.""" +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +VENDOR = ROOT / "vendor" +for path in (VENDOR, VENDOR / "runtime" / "code"): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) diff --git a/research/vehicle_program_v1/code/upstream/runtime/program_loop/catalog_context.py b/research/vehicle_program_v1/code/upstream/runtime/program_loop/catalog_context.py new file mode 100644 index 0000000..a7981f8 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/program_loop/catalog_context.py @@ -0,0 +1,41 @@ +"""Complete P14 contracts and snapshots containing only acquired public state.""" +from copy import deepcopy +from . import _vendor +from pool_catalog import purpose +from .planning_contracts import planning_contract + + +def skill_library(catalog, loader): + return [{"name": spec.name, "purpose": purpose(spec.name), "role": spec.role, + "required_inputs": list(spec.required_inputs), "optional_inputs": list(spec.optional_inputs), + "produces": list(spec.produces), "terminal": spec.terminal, + "contract": loader.load(spec.name).body, + "planning": planning_contract(spec, catalog)} for spec in catalog.values()] + + +def pack_library(library): + """Factor repeated input rules without changing any Skill contract or name.""" + packed = deepcopy(library) + rules = {} + for item in packed: + inputs = item["planning"].pop("inputs") + for key, details in inputs.items(): + rule = {name: value for name, value in details.items() if name != "required"} + if key in rules and rules[key] != rule: + raise ValueError(f"inconsistent planning rule for {key}") + rules[key] = rule + return packed, rules + + +def unpack_library(packed, input_rules): + """Reconstruct original planning inputs for audit and roundtrip tests.""" + library = deepcopy(packed) + for item in library: + required = set(item["required_inputs"]) + item["planning"]["inputs"] = {key: {"required": key in required, **deepcopy(input_rules[key])} + for key in (*item["required_inputs"], *item["optional_inputs"])} + return library + + +def public_state(record, state): + return {"input": {"domain": record.domain, "field_name": record.field_name}, "named_state": deepcopy(state)} diff --git a/research/vehicle_program_v1/code/upstream/runtime/program_loop/cli.py b/research/vehicle_program_v1/code/upstream/runtime/program_loop/cli.py new file mode 100644 index 0000000..bf13728 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/program_loop/cli.py @@ -0,0 +1,72 @@ +"""Local scripted demo, or optional generation from an existing local model.""" +import argparse +import json +from pathlib import Path +import sys + +from . import ProgramConfig, build_program_controller + +DEMO_TOOLS = ("INTERPRET_FIELD_NAME", "SELECT_CATEGORY_TOPK", "INFER_CATEGORY_FROM_TOPK", "INFER_GRADING_LEVEL", "FINALIZE_CLASSIFICATION") + + +def scripted_generate(messages): + system = messages[0]["content"] + if "Global Program Planner" in system: + return json.dumps({"program": [{"tool": tool, "arguments": {}} for tool in DEMO_TOOLS]}) + if "selected Skill INTERPRET_FIELD_NAME." in system: + return json.dumps({"summary": "合成语义", "data": {"field_semantics": "字段表示账户余额"}}) + if "selected Skill SELECT_CATEGORY_TOPK." in system: + return json.dumps({"option_ids": ["C001", "C002", "C003", "C004", "C005"], "summary": "合成候选"}) + if "selected Skill INFER_CATEGORY_FROM_TOPK." in system: + return json.dumps({"option_id": "C001", "summary": "合成选择"}) + raise ValueError("unsupported scripted model request") + + +def local_generate(path): + path = Path(path).resolve() + if not path.is_dir(): raise ValueError(f"local model path does not exist: {path}") + try: + from transformers import AutoModelForCausalLM, AutoTokenizer + except ImportError as exc: + raise RuntimeError("optional local model runtime is unavailable: transformers is not installed") from exc + tokenizer = AutoTokenizer.from_pretrained(str(path), local_files_only=True, trust_remote_code=False) + model = AutoModelForCausalLM.from_pretrained(str(path), local_files_only=True, trust_remote_code=False) + model.eval() + + def generate(messages): + import torch + prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + encoded = tokenizer(prompt, return_tensors="pt") + with torch.inference_mode(): + output = model.generate(**encoded, do_sample=False, max_new_tokens=1024) + return tokenizer.decode(output[0, encoded["input_ids"].shape[1]:], skip_special_tokens=True) + return generate + + +def main(argv=None): + parser = argparse.ArgumentParser(description="Program loop: scripted model I/O + real executor; no accuracy claim") + parser.add_argument("--output", type=Path, help="new JSON output path; refuses overwrite") + parser.add_argument("--local-model", type=Path, help="existing local model directory; local_files_only") + parser.add_argument("--registry", type=Path, help="frozen Registry JSON; optional for synthetic demo") + parser.add_argument("--metadata", type=Path, help="label-free privately scoped metadata JSON") + parser.add_argument("--source-id", default="synthetic-demo", help="private metadata lookup identity") + parser.add_argument("--domain", default="合成金融") + parser.add_argument("--field-name", default="balance") + for key in ("programs", "program-steps", "skill-steps", "planner-attempts"): + parser.add_argument("--max-" + key, type=int, default=getattr(ProgramConfig(), "max_" + key.replace("-", "_"))) + args = parser.parse_args(argv) + try: + if args.output and args.output.exists(): raise FileExistsError(f"output already exists: {args.output}") + generate = local_generate(args.local_model) if args.local_model else scripted_generate + registry = args.registry or {f"cat{i}": {"label": f"合成类别{i}", "description": f"合成定义{i}", "grading_level": "L2"} for i in range(1, 7)} + metadata = args.metadata or [{"id": args.source_id, "domain": args.domain, "metadata": {"field_name": args.field_name}}] + config = ProgramConfig(max_programs=args.max_programs, max_program_steps=args.max_program_steps, max_skill_steps=args.max_skill_steps, max_planner_attempts=args.max_planner_attempts) + controller = build_program_controller(generate, registry, metadata, config) + trace = controller.run({"domain": args.domain, "field_name": args.field_name, "source_id": args.source_id}) + trace.execution_mode = "local model I/O + real executor" if args.local_model else "scripted model I/O + real executor" + if args.output: trace.write_json(args.output) + print(json.dumps({"execution_mode": trace.execution_mode, "terminated": trace.terminated, "stop_reason": trace.stop_reason, "budget": trace.budget, "output": str(args.output) if args.output else None}, ensure_ascii=True)) + return 0 if trace.terminated else 1 + except Exception as exc: + print(f"blocked: {exc}", file=sys.stderr) + return 2 diff --git a/research/vehicle_program_v1/code/upstream/runtime/program_loop/contracts.py b/research/vehicle_program_v1/code/upstream/runtime/program_loop/contracts.py new file mode 100644 index 0000000..42bfcbf --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/program_loop/contracts.py @@ -0,0 +1,59 @@ +"""Run limits and JSON trace; the accepted Program is immutable.""" +from dataclasses import asdict, dataclass, field +import json +import math +from pathlib import Path + +from . import _vendor +from agent.orchestration.contracts import Program, ToolCall + + +def json_snapshot(value): + """Keep valid JSON data exact; represent invalid model I/O explicitly.""" + if value is None or isinstance(value, (str, bool, int)): + return value + if isinstance(value, float) and math.isfinite(value): + return value + if isinstance(value, dict): + return {str(key): json_snapshot(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [json_snapshot(item) for item in value] + return {"invalid_json_type": type(value).__name__, "repr": repr(value)} + + +@dataclass(frozen=True) +class ProgramConfig: + max_programs: int = 4 + max_program_steps: int = 10 + max_skill_steps: int = 20 + max_planner_attempts: int = 8 + max_identical_calls: int = 1 + + def __post_init__(self): + for key, value in asdict(self).items(): + if type(value) is not int or value < 1: + raise ValueError(f"{key} must be a positive integer") + + +@dataclass +class ProgramTrace: + input: dict + include_comparison_explanation: bool = True + execution_mode: str = "caller supplied model I/O + real executor" + planner_attempts: list = field(default_factory=list) + programs: list = field(default_factory=list) + state: dict = field(default_factory=dict) + budget: dict = field(default_factory=lambda: {"programs": 0, "planner_attempts": 0, "skill_steps": 0}) + terminated: bool = False + final_result: dict | None = None + stop_reason: str = "running" + + def to_dict(self): + return json_snapshot(asdict(self)) + + def write_json(self, path): + serialized = json.dumps(self.to_dict(), ensure_ascii=False, indent=2, allow_nan=False) + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x", encoding="utf-8") as stream: + stream.write(serialized + "\n") diff --git a/research/vehicle_program_v1/code/upstream/runtime/program_loop/controller.py b/research/vehicle_program_v1/code/upstream/runtime/program_loop/controller.py new file mode 100644 index 0000000..09924a0 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/program_loop/controller.py @@ -0,0 +1,113 @@ +"""Bounded outer feedback loop; no Planner calls within a Program.""" +from collections import Counter +from collections.abc import Mapping +from copy import deepcopy +import json +from pathlib import Path + +from .contracts import ProgramConfig, ProgramTrace, json_snapshot +from .catalog_context import skill_library +from .execution import ExecutionBridge +from .failure_policy import ComparisonFailureLedger +from .planner import planner_messages +from .validation import parse_program +from .rejection_context import rejected_program_summary +from agent.orchestration.contracts import MetadataRecord + + +def _load(value): + return json.loads(Path(value).read_text(encoding="utf-8-sig")) if isinstance(value, (str, Path)) else deepcopy(value) + + +class ProgramController: + def __init__(self, generate, registry, metadata, config, *, include_comparison_explanation=True, skill_pool="P14"): + self.generate = generate + self.config = config + self.include_comparison_explanation = include_comparison_explanation + self.skill_pool = skill_pool + self.bridge = ExecutionBridge(generate, registry, metadata, config, + include_comparison_explanation=include_comparison_explanation, skill_pool=skill_pool) + self.library = skill_library(self.bridge.catalog, self.bridge.loader) + + def remaining(self, trace): + return {"programs": max(0, self.config.max_programs - trace.budget["programs"]), + "planner_attempts": max(0, self.config.max_planner_attempts - trace.budget["planner_attempts"]), + "skill_steps": max(0, self.config.max_skill_steps - trace.budget["skill_steps"]), + "program_steps": self.config.max_program_steps} + + def run(self, record): + if isinstance(record, MetadataRecord): + record = {"domain": record.domain, "field_name": record.field_name, "source_id": record.source_id} + # Ignore caller optional metadata and labels; this boundary exposes only initial fields. + visible = MetadataRecord.from_mapping({"domain": record.get("domain"), "field_name": record.get("field_name") or record.get("metadata", {}).get("field_name"), "source_id": record.get("source_id")}) + trace = ProgramTrace({"domain": visible.domain, "field_name": visible.field_name}, + include_comparison_explanation=self.include_comparison_explanation) + state = {}; self.bridge.availability.initialize(visible, state) + counts = Counter(); lineage = {"evidence": 0, "candidates": 0, "selection": 0} + comparison_failures = ComparisonFailureLedger() + feedback = None + while not trace.terminated: + remaining = self.remaining(trace) + exhausted = next((key for key in ("skill_steps", "programs", "planner_attempts") if remaining[key] == 0), None) + if exhausted: + trace.stop_reason = {"skill_steps": "skill_budget_exhausted", "programs": "program_budget_exhausted", "planner_attempts": "planner_budget_exhausted"}[exhausted] + break + messages = planner_messages(visible, state, self.library, remaining, feedback, skill_pool=self.skill_pool) + trace.budget["planner_attempts"] += 1 + attempt = {"attempt": trace.budget["planner_attempts"], "messages": deepcopy(messages), "raw_response": None, "status": "error"} + trace.planner_attempts.append(attempt) + try: + raw = self.generate(messages) + attempt["raw_response"] = json_snapshot(raw) + attempt["status"] = "rejected" + plan = parse_program(raw, self.bridge.catalog, state, self.config.max_program_steps) + except Exception as exc: + attempt["error"] = str(exc) + feedback = {"status": "planner_" + attempt["status"], "error": str(exc), "remaining_budget": self.remaining(trace)} + if attempt['status'] == 'rejected': + summary = rejected_program_summary(raw, self.bridge.catalog, self.config.max_program_steps) + if summary is not None: + feedback['rejected_program'] = summary + continue + attempt["status"] = "accepted" + trace.budget["programs"] += 1 + observation = {"program_index": trace.budget["programs"], "plan": plan.to_dict(), "steps": [], "status": "completed", "failure_index": None, "terminated": False, "invalidations": []} + trace.programs.append(observation) + for index, call in enumerate(plan.steps): + if self.remaining(trace)["skill_steps"] == 0: + observation.update(status="budget_exhausted", failure_index=index) + break + trace.budget["skill_steps"] += 1 + step = self.bridge.step(call, visible, state, counts, lineage, comparison_failures) + step["step_index"] = index + observation["steps"].append(step) + observation["invalidations"].extend(step["invalidations"]) + if step["status"] != "success": + observation.update(status="failed", failure_index=index) + break + if self.bridge.catalog[call.tool].terminal: + trace.terminated = True + trace.final_result = deepcopy(step["data"]["final_result"]) + trace.stop_reason = "validated_terminal" + observation.update(status="terminated", terminated=True) + break + observation["remaining_budget"] = self.remaining(trace) + observation["lineage"] = dict(lineage) + feedback = deepcopy(observation) + trace.state = deepcopy(state) + return trace + + +def build_program_controller(generate, registry_entries, metadata_rows, config=None, *, include_comparison_explanation=True, skill_pool="P14"): + if not callable(generate): raise TypeError("generate must be callable") + if skill_pool not in ("P14", "F66"): raise ValueError("skill_pool must be P14 or F66") + registry = _load(registry_entries) + if isinstance(registry, Mapping) and "categories" in registry: + registry = {item["category_id"]: {k: v for k, v in item.items() if k not in ("category", "category_id")} for item in registry["categories"]} + if not isinstance(registry, Mapping) or not registry: raise ValueError("nonempty frozen Registry mapping is required") + for category, entry in registry.items(): + if not isinstance(category, str) or not category or not isinstance(entry, Mapping) or not isinstance(entry.get("grading_level"), str) or not entry["grading_level"]: + raise ValueError("Registry entries require category and frozen grading_level") + config = ProgramConfig(**config) if isinstance(config, Mapping) else config or ProgramConfig() + return ProgramController(generate, registry, _load(metadata_rows), config, + include_comparison_explanation=include_comparison_explanation, skill_pool=skill_pool) diff --git a/research/vehicle_program_v1/code/upstream/runtime/program_loop/execution.py b/research/vehicle_program_v1/code/upstream/runtime/program_loop/execution.py new file mode 100644 index 0000000..ac0ad10 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/program_loop/execution.py @@ -0,0 +1,105 @@ +"""Thin bridge to real frozen binding, executor, publication and terminal guards.""" +from copy import deepcopy +from dataclasses import asdict +import json +from types import SimpleNamespace + +from . import _vendor +from .contracts import json_snapshot +from .failure_policy import COMPARISON +from .validation import SELECTION, GRADE +from agent.orchestration.agent_loop import LoopConfig, PolicyDecision +from agent.orchestration.category_catalog import CategoryCatalog +from agent.orchestration.runtime import Observation +from agent.orchestration.skill_loader import SkillLoader +from agent.orchestration.validator import resolve_reference +from pool_catalog import pool_catalog +from pool_metadata import PoolMetadata +from pool_runtime import PoolController, PoolExecutor, publish_state, SEMANTIC_KEYS, ALIGNMENT_KEYS +from pool_metadata import FIELDS +import pool_availability + + +class ExecutionBridge: + def __init__(self, generate, registry, rows, config, *, include_comparison_explanation=True, skill_pool="P14"): + self.catalog = pool_catalog(skill_pool) + self.loader = SkillLoader(_vendor.VENDOR / "runtime" / "skills" / "full_pool", self.catalog, strict_catalog=False) + self.loader.validate_all() + self.executor = PoolExecutor(generate, self.loader, PoolMetadata(rows), CategoryCatalog(registry), + include_comparison_explanation=include_comparison_explanation) + self.guard = PoolController(policy=SimpleNamespace(missing_awareness=True), executor=self.executor, + catalog=self.catalog, skill_loader=self.loader, config=LoopConfig(recover_ready_finalization=False), + registry_entries=registry, approved_levels={e["grading_level"] for e in registry.values()}, state_publication=publish_state) + self.availability = pool_availability + self.max_identical_calls = config.max_identical_calls + + @staticmethod + def _drop(state, keys): + removed = sorted(keys & set(state)) + for key in removed: + state.pop(key, None) + state.get("_provenance", {}).pop(key, None) + return removed + + def step(self, call, record, state, counts, lineage, comparison_failures=None): + result = {"call": call.to_dict(), "status": "blocked", "binding": None, "invalidations": []} + canonical = None + comparison_fingerprint = None + try: + spec = self.catalog[call.tool] + decision = self.guard._normalize_policy_decision(PolicyDecision(call, ""), record, state, spec) + canonical = decision.call + result["binding"] = decision.metadata["binding"] + resolved = {k: resolve_reference(v, record, state)[1] for k, v in canonical.arguments.items()} + identity = json.dumps({"call": {"tool": call.tool, "arguments": resolved}, "lineage": lineage}, sort_keys=True, ensure_ascii=False, allow_nan=False) + if counts[identity] >= self.max_identical_calls: + raise ValueError("identical successful call blocked for current evidence lineage") + correction_context = None + if call.tool == COMPARISON and comparison_failures is not None: + inputs = self.executor.comparison_inputs(record, state, resolved) + comparison_fingerprint, correction_context = comparison_failures.before_call(inputs) + result["status"] = "error" + if call.tool == COMPARISON: + observation = self.executor.execute(canonical, record, state, self.loader.load(call.tool), + correction_context=correction_context) + else: + observation = self.executor.execute(canonical, record, state, self.loader.load(call.tool)) + # Normalize/availability update is transactional with publication. + updated = deepcopy(state) + observation = self.guard._normalize_observation(canonical, observation, updated) + self.executor._validate_deterministic(observation, spec) + result.update(asdict(observation)) + if observation.status == "success": + changed = {key for key, value in observation.data.items() if state.get(key) != value} + evidence = changed & (set(SEMANTIC_KEYS) | set(ALIGNMENT_KEYS) | set(FIELDS) + | {"field_metadata", "acquired_metadata", "table_fields", "selected_context_fields", + "context_fields", "context_field_descriptions", "context_field_types", + "context_field_values", "table_context", "database_context"}) + invalidated = [] + if evidence or "standard_candidates" in changed: + invalidated.extend(self._drop(updated, SELECTION)) + elif "category" in changed or "aligned_standard_entry" in changed: + invalidated.extend(self._drop(updated, GRADE)) + if spec.terminal and not self.guard._valid_terminal(observation.data.get("final_result"), updated): + raise ValueError("invalid terminal result rejected by real PoolController") + publish_state(record, updated, observation.data, call.tool) + state.clear(); state.update(updated) + # Global success dedup consumes the input lineage, before incrementing it. + counts[identity] += 1 + if evidence: lineage["evidence"] += 1 + if "standard_candidates" in changed: lineage["candidates"] += 1 + if "category" in changed or "aligned_standard_entry" in changed: lineage["selection"] += 1 + result["invalidations"] = invalidated + else: + state.clear(); state.update(updated) + except Exception as exc: + if result["status"] == "success": result["status"] = "error" + metadata = {**result.get("metadata", {}), **deepcopy(getattr(exc, "metadata", {}))} + result.update(summary=str(exc), data={}, metadata=metadata) + if comparison_fingerprint is not None and result["status"] == "error": + comparison_failures.record_failure(comparison_fingerprint, exc, metadata) + state["comparison_failure_ledger"] = comparison_failures.snapshot() + if canonical is not None and result["status"] == "error": + self.availability.observe(canonical, Observation(call.tool, "error", str(exc), {}), state) + result["lineage"] = dict(lineage) + return json_snapshot(result) diff --git a/research/vehicle_program_v1/code/upstream/runtime/program_loop/failure_policy.py b/research/vehicle_program_v1/code/upstream/runtime/program_loop/failure_policy.py new file mode 100644 index 0000000..574e8a0 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/program_loop/failure_policy.py @@ -0,0 +1,60 @@ +"""Run-local comparison failure facts and limits; never execute a retry.""" +from copy import deepcopy +import hashlib +import json + +from .contracts import json_snapshot + + +COMPARISON = "COMPARE_STANDARD_CANDIDATES" +CORRECTABLE = {"invalid_output", "output_truncated"} +FAILURE_KINDS = CORRECTABLE | {"no_match", "backend_error", "unknown_error"} + + +def content_sha256(value): + serialized = json.dumps(json_snapshot(value), sort_keys=True, ensure_ascii=False, + separators=(",", ":"), allow_nan=False) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() + + +def raw_sha256(raw): + if isinstance(raw, str): + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + if isinstance(raw, bytes): + return hashlib.sha256(raw).hexdigest() + return content_sha256(raw) + + +class ComparisonFailureLedger: + def __init__(self): + self.entries = {} + + def before_call(self, inputs): + """Return identity and actual correction facts, or block exhausted identity.""" + fingerprint = content_sha256(inputs) + entry = self.entries.get(fingerprint) + if entry is None: + return fingerprint, None + failures = entry["failures"] + if any(failure["failure_kind"] == "no_match" for failure in failures): + raise ValueError("comparison blocked: observed no_match for identical effective inputs") + correctable = [failure for failure in failures if failure["failure_kind"] in CORRECTABLE] + if len(correctable) >= 2: + raise ValueError("comparison blocked: explicit correction already failed for identical effective inputs") + return fingerprint, deepcopy(correctable[-1]) if correctable else None + + def record_failure(self, fingerprint, error, metadata): + kind = metadata.get("failure_kind", "unknown_error") + if kind not in FAILURE_KINDS: + kind = "unknown_error" + entry = self.entries.setdefault(fingerprint, {"tool": COMPARISON, + "input_fingerprint": fingerprint, "failure_count": 0, "failures": []}) + entry["failure_count"] += 1 + raw = metadata.get("raw_response") + entry["failures"].append({"failure_kind": kind, "attempt": entry["failure_count"], + "request_sha256": content_sha256(metadata["messages"]) if "messages" in metadata else None, + "raw_sha256": raw_sha256(raw), "raw_response": json_snapshot(raw), + "validation_error": str(error), "generation": json_snapshot(metadata.get("generation", {}))}) + + def snapshot(self): + return deepcopy(list(self.entries.values())) diff --git a/research/vehicle_program_v1/code/upstream/runtime/program_loop/planner.py b/research/vehicle_program_v1/code/upstream/runtime/program_loop/planner.py new file mode 100644 index 0000000..9c986d3 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/program_loop/planner.py @@ -0,0 +1,121 @@ +"""Generate one whole Program per outer attempt.""" +import json +from .catalog_context import public_state, pack_library +from .prompt_encoding import pack_prompt +from .recovery_context import recovery_context +from .validation import canonical_reference +from pool_metadata import present + +SYSTEM = """You are the Global Program Planner for a database-field classification task. +Plan the complete remaining classification task in one ordered Program, not just the next Skill. +Choose the Skills and their order yourself from the complete P14 library. +Plan all dependencies through a successful terminal; selecting candidates alone does not finish classification. +You need not see a future output now: use its documented canonical reference after its producer; execution will bind the actual verified output. +Conditional outputs may be missing or execution may fail; the Program then stops and its factual observation returns to you at the outer boundary. +The parser accepts partial Programs, but that is a permissive syntax boundary, not the planning objective. +Each Program must fit remaining_budget.program_steps and remaining_budget.skill_steps; these are hard upper bounds, not suggested plan lengths. +Use a sufficient dependency path for the remaining task; complete classification does not require using every Skill or every possible evidence source. +Before returning JSON, count the planned steps and omit nonessential operations if it exceeds either bound, without omitting required dependencies or inventing evidence. +Each possible_producers list contains alternative ways to supply that input, not a sequence that must all be executed; choose a sufficient dependency path and still verify its conditional outputs at execution. +all_available_evidence means the verified evidence actually accumulated by the chosen Skills, not every possible evidence source; only a nonempty requires_all list requires every named prerequisite. +Return exactly one JSON object with only a program key whose value is an ordered array of steps. Each step has only tool (the chosen Skill name) and arguments (an object). +No Skill name is mandatory and the controller inserts no steps. +Only domain and field_name are initially visible. A planned metadata retrieval is not evidence. +Prefer arguments:{} so the existing execution-time binder supplies available required inputs. +To use an available optional input, supply its documented canonical reference explicitly; omitted optional inputs are not automatically inserted. +If an optional input is unavailable, omit it; do not supply null placeholders or invent a value. Explicit supplied parameters remain checked. +Use each Skill's required_inputs and optional_inputs with shared_input_rules to distinguish inputs and obtain each canonical_reference. +Follow shared_input_rules explicit_value_policy: canonical_reference_only requires the exact documented reference for every explicit dynamic input. Never handwrite dynamic state values, including objects, lists, null, numbers, or ordinary strings; omit required inputs to use the runtime binder. observed_missing_metadata_type is the sole native explicit-choice exception and the runtime binder verifies it against actual missing_metadata_types. +For state dependencies, choose an appropriate possible_producers Skill before the consumer, unless the input is already in actual state. +Possible producers and conditional_outputs describe static possibilities, not successful execution evidence. Metadata and other outputs are conditional and validated at execution. +Initial bindings are $domain, $metadata.field_name for field_name/target_field, and internal $record for current_record; $record discloses no private identity. +semantic_query is a query input; semantic_evidence is semantic evidence. They have distinct canonical paths and contracts. field_description_semantics does not supply field_semantics. +final_evidence_state requires every prerequisite in shared_input_rules.final_evidence_state.requires_all; no single possible producer guarantees it. +Future references are allowed only after a producing Skill. Explicit references must use the documented canonical path; values are checked against actual state at execution. +Finish only by a successful terminal Skill consuming verified final_evidence_state. No steps after terminal. +Each failure ends the current Program. Use its observation to plan within the remaining global budget. +Recovery context reports execution facts only. Unexecuted tail is historical intent, not required steps. State presence and historical success do not establish current readiness; invalidated outputs must be reacquired or recomputed through your chosen Skills. +Consider the remaining complete task and actual budget when choosing the next whole Program, especially the final available Program: one successful recovery step may still leave classification unfinished. Legal partial Programs remain allowed; no automatic completion or bonus budget is provided. +Never invent metadata, candidates, categories, frozen grades, or ground truth. +For compact planning messages, shared_input_rules applies to every Skill input of the same name; required membership is in each Skill's required_inputs. A {'prompt_ref':'vN'} in current, feedback, or recovery_context means the exact observation in shared_values[vN]; resolve such references recursively. Prompt references are never executable arguments.""" + + +def planner_messages(record, state, library, budget, feedback, *, skill_pool="P14"): + compact_library, input_rules = pack_library(library) + payload = {"task": "Plan the complete remaining classification task using verified evidence and frozen Registry grading, through legal terminal execution.", + "current": public_state(record, state), "skill_library": compact_library, + "shared_input_rules": input_rules, + "remaining_budget": budget, "feedback": feedback, + "recovery_context": recovery_context(state, budget, feedback), + "availability_rule": "unknown permits acquisition; canonical source observations establish obtained/unavailable; errors remain retryable; omitted batch attributes remain unknown"} + if skill_pool == "F66" and any( + failure.get("failure_kind") == "no_match" + for entry in state.get("comparison_failure_ledger", []) + for failure in entry.get("failures", [])): + payload["recovery_context"]["comparison_recovery_rule"] = ( + "An observed no_match blocks another comparison with identical effective inputs. " + "Repeating the same failed comparison cannot advance the task and still consumes the remaining budget. " + "no_match does not invalidate the actual retrieved candidates or create a successful alignment; " + "it records that this comparison found no supported match. A comparison success is not a universal " + "prerequisite for every classification Skill: inspect each alternative Skill's actual required inputs " + "and current verified state. Plan the complete remaining task autonomously within the existing budget, " + "using genuinely new evidence or changed candidates where available, or another contract-valid path " + "supported by current observations. Do not reuse a blocked comparison, fabricate alignment, or assume " + "that an unexecuted tail must be followed. Only a verified terminal completes classification.") + system = SYSTEM if skill_pool == "P14" else SYSTEM.replace("complete P14 library", "complete F66 library") + if skill_pool == "F66": + payload["contract_dependency_facts"] = { + "category_and_alignment_producers": [ + {"name": s["name"], "required_inputs": s["required_inputs"], "produces": s["produces"]} + for s in library if {"category", "aligned_standard_entry"} <= set(s["produces"])], + "comparison_dependency_rule": ( + "Comparison is optional for a Skill whose required inputs do not require a comparison output. " + "The listed category producers themselves publish the selected category and its frozen aligned entry " + "after a validated choice from actual retrieved candidates. This is a literal contract subset, " + "not a selected Program or permission to skip any required input.")} + current_facts = [] + for spec in payload["contract_dependency_facts"]["category_and_alignment_producers"]: + observed = [] + for key in spec["required_inputs"]: + reference = canonical_reference(key, state) + if (reference and reference.startswith("$state.") and + present(state.get(reference.removeprefix("$state.")))): + observed.append(key) + current_facts.append({"name": spec["name"], "observed_required_inputs": observed, + "unobserved_required_inputs": [key for key in spec["required_inputs"] if key not in observed]}) + payload["contract_dependency_facts"]["current_category_input_facts"] = current_facts + system += ( + "\nF66 dependency reminder: consult contract_dependency_facts and the full contracts. " + "Do not add a successful comparison as an undocumented prerequisite for category inference. " + "Choose the dependency path yourself. For runtime state inputs prefer arguments:{}; " + "the binder supplies actually available required inputs. Never paste observed metadata or semantic " + "text into dynamic arguments. A missing semantic dependency needs its actual producer before use; " + "a raw table description is not already table_semantics. Observe all original budgets and terminal guards.") + system += ( + "\nF66 completion objective: prioritize a complete contract-valid classification path whose required " + "inputs are already observed. Input presence is factual context, not a guarantee of semantic validity; " + "execution still verifies all contracts. Optional diagnostics must not become undocumented gating " + "steps. A no_match observation does not require repeatedly searching until an optional comparison " + "agrees. Choose the remaining whole Program yourself from actual evidence and dependencies.") + if feedback and feedback.get("status") == "planner_rejected": + payload["request_mode"] = "revise_rejected_program" + payload["revision_request"] = { + "executed": False, + "validation_error": feedback.get("error"), + "allowed_skill_names": [spec["name"] for spec in library], + "argument_rule": ( + "Prefer arguments:{} for required runtime-bound inputs. For explicit state inputs copy the " + "exact shared_input_rules canonical_reference; do not substitute a different alias or observed " + "text. Native observed_missing_metadata_type choices remain subject to their actual contract.")} + payload["task"] = ( + "Revise the previous unexecuted Program to correct the actual validation error in feedback. " + "Return one complete remaining Program, using only exact allowed Skill names and canonical " + "arguments from the contracts. The rejected Program produced no observations. Use current " + "verified state and the unchanged remaining budget; do not repeat the rejected error or invent " + "a new Skill or dependency. Choose the sequence yourself and finish through the validated terminal.") + encoded = pack_prompt(payload) + if skill_pool == "F66": + tail = ("current", "feedback", "recovery_context", "remaining_budget", "task") + encoded = {**{key: value for key, value in encoded.items() if key not in tail}, + **{key: encoded[key] for key in tail}} + return [{"role": "system", "content": system}, {"role": "user", "content": json.dumps(encoded, ensure_ascii=False, allow_nan=False, separators=(",", ":"))}] diff --git a/research/vehicle_program_v1/code/upstream/runtime/program_loop/planning_contracts.py b/research/vehicle_program_v1/code/upstream/runtime/program_loop/planning_contracts.py new file mode 100644 index 0000000..67c396a --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/program_loop/planning_contracts.py @@ -0,0 +1,30 @@ +"""Value-free planning guidance from catalog contracts and shared static rules. + +Possible producers and outputs are conditional, never proof of execution success. +Actual-state binding and the executor still validate every supplied input/output. +""" +from .validation import canonical_reference, explicit_value_policy, possible_outputs, FINAL_EVIDENCE_REQUIREMENTS + + +def planning_contract(spec, catalog): + known = {key for entry in catalog.values() for key in (*entry.required_inputs, *entry.optional_inputs)} + known.update(key for entry in catalog.values() for key in possible_outputs(entry)) + known.add("final_evidence_state") + inputs = {} + for key in (*spec.required_inputs, *spec.optional_inputs): + ref = canonical_reference(key, known) + target = ref.removeprefix("$state.") if ref and ref.startswith("$state.") else None + inputs[key] = { + "required": key in spec.required_inputs, + "canonical_reference": ref, + "explicit_value_policy": "observed_missing_metadata_type" if spec.name == "ACQUIRE_MISSING_METADATA" and key == "missing_metadata_type" else explicit_value_policy(ref), + "source": "state" if target else "initial_binding", + "possible_producers": [other.name for other in catalog.values() if target in possible_outputs(other)] if target and key != "final_evidence_state" else [], + "requires_all": sorted(FINAL_EVIDENCE_REQUIREMENTS) if key == "final_evidence_state" else [], + } + if spec.name == "ACQUIRE_MISSING_METADATA" and key == "missing_metadata_type": + inputs[key]["choice_rule"] = ("Choose an explicit native type from actual missing_metadata_types, " + "or a type conditionally reported missing by a preceding metadata check/acquisition " + "in this Program. The runtime binder checks actual missing_metadata_types; " + "a future retrieval is not guaranteed to succeed.") + return {"argument_template": {}, "inputs": inputs, "conditional_outputs": sorted(possible_outputs(spec))} diff --git a/research/vehicle_program_v1/code/upstream/runtime/program_loop/prompt_encoding.py b/research/vehicle_program_v1/code/upstream/runtime/program_loop/prompt_encoding.py new file mode 100644 index 0000000..4ab1d3f --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/program_loop/prompt_encoding.py @@ -0,0 +1,96 @@ +"""Lossless repeated-observation references, not execution/state references.""" +from collections import Counter +from copy import deepcopy +import json + +SCOPE = ('current', 'feedback', 'recovery_context') +TAG = 'prompt_ref' +TABLE = 'shared_values' +SCHEMA = 'prompt_encoding_schema' + + +def _canonical(value): + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':'), allow_nan=False) + + +def pack_prompt(payload): + if not isinstance(payload, dict) or TABLE in payload or SCHEMA in payload: + raise ValueError('reserved prompt encoding fields already present') + counts = Counter() + originals = {} + + def count(value): + if isinstance(value, dict) and set(value) == {TAG}: + raise ValueError('reserved prompt reference collision') + if isinstance(value, (dict, list, str)): + key = _canonical(value) + if len(key) >= 256: + counts[key] += 1 + originals[key] = value + children = value.values() if isinstance(value, dict) else value if isinstance(value, list) else () + for child in children: + count(child) + + for name in SCOPE: + if name in payload: + count(payload[name]) + refs = {} + values = {} + + def walk(value, excluded=None): + key = _canonical(value) if isinstance(value, (dict, list, str)) else '' + if counts[key] > 1 and key != excluded: + if key not in refs: + name = 'v' + str(len(refs) + 1) + refs[key] = name + values[name] = walk(originals[key], key) + return {TAG: refs[key]} + if isinstance(value, dict): + return {name: walk(child) for name, child in value.items()} + if isinstance(value, list): + return [walk(child) for child in value] + return value + + encoded = deepcopy(payload) + for name in SCOPE: + if name in encoded: + encoded[name] = walk(encoded[name]) + encoded[SCHEMA] = 1 + encoded[TABLE] = values + if _canonical(unpack_prompt(encoded)) != _canonical(payload): + raise ValueError('lossless prompt roundtrip failed') + return encoded + + +def unpack_prompt(encoded): + if not isinstance(encoded, dict) or encoded.get(SCHEMA) != 1 or not isinstance(encoded.get(TABLE), dict): + raise ValueError('invalid prompt encoding schema') + values = encoded[TABLE] + used = set() + active = set() + cache = {} + + def expand(value): + if isinstance(value, dict) and set(value) == {TAG}: + name = value[TAG] + if not isinstance(name, str) or name not in values or name in active: + raise ValueError('dangling or cyclic prompt reference') + used.add(name) + if name not in cache: + active.add(name) + cache[name] = expand(values[name]) + active.remove(name) + return deepcopy(cache[name]) + if isinstance(value, dict): + return {name: expand(child) for name, child in value.items()} + if isinstance(value, list): + return [expand(child) for child in value] + return value + + result = {name: deepcopy(value) for name, value in encoded.items() if name not in (TABLE, SCHEMA)} + for name in SCOPE: + if name in result: + result[name] = expand(result[name]) + if used != set(values): + raise ValueError('unused shared value could conceal unobserved evidence') + return result diff --git a/research/vehicle_program_v1/code/upstream/runtime/program_loop/recovery_context.py b/research/vehicle_program_v1/code/upstream/runtime/program_loop/recovery_context.py new file mode 100644 index 0000000..6f75a96 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/program_loop/recovery_context.py @@ -0,0 +1,41 @@ +"""Summarize actual outer-loop observations without proposing or executing steps.""" +from .contracts import json_snapshot + + +def recovery_context(state, budget, feedback): + """Return detached JSON facts; presence and historical success imply no readiness. + + Feedback is the controller's last observation, not model-authored recovery advice. + Keep raw feedback at its existing boundary; this summary omits values and arguments. + """ + previous = None + if feedback and feedback.get('status') in ('completed', 'failed', 'budget_exhausted', 'terminated') and 'plan' in feedback: + steps = feedback.get('steps', []) + executed = {step['step_index'] for step in steps} + previous = { + 'status': feedback['status'], + 'failure_index': feedback.get('failure_index'), + 'successful_steps': [{'step_index': s['step_index'], 'tool': s['call']['tool']} + for s in steps if s['status'] == 'success'], + 'failed_steps': [{'step_index': s['step_index'], 'tool': s['call']['tool'], 'status': s['status']} + for s in steps if s['status'] != 'success'], + 'unexecuted_tail': [{'step_index': i, 'tool': call['tool']} + for i, call in enumerate(feedback['plan']['program']) if i not in executed], + } + return json_snapshot({ + 'previous_program': previous, + 'classification_completed': bool(previous and previous['status'] == 'terminated' + and feedback.get('terminated') is True + and feedback.get('steps') + and feedback['steps'][-1]['status'] == 'success'), + 'remaining_budget': budget, + 'comparison_failures': state.get('comparison_failure_ledger', []), + 'invalidations': feedback.get('invalidations', []) if previous else [], + 'observed_state_keys': sorted(state), + 'state_rule': 'Observed keys indicate presence only; neither presence nor historical success verifies readiness. Removed invalidated outputs are not available in current state.', + 'availability_observations': [{k: entry[k] for k in ('scope', 'type', 'status', 'skill') if k in entry} + for entry in state.get('availability_ledger', [])], + 'availability_rule': 'These are recorded observations, not new checks. Missing observations remain unknown/unqueried; omitted batch attributes remain unknown. Obtained and canonical unavailable are distinct; error is retryable and does not establish absence. Target details remain in actual public state.', + 'tail_rule': 'Unexecuted tail is historical intent only, without arguments; choose a new whole Program autonomously using current state and contracts.', + 'task_reminder': 'Consider the remaining complete classification task within the actual global budget. Only a validated successful terminal completes classification. A successful recovery step alone may leave the task unfinished, especially on the last available Program. Partial legal Programs remain allowed; no steps or extra budget are supplied.', + }) diff --git a/research/vehicle_program_v1/code/upstream/runtime/program_loop/rejection_context.py b/research/vehicle_program_v1/code/upstream/runtime/program_loop/rejection_context.py new file mode 100644 index 0000000..a75322b --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/program_loop/rejection_context.py @@ -0,0 +1,36 @@ +"""Read-only summary of unexecuted model intent; never execution evidence.""" +import json +import math +from .validation import _unique_object + +def _reject_constant(value): + raise ValueError(f'nonfinite JSON {value}') + +def _finite_float(value): + number = float(value) + if not math.isfinite(number): + raise ValueError('nonfinite JSON number') + return number + +def rejected_program_summary(raw, catalog, max_steps): + if not isinstance(raw, str): + return None + try: + value = json.loads(raw, object_pairs_hook=_unique_object, parse_constant=_reject_constant, parse_float=_finite_float) + except (ValueError, TypeError, RecursionError): + return None + if not isinstance(value, dict) or set(value) != {'program'}: + return None + steps = value['program'] + if not isinstance(steps, list) or not steps: + return None + tools = [] + for item in steps: + if not isinstance(item, dict) or set(item) != {'tool','arguments'}: + return None + name = item['tool'] + if not isinstance(name, str) or name not in catalog or not isinstance(item['arguments'], dict): + return None + tools.append(name) + return {'executed':False,'trusted':False,'step_count':len(steps), + 'max_program_steps':max_steps,'tool_sequence':tools} diff --git a/research/vehicle_program_v1/code/upstream/runtime/program_loop/validation.py b/research/vehicle_program_v1/code/upstream/runtime/program_loop/validation.py new file mode 100644 index 0000000..7f06958 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/program_loop/validation.py @@ -0,0 +1,108 @@ +"""Validate possible dependencies, never pretend a future retrieval succeeded.""" +import json +from .contracts import Program, ToolCall +from pool_metadata import FIELDS +from pool_runtime import SEMANTIC_KEYS, ALIGNMENT_KEYS + +SELECTION = {"category", "aligned_category", "candidate_category", "category_candidate", "aligned_standard_entry", "grading_level", "grading_level_candidate", "final_evidence_state", "final_result"} +GRADE = {"grading_level", "grading_level_candidate", "final_evidence_state", "final_result"} +EVIDENCE_BUNDLES = {"semantic_query", "semantic_evidence", "current_semantic_state", "current_evidence", "current_evidence_state", "all_available_evidence", "available_evidence", "evidence", "evidence_set"} +FINAL_EVIDENCE_REQUIREMENTS = frozenset({"grading_level", "category", "aligned_standard_entry", "standard_candidates"}) +RAW_CONTEXT = set(FIELDS) | {"table_fields", "selected_context_fields", "context_fields", "context_field_descriptions", + "context_field_types", "context_field_values", "table_context", "database_context", + "acquired_metadata"} + + +def _unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: raise ValueError(f"duplicate JSON key {key}") + result[key] = value + return result + + +def canonical_reference(key, available): + if key == "current_record": return "$record" + if key == "domain": return "$domain" + if key in ("field_name", "target_field"): return "$metadata.field_name" + if key == "candidate_standard_categories" and "standard_candidates" in available: return "$state.standard_candidates" + return "$state." + key if key in available else None + + +_reference = canonical_reference + + +def explicit_value_policy(reference): + return "canonical_reference_only" if reference and reference.startswith("$state.") else "initial_binding_contract" + + +def possible_outputs(spec): + produced = set(spec.produces) + outputs = set(produced) + semantic = produced & set(SEMANTIC_KEYS) + alignment = produced & set(ALIGNMENT_KEYS) + if semantic: + outputs.update(EVIDENCE_BUNDLES | {"available_metadata_evidence"}) + if any(key.startswith(("field_", "disambiguated_", "value_")) for key in semantic): + outputs.add("available_field_evidence") + if any(key.startswith("context_") for key in semantic): outputs.add("context_field_evidence") + if any(key.startswith("table_") for key in semantic): outputs.add("available_table_evidence") + if any(key.startswith("database_") for key in semantic): outputs.add("available_database_evidence") + if alignment: + outputs.update(("standard_alignment_evidence", "aligned_standard_candidates", "conflicting_standard_alignments")) + if "context_field_semantics" in produced: outputs.add("context_semantics") + if "field_semantics" in produced: outputs.add("candidate_field_semantics") + if "field_metadata" in produced: + outputs.update((*FIELDS, "current_metadata", "available_metadata", "semantic_query", "available_evidence", "all_available_evidence")) + if "acquired_metadata" in produced: + outputs.update(FIELDS) + if "metadata_availability" in produced or produced & (set(FIELDS) | {"field_metadata", "acquired_metadata"}): + outputs.update(("current_metadata", "available_metadata", "missing_metadata_types", "missing_metadata_type")) + if "category" in produced: outputs.update(("aligned_category", "candidate_category", "category_candidate")) + if "standard_candidates" in produced: outputs.update(("candidate_standard_categories", "candidate_categories", "previous_standard_results")) + if "grading_level" in produced: outputs.add("grading_level_candidate") + return outputs + + +def advance_available(spec, available): + available = set(available) + produced = set(spec.produces) + if produced & (set(SEMANTIC_KEYS) | set(ALIGNMENT_KEYS) | RAW_CONTEXT | {"field_metadata"}): available -= SELECTION + if "standard_candidates" in produced: available -= SELECTION + if "category" in produced: available -= GRADE + available.update(possible_outputs(spec)) + if FINAL_EVIDENCE_REQUIREMENTS <= available: available.add("final_evidence_state") + return available + + +def parse_program(raw, catalog, state, max_steps): + if not isinstance(raw, str): raise ValueError("Planner must return JSON text") + value = json.loads(raw, object_pairs_hook=_unique_object, parse_constant=lambda v: (_ for _ in ()).throw(ValueError(f"nonfinite JSON {v}"))) + if not isinstance(value, dict) or set(value) != {"program"}: raise ValueError("Expected exactly one JSON Program object") + steps = value["program"] + if not isinstance(steps, list) or not steps: raise ValueError("Program must be nonempty") + if len(steps) > max_steps: raise ValueError("Program length exceeds max_program_steps") + available = set(state) + calls = [] + for index, item in enumerate(steps): + if not isinstance(item, dict) or set(item) != {"tool", "arguments"}: raise ValueError(f"step {index}: expected only tool and arguments") + name, args = item["tool"], item["arguments"] + if not isinstance(name, str) or name not in catalog: raise ValueError(f"step {index}: unknown Skill") + spec = catalog[name] + if not isinstance(args, dict) or set(args) - set((*spec.required_inputs, *spec.optional_inputs)): raise ValueError(f"step {index}: unknown arguments") + for key in sorted(set(spec.required_inputs) | set(args)): + reference = canonical_reference(key, available) + if reference is None: raise ValueError(f"step {index}: missing dependency {key}") + supplied = args.get(key) + if name == "ACQUIRE_MISSING_METADATA" and key == "missing_metadata_type": + choices = state.get("missing_metadata_types") if "missing_metadata_types" in state else None + if not isinstance(supplied, str) or supplied not in FIELDS or supplied == "field_name" or (choices is not None and supplied not in choices): + raise ValueError(f"step {index}: missing_metadata_type must be an observed or conditionally missing metadata type") + continue + if key in args and ((explicit_value_policy(reference) == "canonical_reference_only" and supplied != reference) + or (isinstance(supplied, str) and supplied.startswith("$") and supplied != reference)): + raise ValueError(f"step {index}: incorrect canonical reference for {key}; expected {reference}") + if spec.terminal and index != len(steps) - 1: raise ValueError(f"step {index}: terminal Skill must be last") + available = advance_available(spec, available) + calls.append(ToolCall(name, args)) + return Program(tuple(calls)) diff --git a/research/vehicle_program_v1/code/upstream/runtime/runtime_identity.py b/research/vehicle_program_v1/code/upstream/runtime/runtime_identity.py new file mode 100644 index 0000000..5ed47a0 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/runtime_identity.py @@ -0,0 +1,31 @@ +"""Exclusive core/test byte identity for this explicit protocol variant.""" +import hashlib +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parent + + +def sha(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def identity(): + files = [p for name in ('program_loop', 'vendor', 'tests') for p in (ROOT / name).rglob('*') + if p.is_file() and '__pycache__' not in p.parts] + files.append(ROOT / 'runtime_identity.py') + entries = [{'path': p.relative_to(ROOT).as_posix(), 'sha256': sha(p)} for p in sorted(files)] + signature = hashlib.sha256(json.dumps(entries, sort_keys=True).encode()).hexdigest() + return {'schema': 'grounded-runtime-v1', 'core_signature_sha256': signature, 'files': entries, + 'old_nontraining_comparable_without_rebaseline': False} + + +def verify(): + current = identity() + assert current == json.loads((ROOT / 'reviews/runtime-manifest.json').read_text(encoding='utf-8')) + return current + + +if __name__ == '__main__': + path = ROOT / 'reviews/runtime-manifest.json'; path.parent.mkdir(parents=True, exist_ok=True) + with path.open('x', encoding='utf-8') as stream:json.dump(identity(), stream, indent=2) + print(json.dumps({'runtime_signature': verify()['core_signature_sha256'], 'files': len(verify()['files'])})) diff --git a/research/vehicle_program_v1/code/upstream/runtime/tests/test_grounding.py b/research/vehicle_program_v1/code/upstream/runtime/tests/test_grounding.py new file mode 100644 index 0000000..7a77fdf --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/tests/test_grounding.py @@ -0,0 +1,41 @@ +import copy +from pathlib import Path +import sys +import unittest +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / 'vendor')) +import grounded_semantics as g + + +class GroundingTests(unittest.TestCase): + def context(self):return {'field_name': 'OPAQUE', 'field_description': '真实字段说明'} + def value(self): + return {'schema': 'grounded-source-v1', 'status': 'observed_description', 'field_name': 'OPAQUE', + 'description': '真实字段说明', 'source_consistency': 'not_adjudicated'} + + def test_exact_observed_description_passes_without_mutation(self): + value = self.value(); before = copy.deepcopy(value) + self.assertTrue(g.validate_value(value, self.context())) + self.assertEqual(value, before) + + def test_unknown_only_when_no_observed_description(self): + value = self.value(); value.update(status='unknown', description=None) + self.assertTrue(g.validate_value(value, {'field_name': 'OPAQUE'})) + with self.assertRaises(ValueError):g.validate_value(value, self.context()) + + def test_fabricated_or_extra_claims_rejected(self): + for change in ({'description': 'invented business meaning'}, {'guess': 'vaccine'}, + {'source_consistency': 'verified'}, {'status': 'conflict'}, {'field_name': 'OTHER'}): + value = self.value(); value.update(change) + with self.assertRaises(ValueError):g.validate_value(value, self.context()) + + def test_unobtained_context_cannot_support_description(self): + with self.assertRaises(ValueError):g.validate_value(self.value(), {'field_name': 'OPAQUE'}) + + def test_string_or_incomplete_object_rejected(self): + for value in ('guess', {}, None, {'status': 'unknown'}): + with self.assertRaises(ValueError):g.validate_value(value, self.context()) + + +if __name__ == '__main__':unittest.main() diff --git a/research/vehicle_program_v1/code/upstream/runtime/tests/test_integration.py b/research/vehicle_program_v1/code/upstream/runtime/tests/test_integration.py new file mode 100644 index 0000000..d29a6f3 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/tests/test_integration.py @@ -0,0 +1,59 @@ +import json +from pathlib import Path +import sys +import unittest +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from program_loop import build_program_controller + + +class IntegrationTests(unittest.TestCase): + def make(self, *, acquire=True, corrupt=False): + registry = {'d:A/B/C' + str(i): {'name': 'C' + str(i), 'path': ['A', 'B', 'C' + str(i)], + 'description': 'synthetic standard ' + str(i), 'grading_level': 'L1'} for i in range(5)} + rows = [{'id': 'synthetic', 'domain': 'd', 'metadata': {'field_name': 'OPAQUE', + 'field_description': '真实字段说明', 'table_name': 'TABLE', 'gold': 'SECRET_GOLD'}}] + messages = [] + def generate(value): + messages.append(value); head = value[0]['content'].splitlines()[0]; user = json.loads(value[-1]['content']) + if head.startswith('You are the Global Program Planner'): + tools = (['GET_FIELD_METADATA'] if acquire else []) + ['INTERPRET_FIELD_NAME', 'SELECT_CATEGORY_TOPK', 'INFER_CATEGORY_FROM_TOPK', 'INFER_GRADING_LEVEL', 'FINALIZE_CLASSIFICATION'] + return json.dumps({'program': [{'tool': name, 'arguments': {}} for name in tools]}) + skill = head.removeprefix('Execute only the selected Skill ').removesuffix('.') + if skill == 'INTERPRET_FIELD_NAME': + ctx = user.get('grounding_context', {'field_name': 'OPAQUE'}) + desc = ctx.get('field_description'); status = 'observed_description' if desc else 'unknown' + if corrupt:desc = 'invented debt service meaning' + return json.dumps({'summary': status, 'data': {'field_semantics': {'schema': 'grounded-source-v1', + 'status': status, 'field_name': 'OPAQUE', 'description': desc, 'source_consistency': 'not_adjudicated'}}}) + if skill == 'SELECT_CATEGORY_TOPK': + return json.dumps({'option_ids': [r['option_id'] for r in user['category_options'][:5]], 'summary': 'synthetic'}) + if skill == 'INFER_CATEGORY_FROM_TOPK': + return json.dumps({'option_id': user['candidate_details'][0]['option_id'], 'summary': 'synthetic'}) + raise AssertionError('unexpected model role') + c = build_program_controller(generate, registry, rows, {'max_programs': 1, 'max_program_steps': 10, + 'max_skill_steps': 12, 'max_planner_attempts': 1, 'max_identical_calls': 1}, skill_pool='F66') + trace = c.run({'source_id': 'synthetic', 'domain': 'd', 'field_name': 'OPAQUE'}).to_dict() + return trace, messages + + def test_real_controller_publishes_grounded_value_and_terminates(self): + trace, messages = self.make() + self.assertTrue(trace['terminated']) + self.assertEqual(trace['state']['field_semantics']['description'], '真实字段说明') + self.assertNotIn('SECRET_GOLD', json.dumps(messages, ensure_ascii=False)) + + def test_no_hidden_retrieval_before_acquisition(self): + trace, messages = self.make(acquire=False) + self.assertTrue(trace['terminated']) + call = next(json.loads(m[-1]['content']) for m in messages if 'selected Skill INTERPRET_FIELD_NAME.' in m[0]['content']) + self.assertNotIn('field_description', call['grounding_context']) + self.assertEqual(trace['state']['field_semantics']['status'], 'unknown') + + def test_unsupported_output_fails_without_autofinalize(self): + trace, messages = self.make(corrupt=True) + self.assertFalse(trace['terminated']) + self.assertNotIn('field_semantics', trace['state']) + self.assertTrue(any(s['status'] == 'error' for p in trace['programs'] for s in p['steps'])) + + +if __name__ == '__main__':unittest.main() diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/common.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/common.py new file mode 100644 index 0000000..16542c7 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/common.py @@ -0,0 +1,118 @@ +"""Pure helpers for an isolated, assistant-only semantic-Skill SFT pilot.""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import random + +SEED = 20261001 +SKILLS = ("SELECT_CATEGORY_TOPK", "INFER_CATEGORY_FROM_TOPK") +REVISION = "local-zip-003494285b21f01a5a6242cb45ae84394433fe72c41f447f64372720f78a2b20" + + +def sha256(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def read_json(path): + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def read_jsonl(path): + return [json.loads(line) for line in Path(path).read_text(encoding="utf-8").splitlines() if line.strip()] + + +def write_json(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temp = path.with_name(path.name + ".tmp") + temp.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + temp.replace(path) + + +def fingerprint(value): + return hashlib.sha256(json.dumps(value, ensure_ascii=False, sort_keys=True).encode()).hexdigest() + + +def validate_completed_adapter(adapter): + adapter=Path(adapter) + completion=read_json(adapter.parent/'completion.json') + for file,key in [('adapter_model.safetensors','adapter_sha256'),('adapter_config.json','adapter_config_sha256')]: + if sha256(adapter/file)!=completion[key]: + raise ValueError(f'adapter {file} differs from completed training') + + +def validate_matched_baseline(base, adapted): + if base['model']['adapter_sha256'] is not None: + raise ValueError('control must use base weights without adapter') + excluded={'adapter_sha256','adapter_config_sha256','adapter_routing'} + if {k:v for k,v in base['model'].items() if k not in excluded} != {k:v for k,v in adapted['model'].items() if k not in excluded}: + raise ValueError('matched model/generation parameters differ') + if base['loop_config']!=adapted['loop_config']: + raise ValueError('matched loop configuration differs') + def behavior(manifest): + return {k:v['sha256'] for k,v in manifest['files'].items() if k not in {'adapter_weights','adapter_config'}} + if behavior(base)!=behavior(adapted): + raise ValueError('matched non-adapter behavior artifacts differ') + + +def selected_skill(messages): + """Route only trusted, exact Skill system headers, never user text.""" + if not messages or messages[0].get("role") != "system": + return None + system = messages[0].get("content", "") + return next((name for name in SKILLS if system.startswith(f"Execute only the selected Skill {name}.\n")), None) + + +def category_of(row): + levels = row["classification"] + return str(row["domain"]) + ":" + "/".join(levels[k] for k in ( + "category_root_level", "category_branch_level", "category_leaf_level" + )) + + +def auxiliary_candidates(gold, categories, source_id, seed=SEED): + """Four reproducible distractors, explicitly NOT a gold semantic ranking.""" + rng = random.Random(f"{seed}:{source_id}") + others = rng.sample(sorted(set(categories) - {gold}), 4) + presented = [gold, *others] + rng.shuffle(presented) + return others, presented + + +def assistant_encoding(tokenizer, row, max_length=3072): + """Mask prompts and weak candidate IDs; fail instead of truncating targets. + + Top-K has one true category only. IDs at positions 2..5 are scaffolding; + any token overlapping these character spans receives label -100. + """ + prompt = tokenizer.apply_chat_template(row["messages"], tokenize=False, + add_generation_prompt=True, enable_thinking=False) + response = row["response"] + if not tokenizer.eos_token: + raise ValueError("tokenizer requires an EOS token") + text = prompt + response + tokenizer.eos_token + encoded = tokenizer(text, add_special_tokens=False, return_offsets_mapping=True) + prefix = tokenizer(prompt, add_special_tokens=False)["input_ids"] + ids = encoded["input_ids"] + if ids[:len(prefix)] != prefix: + raise ValueError("generation prefix is not a token prefix of training sequence") + if len(ids) > max_length: + raise ValueError(f"{row['source_id']} exceeds max_length: {len(ids)} > {max_length}; no silent truncation") + spans = [(len(prompt) + a, len(prompt) + b) for a, b in row["weak_response_spans"]] + labels = [] + for index, (token, (a, b)) in enumerate(zip(ids, encoded["offset_mapping"])): + ignored = index < len(prefix) or any(a < end and b > start for start, end in spans) + labels.append(-100 if ignored else token) + if not any(label != -100 for label in labels): + raise ValueError("empty supervised completion") + return {"input_ids": ids, "attention_mask": [1] * len(ids), "labels": labels}, { + "length": len(ids), "prompt_tokens": len(prefix), + "supervised_tokens": sum(x != -100 for x in labels), + "weak_tokens_masked": sum(index >= len(prefix) and label == -100 for index, label in enumerate(labels)), + } diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/comparison_output.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/comparison_output.py new file mode 100644 index 0000000..6f35b61 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/comparison_output.py @@ -0,0 +1,48 @@ +"""Strict compact comparison wire parser; no registry, backend, or state access.""" +import json +from collections.abc import Mapping + + +MAX_REASON_LENGTH = 240 + + +class ComparisonOutputError(ValueError): + """A model response violates the compact comparison wire contract.""" + + +def _unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ComparisonOutputError(f"duplicate comparison key: {key}") + result[key] = value + return result + + +def _reject_constant(value): + raise ComparisonOutputError(f"non-JSON comparison constant: {value}") + + +def parse_comparison_output(raw, candidate_option_ids): + """Validate one exact JSON decision against the actual current candidate IDs.""" + if not isinstance(raw, str) or not raw.strip(): + raise ComparisonOutputError("comparison response must be nonempty JSON text") + try: + value = json.loads(raw, object_pairs_hook=_unique_object, parse_constant=_reject_constant) + except (json.JSONDecodeError, RecursionError) as exc: + raise ComparisonOutputError(f"invalid comparison JSON: {exc}") from exc + if not isinstance(value, Mapping) or set(value) != {"decision", "option_id", "reason"}: + raise ComparisonOutputError("comparison must contain exactly decision, option_id, reason") + reason = value["reason"] + if not isinstance(reason, str) or not reason.strip() or len(reason) > MAX_REASON_LENGTH: + raise ComparisonOutputError(f"comparison reason must be nonempty text of at most {MAX_REASON_LENGTH} characters") + decision, option_id = value["decision"], value["option_id"] + if decision == "match": + if not isinstance(option_id, str) or option_id not in candidate_option_ids: + raise ComparisonOutputError("comparison match option_id must be an actual current candidate short ID") + elif decision == "no_match": + if option_id is not None: + raise ComparisonOutputError("comparison no_match option_id must be null") + else: + raise ComparisonOutputError("comparison decision must be match or no_match") + return dict(value) diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/evidence_roles.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/evidence_roles.py new file mode 100644 index 0000000..1df96fb --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/evidence_roles.py @@ -0,0 +1,77 @@ +"""Source roles and a copied Top1-only comparison explanation projection.""" +from copy import deepcopy +from collections.abc import Mapping + +from pool_catalog import pool_catalog +from pool_metadata import FIELDS +from grounded_semantics import valid_shape as grounded_shape + + +def acquired_source(key, source): + """Only declared deterministic acquisition outputs establish raw facts.""" + spec = pool_catalog('F66').get(source) + if spec is None or spec.role != 'metadata_acquisition': + return False + return key in spec.produces or (key in FIELDS and + bool({'field_metadata', 'acquired_metadata'} & set(spec.produces))) + + +def evidence_roles(metadata, semantics, alignments, observations, provenance): + result = {} + for values, default in ((metadata, 'raw_fact'), (observations, 'raw_fact'), + (semantics, 'model_interpretation'), (alignments, 'comparison_inference')): + for key, value in values.items(): + source = 'initial_record' if key == 'field_name' else provenance.get(key) + role = default + if default == 'model_interpretation' and grounded_shape(value): + result[key] = {'role': 'source_anchored_description' if value['status']=='observed_description' else 'unresolved_hypothesis', + 'value': deepcopy(value), 'provenance': source, 'semantic_consistency_verified': False} + continue + if default == 'model_interpretation' and source and ( + source.endswith('_NAME') or 'ABBREVIATION' in source): + role = 'model_hypothesis' + result[key] = {'role': role, 'value': deepcopy(value), 'provenance': source} + return result + + +def project_top1_evidence(evidence, comparison_reason): + """Remove comparison reasons and value aliases, retaining acquired facts. + + Only callers constructing Top1 messages use this view. Shared publication, + binding, Planner feedback and all other Skill messages retain their inputs. + """ + omitted = object() + catalog = pool_catalog('F66') + independent_keys = {key for spec in catalog.values() + if spec.role in ('semantic_interpretation', 'standard_alignment', 'evidence_reasoning') + and spec.name != 'COMPARE_STANDARD_CANDIDATES' for key in spec.produces} + + def project(value, protected=False): + if protected: + return deepcopy(value) + if isinstance(value, Mapping): + provenance = value.get('provenance') + source = provenance if isinstance(provenance, str) else None + if value.get('value') == comparison_reason and (source == 'COMPARE_STANDARD_CANDIDATES' or + value.get('role') == 'comparison_inference' and source not in catalog): + return omitted + protected = protected or value.get('role') == 'raw_fact' or ( + source in catalog and source != 'COMPARE_STANDARD_CANDIDATES' and 'role' in value) + if protected: + return deepcopy(value) + result = {} + for key, item in value.items(): + if key == 'standard_comparison': + continue + projected = project(item, protected or key in ('metadata', 'observations') or key in independent_keys) + if projected is not omitted: + result[key] = projected + return result + if isinstance(value, list): + return [item for original in value if (item := project(original, protected)) is not omitted] + if not protected and isinstance(comparison_reason, str) and value == comparison_reason: + return omitted + return deepcopy(value) + + result = project(evidence) + return {} if result is omitted else result diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/grounded_semantics.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/grounded_semantics.py new file mode 100644 index 0000000..d7a6b67 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/grounded_semantics.py @@ -0,0 +1,64 @@ +"""Versioned extractive grounding, not an automatic semantic-truth judge.""" +from copy import deepcopy +import json + +TARGETS = {'EXPAND_FIELD_ABBREVIATION': 'field_abbreviation_semantics', + 'INTERPRET_FIELD_NAME': 'field_semantics', + 'INTERPRET_FIELD_DESCRIPTION': 'field_description_semantics'} + + +def validate_value(value, context): + keys = {'schema', 'status', 'field_name', 'description', 'source_consistency'} + if not isinstance(value, dict) or set(value) != keys: + raise ValueError('grounded evidence requires exact typed fields; free interpretation prohibited') + if value['schema'] != 'grounded-source-v1' or value['source_consistency'] != 'not_adjudicated': + raise ValueError('grounding schema or non-adjudication status differs') + if not isinstance(context.get('field_name'), str) or not context['field_name'].strip() or value['field_name'] != context['field_name']: + raise ValueError('grounded field identity differs') + desc = context.get('field_description') + observed = isinstance(desc, str) and bool(desc.strip()) and desc.strip().casefold() not in ('none', 'null', 'n/a', 'nan') + if observed: + if value['status'] != 'observed_description' or value['description'] != desc: + raise ValueError('grounded description must equal entire actual acquired description') + elif value['status'] != 'unknown' or value['description'] is not None: + raise ValueError('absent acquired description permits unknown only') + return True + + +def valid_shape(value): + try: + if not isinstance(value, dict):return False + validate_value(value, {'field_name': value.get('field_name'), 'field_description': value.get('description')}) + except (ValueError, TypeError):return False + return True + + +def make_messages(contract, resolved, context): + assert contract.name in TARGETS + system = (f'Execute only the selected Skill {contract.name}.\n' + f'Versioned contract:\n{contract.body}\n' + 'Use only resolved_inputs and the explicitly declared grounding_context from already acquired raw metadata. ' + 'Return exactly summary and data. summary must equal the status string. ' + 'The produced value must be an object with exactly schema,status,field_name,description,source_consistency. ' + 'schema="grounded-source-v1"; field_name must copy grounding_context.field_name; ' + 'source_consistency="not_adjudicated" always. If grounding_context contains a real field_description, ' + 'status="observed_description" and description must copy that entire original string exactly. ' + 'Otherwise status="unknown" and description=null. No guessed expansion, translation, paraphrase, ' + 'candidate meanings, extra keys or assertion that name and description agree/conflict. ' + 'Observed description is a source observation, not semantic truth or verified consistency. ' + 'Do not retrieve hidden metadata, labels, registry outcomes or future observations.') + payload = {'skill': contract.name, 'resolved_inputs': deepcopy(resolved), 'grounding_context': deepcopy(context), + 'grounding_protocol': 'extractive-grounded-source-v1', + 'output_schema': {'summary': 'same as status', 'data': {TARGETS[contract.name]: { + 'schema': 'grounded-source-v1', 'status': 'observed_description or unknown', + 'field_name': 'copy exact context field_name', 'description': 'exact context description or null', + 'source_consistency': 'not_adjudicated'}}}} + return [{'role': 'system', 'content': system}, {'role': 'user', 'content': json.dumps(payload, ensure_ascii=False)}] + + +def validate_observation(observation, context): + key = TARGETS[observation.skill] + value = observation.data[key] + validate_value(value, context) + if observation.summary != value['status']:raise ValueError('summary must equal the grounded status, not free prose') + return observation diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/pool_availability.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/pool_availability.py new file mode 100644 index 0000000..74c15b8 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/pool_availability.py @@ -0,0 +1,115 @@ +"""Record-local query evidence. Never accesses the private metadata lookup.""" +from copy import deepcopy +from pool_metadata import FIELDS, present + +RULE = ('observed-query-availability-v1: unknown permits acquisition; only canonical-val ' + 'query observations establish obtained or unavailable; errors remain retryable; ' + 'reuse obtained values; suppress repeat targets in the same record/object scope; ' + 'CHECK reports visibility and omitted batch fields stay unknown; missing_metadata_types ' + 'retains its acquired-visibility meaning; continue evidence and classification when sufficient.') +DIRECT = {'GET_FIELD_DESCRIPTION':'field_description', 'GET_TABLE_DESCRIPTION':'table_description', + 'GET_DATABASE_DESCRIPTION':'database_description'} +CONTEXT = {'GET_CONTEXT_FIELD_DESCRIPTIONS':('context_field_descriptions','field_description'), + 'GET_CONTEXT_FIELD_TYPES':('context_field_types','field_type'), + 'GET_CONTEXT_FIELD_VALUES':('context_field_values','value')} +BATCH = {'GET_FIELD_METADATA':'field_metadata', 'GET_TABLE_FIELDS':'table_fields', + 'GET_COMPLETE_TABLE_CONTEXT':'table_context', 'GET_COMPLETE_DATABASE_CONTEXT':'database_context', + 'EXPAND_SELECTED_CONTEXT':'context_fields'} + + +def initialize(record, state): + state.setdefault('availability_ledger', [{'scope':'record','target':record.field_name, + 'type':'field_name','status':'obtained','skill':'initial-input','source':'public-input', + 'value':record.field_name}]) + + +def status(state, scope, target, kind): + return next((e['status'] for e in state.get('availability_ledger',[]) + if (e['scope'],e['target'],e['type']) == (scope,target,kind)), 'unknown') + + +def target_name(state): + return next(e['value'] for e in state['availability_ledger'] + if e['scope']=='record' and e['type']=='field_name') + + +def targets(call, state): + name=call.tool; own=target_name(state) + if name in DIRECT: return [('record',own,DIRECT[name])] + if name=='ACQUIRE_MISSING_METADATA': return [('record',own,call.arguments.get('missing_metadata_type'))] + if name in BATCH: + # Expansion depends on the selected context object set, not a global tool flag. + target=own if name!='EXPAND_SELECTED_CONTEXT' else tuple(state.get('selected_context_fields',[])) + return [('query',list(target) if isinstance(target,tuple) else target,BATCH[name])] + if name in CONTEXT: + return [('context-field',f['field_name'],CONTEXT[name][1]) for f in state.get('context_fields',[])] + return [] + + +def blocked(call, state): + queried=targets(call,state) + return queried if queried and all(status(state,*t) in ('obtained','unavailable') for t in queried) else [] + + +def _put(state, scope, target, kind, outcome, skill, source, value=None): + entry={'scope':scope,'target':deepcopy(target),'type':kind,'status':outcome,'skill':skill,'source':source} + if outcome=='obtained': entry['value']=deepcopy(value) + ledger=state['availability_ledger'] + for i,old in enumerate(ledger): + if (old['scope'],old['target'],old['type'])==(scope,target,kind): + if old['status'] in ('obtained','unavailable') and outcome=='error': return + ledger[i]=entry;break + else: ledger.append(entry) + + +def observe(call, observation, state): + if call.tool=='CHECK_METADATA_AVAILABILITY': return + queried=targets(call,state) + if not queried: return + source=observation.metadata.get('source'); outcome=observation.status + if outcome=='error': + for t in queried: _put(state,*t,'error',call.tool,None) + return + if source!='canonical-val': return + own=target_name(state);data=observation.data + if call.tool in CONTEXT: + wrapper=data.get(CONTEXT[call.tool][0],{}) + available={f['field_name']:f[CONTEXT[call.tool][1]] for f in wrapper.get('fields',[]) + if present(f.get(CONTEXT[call.tool][1]))} + missing=wrapper.get('missing_fields',[]) if outcome=='success' else observation.metadata.get('missing_fields',[]) + for scope,target,kind in queried: + if outcome=='success' and target in available: _put(state,scope,target,kind,'obtained',call.tool,source,available[target]) + elif target in missing: _put(state,scope,target,kind,'unavailable',call.tool,source) + return + if outcome=='unavailable': + for t in queried: + if t[0]=='record' and t[2] in observation.metadata.get('missing',[]): + _put(state,*t,'unavailable',call.tool,source) + return + if outcome!='success': return + raw={} + for key in ('field_metadata','acquired_metadata'): + raw.update({k:v for k,v in data.get(key,{}).items() if k in FIELDS and present(v)}) + raw.update({k:v for k,v in data.items() if k in FIELDS and present(v)}) + if 'field_name' in raw and raw['field_name']!=own: raise ValueError('observed field_name differs from initial public input') + # JSON snapshots sort mapping keys; canonical ledger order must survive that + # round trip because the next Planner payload includes this ordered list. + for k,v in sorted(raw.items()): _put(state,'record',own,k,'obtained',call.tool,source,v) + if call.tool in BATCH: + for t in queried: _put(state,*t,'obtained',call.tool,source,data.get(BATCH[call.tool])) + if call.tool=='EXPAND_SELECTED_CONTEXT': + for field in data.get('context_fields',[]): + for key in FIELDS: + if present(field.get(key)): _put(state,'context-field',field['field_name'],key,'obtained',call.tool,source,field[key]) + + +def filter_callable(state, catalog): + result={} + from agent.orchestration.contracts import ToolCall + for name,spec in catalog.items(): + if name=='ACQUIRE_MISSING_METADATA': + if not any(not blocked(ToolCall(name,{'missing_metadata_type':key}),state) + for key in state.get('missing_metadata_types',[])): continue + elif blocked(ToolCall(name,{}),state): continue + result[name]=spec + return result diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/pool_bindings.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/pool_bindings.py new file mode 100644 index 0000000..18359c0 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/pool_bindings.py @@ -0,0 +1,56 @@ +"""Controller-owned argument transport; no semantic or execution decisions.""" +from copy import deepcopy + +from agent.orchestration.agent_loop import ActionError +from agent.orchestration.contracts import ToolCall +from agent.orchestration.validator import resolve_reference + +BINDING_RULE_VERSION = 'canonical-state-bindings-v1' +EVIDENCE_SCHEMA_VERSION = 'typed-observed-evidence-v2' + + +def strict_equal(left,right): + if type(left) is not type(right): + return False + if isinstance(right,dict): + return left.keys()==right.keys() and all(strict_equal(left[k],v) for k,v in right.items()) + if isinstance(right,(list,tuple)): + return len(left)==len(right) and all(strict_equal(a,b) for a,b in zip(left,right)) + return left==right + + +def bind_call(call,spec,record,state,binding_resolver): + """Fill required state parameters; validate explicit values before normalization.""" + allowed=set(spec.required_inputs)|set(spec.optional_inputs) + if call.tool!=spec.name or set(call.arguments)-allowed: + raise ActionError('binding contains unknown Skill or arguments') + normalized={}; changes=[] + for key in (*spec.required_inputs,*spec.optional_inputs): + supplied=key in call.arguments + if call.tool=='ACQUIRE_MISSING_METADATA' and key=='missing_metadata_type': + choice=call.arguments.get(key) + if not isinstance(choice,str) or choice not in state.get('missing_metadata_types',[]): + raise ActionError('missing_metadata_type choice must be an actual missing type') + normalized[key]=choice + continue + reference=binding_resolver(key,record,state) + if reference is None: + if supplied or key in spec.required_inputs: + raise ActionError(f'binding unavailable for {key}') + continue + found,value=resolve_reference(reference,record,state) + if not found: + raise ActionError(f'binding unresolved for {key}') + if supplied: + original=call.arguments[key] + if not (type(original) is str and original==reference) and not strict_equal(original,value): + raise ActionError(f'argument {key} differs from canonical binding {reference}') + normalized[key]=reference + if original!=reference: + changes.append({'argument':key,'kind':'literal_to_reference','from':deepcopy(original),'to':reference}) + elif key in spec.required_inputs: + normalized[key]=reference + changes.append({'argument':key,'kind':'injected','to':reference}) + canonical=ToolCall(call.tool,normalized) + return canonical,{'version':BINDING_RULE_VERSION,'raw_call':deepcopy(call.to_dict()), + 'normalized_call':deepcopy(canonical.to_dict()),'changes':changes} diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/pool_catalog.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/pool_catalog.py new file mode 100644 index 0000000..4fd6b2d --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/pool_catalog.py @@ -0,0 +1,111 @@ +"""Frozen P14/F66 interfaces and one-time separate contract installation.""" +from pathlib import Path +from dataclasses import replace +import json +from agent.orchestration.catalog import SkillSpec, default_catalog, dynamic_loop_catalog + +ROOT = Path(__file__).resolve().parent +REFERENCE = json.loads((ROOT/'skill_reference.json').read_text(encoding='utf-8')) +NAMES = json.loads((ROOT/'reference_skills_63.json').read_text(encoding='utf-8')) +SHARED = dynamic_loop_catalog(include_field_metadata=True, include_description_skills=True, two_stage=True) +SEARCH_NAMES = ('SEARCH_STANDARD','SEARCH_STANDARD_BY_FIELD','SEARCH_STANDARD_BY_CONTEXT','SEARCH_STANDARD_BY_TABLE','SEARCH_STANDARD_BY_DATABASE','RESEARCH_STANDARD') +OUTPUTS = { + 'GET_FIELD_DESCRIPTION':'field_description','GET_TABLE_DESCRIPTION':'table_description', + 'GET_DATABASE_DESCRIPTION':'database_description','GET_TABLE_FIELDS':'table_fields', + 'GET_CONTEXT_FIELD_DESCRIPTIONS':'context_field_descriptions','GET_CONTEXT_FIELD_TYPES':'context_field_types', + 'GET_CONTEXT_FIELD_VALUES':'context_field_values','SELECT_CONTEXT_FIELDS':'selected_context_fields', + 'EXPAND_SELECTED_CONTEXT':'context_fields','GET_COMPLETE_TABLE_CONTEXT':'table_context', + 'GET_COMPLETE_DATABASE_CONTEXT':'database_context','ACQUIRE_MISSING_METADATA':'acquired_metadata', + 'INTERPRET_FIELD_TYPE':'field_type_semantics','INTERPRET_FIELD_VALUE':'value_semantics', + 'EXPAND_TABLE_ABBREVIATION':'table_abbreviation_semantics','EXPAND_DATABASE_ABBREVIATION':'database_abbreviation_semantics', + 'INTERPRET_DATABASE_NAME':'database_semantics','INTERPRET_DATABASE_DESCRIPTION':'database_description_semantics', + 'REFINE_STANDARD_QUERY':'refined_semantic_query','SUMMARIZE_STANDARD_EVIDENCE':'standard_evidence_summary', + 'FUSE_FIELD_EVIDENCE':'field_evidence','FUSE_CONTEXT_EVIDENCE':'context_evidence', + 'FUSE_TABLE_EVIDENCE':'table_evidence','FUSE_DATABASE_EVIDENCE':'database_evidence', + 'FUSE_METADATA_EVIDENCE':'metadata_evidence','FUSE_STANDARD_EVIDENCE':'standard_evidence', + 'WEIGH_EVIDENCE':'evidence_weights','RESOLVE_FIELD_CONTEXT_CONFLICT':'field_context_resolution', + 'RESOLVE_METADATA_CONFLICT':'metadata_conflict_resolution','RESOLVE_STANDARD_CONFLICT':'standard_conflict_resolution', + 'ASSESS_CATEGORY_SUPPORT':'category_support','COMPARE_CATEGORY_HYPOTHESES':'category_comparison', + 'IDENTIFY_MISSING_EVIDENCE':'missing_evidence','DECIDE_NEXT_EVIDENCE_SOURCE':'next_evidence_source', + 'VERIFY_CATEGORY_WITH_STANDARD':'category_verification', + 'VERIFY_CLASSIFICATION_GRADING_CONSISTENCY':'grading_consistency', +} + + +def _inputs(text): + return () if text in ('','无') else tuple(v.strip() for v in text.split(',')) + + +def pool_catalog(pool: str): + if pool not in ('P14','F66'): + raise ValueError('pool must be P14 or F66') + if pool=='P14': + return dict(SHARED) + legacy=default_catalog() + result={} + for role,names in NAMES.items(): + if not isinstance(names,list): + continue + for name in names: + if name in SHARED: + result[name]=SHARED[name] + elif name in legacy: + result[name]=legacy[name] + if name=='SYNTHESIZE_METADATA_SEMANTICS': + result[name]=replace(result[name],optional_inputs=_inputs(REFERENCE[name]['optional'])) + else: + ref=REFERENCE[name] + produced='standard_candidates' if name in SEARCH_NAMES else OUTPUTS.get(name,name.lower()) + result[name]=SkillSpec(name,role,_inputs(ref['required']),_inputs(ref['optional']),(produced,)) + result.update(SHARED) + return result + + +def purpose(name): + if name in REFERENCE: + return REFERENCE[name]['purpose'] + return { + 'GET_FIELD_METADATA':'Retrieve actual allowlisted metadata for the privately identified target record.', + 'SELECT_CATEGORY_TOPK':'Rank exactly five actual frozen Registry options by supplied evidence.', + 'INFER_CATEGORY_FROM_TOPK':'Select one actual prior Top-5 option and bind its frozen standard entry.' + }[name] + + +def install_contracts(skill_root): + """Explicit pre-freeze action; creates missing contracts, never edits existing files.""" + destination=Path(skill_root)/'full_pool' + existing=[p for p in (ROOT/'runtime/skills').rglob('*.md') if 'full_pool' not in p.parts] + patches=[] + for name,spec in pool_catalog('F66').items(): + target=destination/f'{spec.role}_{name}.md' + if target.exists(): + continue + source=next((p for p in existing if p.name==target.name and ('neighbor_extra' not in p.parts)),None) + if source is not None and name!='SYNTHESIZE_METADATA_SEMANTICS': + content=source.read_bytes().decode('utf-8') + else: + header='\n'.join([ + '---',f'name: {name}',f'dsl: {spec.role}', + f'required_inputs: [{", ".join(spec.required_inputs)}]', + f'optional_inputs: [{", ".join(spec.optional_inputs)}]', + f'produces: [{", ".join(spec.produces)}]',f'terminal: {str(spec.terminal).lower()}','---','', + ]) + ref=REFERENCE[name] + body=f'# {name}\n\nPurpose: {purpose(name)}\n\nObservation: {ref["output"]}\n\n' + boundaries={ + 'metadata_acquisition':'Use deterministic canonical Val access only. Verify private record identity. Never invent missing descriptions, types, values or locators; explicitly report unavailable data. Context belongs to the actual domain/database/table and has bounded, disclosed coverage.', + 'semantic_interpretation':'Interpret only supplied evidence. Separate direct evidence, auxiliary evidence, and uncertain hypotheses. Do not retrieve standards or predict category/grade. Never turn inferred semantics into raw metadata.', + 'standard_alignment':'Use only the actual frozen standard candidates and their disclosed descriptions. Explain support, contradictions, auxiliary context and unresolved distinctions. Never invent standard entries or overwrite canonical candidate/category/grade state.', + 'evidence_reasoning':'Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state.' + } + body+=boundaries[spec.role]+'\n\nReturn exactly {"summary": "nonempty natural-language observation", "data": {...}}. ' + body+=f'Data keys must be exactly {json.dumps(spec.produces)}. ' + body+='Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys.\n' + content=header+body + patches.append((target,content)) + # Runtime contract installation is an explicit artifact generation operation. + destination.mkdir(parents=True,exist_ok=True) + for target,content in patches: + with target.open('xb') as stream: + stream.write(content.encode('utf-8')) + return destination diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/pool_metadata.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/pool_metadata.py new file mode 100644 index 0000000..a7bc2d9 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/pool_metadata.py @@ -0,0 +1,111 @@ +"""Private-identity, label-free access to one canonical Val metadata source.""" +import copy +import json +import math +from pathlib import Path +from collections.abc import Mapping + +FIELDS=('field_name','database_name','database_description','table_name','table_description','field_description','field_type','value') +MISSING={'','none','null','n/a','nan'} + + +def present(value): + if value is None: + return False + if isinstance(value,str): + return value.strip().casefold() not in MISSING + if isinstance(value,float) and math.isnan(value): + return False + return bool(value) or value==0 + + +class PoolMetadata: + def __init__(self,rows): + if not isinstance(rows,list): + raise ValueError('canonical source must be a list') + self._rows={} + for row in rows: + if not isinstance(row,Mapping) or not isinstance(row.get('id'),str) or not row['id'] or row['id'] in self._rows: + raise ValueError('canonical rows require unique nonempty private IDs') + raw=row.get('metadata') + if not isinstance(raw,Mapping) or not present(row.get('domain')) or not present(raw.get('field_name')): + raise ValueError('canonical row requires real domain and field_name') + fields={k:copy.deepcopy(raw[k]) for k in FIELDS if present(raw.get(k))} + self._rows[row['id']]={'domain':row['domain'],'fields':fields} + + @classmethod + def from_canonical(cls,path): + return cls(json.loads(Path(path).read_text(encoding='utf-8'))) + + def _entry(self,record): + entry=self._rows.get(record.source_id) + if entry is None: + raise ValueError('source id is absent from canonical metadata lookup') + if entry['domain']!=record.domain or entry['fields']['field_name']!=record.field_name: + raise ValueError('private lookup does not match domain and field_name') + return entry + + def get(self,record): + return copy.deepcopy(self._entry(record)['fields']) + + def _same_table(self,record): + entry=self._entry(record); fields=entry['fields'] + db,table=fields.get('database_name'),fields.get('table_name') + if not present(db) or not present(table): + return [] + return sorted([r['fields'] for r in self._rows.values() if r['domain']==record.domain and r['fields'].get('database_name')==db and r['fields'].get('table_name')==table],key=lambda r:(str(r['field_name']),json.dumps(r,sort_keys=True,ensure_ascii=False))) + + def table_fields(self,record): + own=self.get(record) + neighbors=[r for r in self._same_table(record) if r['field_name']!=record.field_name] + # One actual field name can have several source rows. Keep deterministic first. + unique={} + for row in neighbors: + unique.setdefault(row['field_name'],row) + neighbors=list(unique.values()) + visible=[{k:copy.deepcopy(v) for k,v in r.items() if k in ('field_name','field_description','field_type','value')} for r in neighbors[:30]] + return {'database_name':own.get('database_name'),'table_name':own.get('table_name'),'fields':visible, + 'available_count':len(neighbors),'returned_count':len(visible),'truncated':len(neighbors)>30, + 'omitted_count':max(0,len(neighbors)-30),'coverage':'canonical-val-same-table-only', + 'missing_locators':[k for k in ('database_name','table_name') if k not in own]} + + def expand(self,record,names): + if not isinstance(names,list) or not names or any(not isinstance(n,str) or not n.strip() for n in names) or len(set(names))!=len(names): + raise ValueError('selected context must be a nonempty distinct list of actual field names') + actual={r['field_name']:r for r in self.table_fields(record)['fields']} + if any(name not in actual for name in names): + raise ValueError('context field is outside actual bounded same-table candidates') + return [copy.deepcopy(actual[n]) for n in names] + + def table_context(self,record): + own=self.get(record) + context=self.table_fields(record) + return {'target':own,**context,'missing_target_attributes':[k for k in FIELDS if k not in own]} + + def database_context(self,record): + own=self.get(record) + db=own.get('database_name') + if not present(db): + return {'tables':[],'coverage':'canonical-val-available-tables-only','missing_locators':['database_name'],'truncated':False,'omitted_count':0,'available_count':0} + tables={} + omitted_rows=0 + for entry in self._rows.values(): + fields=entry['fields'] + if entry['domain']!=record.domain or fields.get('database_name')!=db: + continue + if not present(fields.get('table_name')): + omitted_rows+=1 + continue + table=tables.setdefault(fields['table_name'],{'table_name':fields['table_name'],'field_names':set(),'descriptions':set()}) + table['field_names'].add(fields['field_name']) + if present(fields.get('table_description')): + table['descriptions'].add(fields['table_description']) + ordered=[] + for name in sorted(tables): + table=tables[name]; fields=sorted(table['field_names']) + ordered.append({'table_name':name,'table_descriptions':sorted(table['descriptions']), + 'field_names':fields[:30],'available_field_count':len(fields),'omitted_field_count':max(0,len(fields)-30)}) + return {'database_name':db,'database_description':own.get('database_description'), + 'tables':ordered[:10],'available_count':len(ordered),'returned_count':min(10,len(ordered)), + 'truncated':len(ordered)>10,'omitted_count':max(0,len(ordered)-10),'omitted_unlocated_rows':omitted_rows, + 'coverage':'canonical-val-available-tables-only','missing_locators':[]} diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/pool_runtime.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/pool_runtime.py new file mode 100644 index 0000000..2fdeb0b --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/pool_runtime.py @@ -0,0 +1,564 @@ +"""Common dynamic controller, scoped metadata paths and native semantic execution.""" +import json +from collections.abc import Mapping +from dataclasses import replace +from pathlib import Path + +from agent.orchestration.agent_loop import (AgentLoopController, LoopConfig, LoopState, PolicyDecision, + ActionError, parse_policy_action, build_final_evidence_state) +from agent.orchestration.category_catalog import CategoryCatalog +from agent.orchestration.contracts import ToolCall +from agent.orchestration.model_skill_executor import (ModelSkillExecutor, SkillExecutionError, + parse_skill_observation, _generation_metadata) +from agent.orchestration.runtime import Observation +from agent.orchestration.skill_loader import SkillLoader +from pool_catalog import ROOT, pool_catalog, purpose, SEARCH_NAMES +from pool_metadata import PoolMetadata, FIELDS, present +from pool_bindings import bind_call, BINDING_RULE_VERSION +from comparison_output import parse_comparison_output, ComparisonOutputError, MAX_REASON_LENGTH +from evidence_roles import acquired_source, evidence_roles +from grounded_semantics import TARGETS as GROUNDED_TARGETS, valid_shape as grounded_shape, make_messages as grounded_messages, validate_observation as validate_grounded + +POLICY_SYSTEM = '''You are a database-field classification orchestration policy. +Choose exactly one currently callable Skill, using the global capability index to plan a minimum sufficient evidence program. +Only domain and field_name are visible initially. Other raw metadata requires explicit acquisition; pending metadata is not evidence. +Choose the Skill; the controller binds required state arguments. Omit controller-bound arguments, or supply their exact advertised references or strictly identical values. Explicit values are checked before normalization. +ACQUIRE_MISSING_METADATA requires an explicit missing_metadata_type chosen from actual missing_metadata_types. Optional arguments may be supplied only when actually bound. Never use provenance source names as argument data. +Never invent raw metadata, standard entries, labels or grading levels. Intermediate semantic strings are evidence, not acquired raw values. +Stop acquiring when evidence is sufficient, select a supported category, confirm its frozen grade and finalize within the remaining budget. +Return exactly one JSON object: {"tool":"SKILL_NAME","arguments":{}}. No Program, Markdown, plain stop, or extra JSON.''' + +SEMANTIC_KEYS = ( + 'domain_semantics','field_abbreviation_semantics','field_semantics','disambiguated_field_semantics', + 'field_description_semantics','field_type_semantics','value_semantics','table_abbreviation_semantics', + 'table_semantics','table_description_semantics','database_abbreviation_semantics','database_semantics', + 'database_description_semantics','context_field_semantics','metadata_semantics', + 'field_evidence','context_evidence','table_evidence','database_evidence','metadata_evidence', +) +ALIGNMENT_KEYS=('field_standard_alignment','context_standard_alignment','table_standard_alignment', + 'database_standard_alignment','value_standard_alignment','description_standard_alignment','standard_comparison') + +# Every validated, declared-type native output participates in the evidence +# boundary, including refinement, conflict resolution and support assessments. +SEMANTIC_KEYS=tuple(dict.fromkeys((*SEMANTIC_KEYS,*(key for spec in pool_catalog('F66').values() + if spec.role in ('semantic_interpretation','evidence_reasoning') for key in spec.produces + if key not in ('category','grading_level','aligned_standard_entry','final_result'))))) +ALIGNMENT_KEYS=tuple(dict.fromkeys((*ALIGNMENT_KEYS,*(key for spec in pool_catalog('F66').values() + if spec.role=='standard_alignment' for key in spec.produces + if key not in ('standard_candidates','aligned_standard_entry'))))) + + +def valid_evidence_value(key,value): + if key in GROUNDED_TARGETS.values(): + return grounded_shape(value) + if key=='evidence_sufficiency': + return (isinstance(value,Mapping) and set(value)=={'sufficient','rationale'} + and type(value['sufficient']) is bool + and isinstance(value['rationale'],str) and bool(value['rationale'].strip())) + return isinstance(value,str) and bool(value.strip()) + + +def validated_evidence_values(state,keys): + catalog=pool_catalog('F66');provenance=state.get('_provenance',{}) + return {key:state[key] for key in keys + if key in state and provenance.get(key) in catalog + and key in catalog[provenance[key]].produces + and valid_evidence_value(key,state[key])} + + +def acquired_raw(record,state): + provenance=state.get('_provenance',{}) + return {'field_name':record.field_name,**{k:state[k] for k in FIELDS + if k!='field_name' and acquired_source(k,provenance.get(k)) and present(state.get(k))}} + + +def assemble_pool_evidence(record,state): + """Single allowlisted boundary for publication, model search and finalization.""" + provenance=state.get('_provenance',{}) + semantics=validated_evidence_values(state,SEMANTIC_KEYS) + alignments=validated_evidence_values(state,ALIGNMENT_KEYS) + observations={k:state[k] for k in ('table_fields','selected_context_fields','context_fields', + 'context_field_descriptions','context_field_types','context_field_values','table_context','database_context') + if acquired_source(k,provenance.get(k)) and present(state.get(k))} + metadata=acquired_raw(record,state) if record is not None else dict(state.get('current_metadata',{})) + names=set(metadata)|set(semantics)|set(alignments)|set(observations) + prior=state.get('available_evidence',{}) + return {'domain':record.domain if record is not None else prior.get('domain'), + 'field_name':record.field_name if record is not None else metadata.get('field_name'), + 'metadata':metadata,'semantics':semantics,'alignments':alignments,'observations':observations, + 'provenance':{k:provenance[k] for k in sorted(names) if k in provenance}, + 'roles':evidence_roles(metadata,semantics,alignments,observations,provenance)} + + +def pool_final_evidence(record,state): + final=build_final_evidence_state(state) + if final is not None: + evidence=assemble_pool_evidence(record,state) + final['semantic_evidence']={**evidence['semantics'],**evidence['alignments']} + final['available_evidence']=evidence + return final + + +def publish_state(record,state,data,skill): + """Publish only typed observed data and explicit evidence bundles, with provenance.""" + spec=pool_catalog('F66').get(skill) + if spec is None or not set(data)<=set(spec.produces): + raise SkillExecutionError('publication contains undeclared outputs or unknown Skill') + for key,value in data.items(): + if key in (*SEMANTIC_KEYS,*ALIGNMENT_KEYS) and not valid_evidence_value(key,value): + raise SkillExecutionError(f'{key} violates its declared evidence type') + state.update(data) + provenance=state.setdefault('_provenance',{}) + provenance.update({k:skill for k in data}) + if isinstance(data.get('metadata_availability'),Mapping): + state['current_metadata']=acquired_raw(record,state) + state['available_metadata']={'domain':record.domain,**state['current_metadata']} + missing=data['metadata_availability'].get('missing',[]) + state['missing_metadata_types']=[k for k in missing if k in FIELDS] + if state['missing_metadata_types']: + state['missing_metadata_type']=state['missing_metadata_types'][0] + provenance['missing_metadata_type']=skill + else: + state.pop('missing_metadata_type',None) + provenance.pop('missing_metadata_type',None) + raw={} + for key in ('field_metadata','acquired_metadata'): + value=data.get(key) + if isinstance(value,Mapping): + raw.update({k:v for k,v in value.items() if k in FIELDS and present(v)}) + if skill.startswith('GET_'): + raw.update({k:v for k,v in data.items() if k in FIELDS and present(v)}) + for key,value in raw.items(): + state[key]=value; provenance[key]=skill + if raw: + # Initial field_name belongs to the record, not necessarily to state. + # Publish the same acquired-only boundary consumed by the executor. + state['current_metadata']=acquired_raw(record,state) + state['available_metadata']={'domain':record.domain,**state['current_metadata']} + state['missing_metadata_types']=[k for k in FIELDS if k not in state['current_metadata']] + if state['missing_metadata_types']: + state['missing_metadata_type']=state['missing_metadata_types'][0] + provenance['missing_metadata_type']=skill + else: + state.pop('missing_metadata_type',None) + provenance.pop('missing_metadata_type',None) + aliases={'context_field_semantics':'context_semantics','field_semantics':'candidate_field_semantics', + 'category':'aligned_category','grading_level':'grading_level_candidate'} + for original,alias in aliases.items(): + if original in state and present(state[original]): + state[alias]=state[original]; provenance[alias]=provenance.get(original,skill) + if 'category' in state: + for alias in ('candidate_category','category_candidate'): + state[alias]=state['category']; provenance[alias]=provenance.get('category',skill) + if isinstance(state.get('standard_candidates'),list) and state['standard_candidates']: + for alias in ('candidate_standard_categories','candidate_categories','previous_standard_results'): + state[alias]=state['standard_candidates']; provenance[alias]=provenance.get('standard_candidates',skill) + semantics=validated_evidence_values(state,SEMANTIC_KEYS) + alignments=validated_evidence_values(state,ALIGNMENT_KEYS) + groups={ + 'available_field_evidence':{k:v for k,v in semantics.items() if k.startswith(('field_','disambiguated_','value_'))}, + 'context_field_evidence':{k:v for k,v in semantics.items() if k.startswith('context_')}, + 'available_table_evidence':{k:v for k,v in semantics.items() if k.startswith('table_')}, + 'available_database_evidence':{k:v for k,v in semantics.items() if k.startswith('database_')}, + 'available_metadata_evidence':semantics,'standard_alignment_evidence':alignments, + 'aligned_standard_candidates':alignments,'available_semantic_context':{k:v for k,v in semantics.items() if k not in ('field_semantics','field_abbreviation_semantics')}, + } + if semantics: + for name in ('semantic_evidence','current_semantic_state','semantic_query','current_evidence', + 'current_evidence_state','all_available_evidence','evidence','evidence_set'): + groups[name]=semantics + # Multiple model interpretations do not establish a business-data conflict. + if len(alignments)>=2: + groups['conflicting_standard_alignments']=alignments + for name,value in groups.items(): + if value: + state[name]=value; provenance[name]='validated-evidence-bundle' + # Raw retrieved metadata is also usable search evidence, without pretending it is semantics. + if isinstance(state.get('field_metadata'),Mapping) or semantics: + state['available_evidence']=assemble_pool_evidence(record,state) + provenance['available_evidence']='validated-evidence-bundle' + state['all_available_evidence']=state['available_evidence'] + provenance['all_available_evidence']='validated-evidence-bundle' + if 'semantic_query' not in state: + state['semantic_query']=state['available_evidence'] + evidence=pool_final_evidence(record,state) + if evidence is not None: + state['final_evidence_state']=evidence + provenance['final_evidence_state']='frozen-classification-bundle' + + +def _binding(name,record,state): + if name=='current_record': return '$record' + if name=='domain': return '$domain' + if name in ('field_name','target_field'): return '$metadata.field_name' + if name=='table_fields': + value=state.get(name) + if not isinstance(value,Mapping) or value.get('coverage')!='canonical-val-same-table-only' or value.get('missing_locators')!=[]: + return None + fields=value.get('fields') + if (not isinstance(fields,list) or not fields or len(fields)>30 + or type(value.get('returned_count')) is not int or value['returned_count']!=len(fields) + or type(value.get('available_count')) is not int or value['available_count'] None: + if self.max_turns < 1: + raise ValueError("max_turns must be at least 1") + if self.max_identical_calls < 1: + raise ValueError("max_identical_calls must be at least 1") + if ( + self.action_retries < 0 + or self.forced_finalize_retries < 0 + or self.repetition_retries < 0 + ): + raise ValueError("retry limits must be non-negative") + + +@dataclass(frozen=True) +class LoopEvent: + kind: str + turn: int + data: Mapping[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return {"kind": self.kind, "turn": self.turn, "data": dict(self.data)} + + +@dataclass +class LoopState: + input: dict[str, str] + named_state: dict[str, Any] = field(default_factory=dict) + call_history: list[dict[str, Any]] = field(default_factory=list) + turns: int = 0 + max_turns: int = 8 + + @classmethod + def from_record(cls, record: MetadataRecord, *, max_turns: int) -> "LoopState": + return cls( + input={"domain": record.domain, "field_name": record.field_name}, + max_turns=max_turns, + ) + + @property + def remaining_turns(self) -> int: + return max(self.max_turns - self.turns, 0) + + def to_model_dict(self) -> dict[str, Any]: + return { + "input": dict(self.input), + "named_state": dict(self.named_state), + "call_history": [dict(item) for item in self.call_history], + "turns": self.turns, + "remaining_turns": self.remaining_turns, + } + + +@dataclass(frozen=True) +class PolicyDecision: + call: ToolCall + raw_response: str + messages: tuple[dict[str, str], ...] = () + metadata: Mapping[str, Any] = field(default_factory=dict) + + +@dataclass +class LoopTrace: + input: dict[str, str] + events: list[LoopEvent] = field(default_factory=list) + terminated: bool = False + final_result: dict[str, Any] | None = None + stop_reason: str = "not_started" + turns: int = 0 + state: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "input": dict(self.input), + "events": [event.to_dict() for event in self.events], + "terminated": self.terminated, + "final_result": dict(self.final_result) if self.final_result is not None else None, + "stop_reason": self.stop_reason, + "turns": self.turns, + "state": dict(self.state), + } + + +class LoopPolicy(Protocol): + def select_action( + self, + record: MetadataRecord, + state: LoopState, + callable_skills: Mapping[str, SkillSpec], + *, + feedback: str | None = None, + forced_finalize: bool = False, + ) -> PolicyDecision: ... + + +class PrefetchFieldMetadataPolicy: + """Spend the first normal turn querying metadata, then delegate unchanged.""" + + def __init__(self, delegate: LoopPolicy) -> None: + self.delegate = delegate + + def select_action( + self, + record: MetadataRecord, + state: LoopState, + callable_skills: Mapping[str, SkillSpec], + *, + feedback: str | None = None, + forced_finalize: bool = False, + ) -> PolicyDecision: + if ( + not forced_finalize + and state.turns == 0 + and not any(item.get("tool") == "GET_FIELD_METADATA" for item in state.call_history) + and "GET_FIELD_METADATA" in callable_skills + ): + call = ToolCall("GET_FIELD_METADATA", {"current_record": "$record"}) + return PolicyDecision( + call=call, + raw_response=json.dumps(call.to_dict(), ensure_ascii=False), + metadata={"fixed_metadata_prefetch": True}, + ) + return self.delegate.select_action( + record, state, callable_skills, + feedback=feedback, forced_finalize=forced_finalize, + ) + + +class PrefetchNeighborFieldsPolicy: + """Fixed one-time neighbor lookup followed by an unchanged dynamic policy.""" + + def __init__(self, delegate: LoopPolicy) -> None: + self.delegate = delegate + + def select_action( + self, + record: MetadataRecord, + state: LoopState, + callable_skills: Mapping[str, SkillSpec], + *, + feedback: str | None = None, + forced_finalize: bool = False, + ) -> PolicyDecision: + previously_called = any( + item.get("tool") == "GET_TABLE_FIELDS" for item in state.call_history + ) + if ( + not forced_finalize + and not previously_called + and "table_fields" not in state.named_state + and "GET_TABLE_FIELDS" in callable_skills + ): + call = ToolCall("GET_TABLE_FIELDS", {"current_record": "$record"}) + return PolicyDecision( + call=call, + raw_response=json.dumps(call.to_dict(), ensure_ascii=False), + metadata={"fixed_neighbor_prefetch": True}, + ) + return self.delegate.select_action( + record, state, callable_skills, + feedback=feedback, forced_finalize=forced_finalize, + ) + + +class SkillExecutor(Protocol): + def execute( + self, + call: ToolCall, + record: MetadataRecord, + state: Mapping[str, Any], + contract: SkillContract, + ) -> Observation: ... + + +def _decode_one_object(text: str) -> Mapping[str, Any]: + if not isinstance(text, str) or not text.strip(): + raise ActionError("policy action must be non-empty JSON") + cleaned = text.strip() + if cleaned.startswith("```"): + lines = cleaned.splitlines()[1:] + if lines and lines[-1].strip() == "```": + lines.pop() + cleaned = "\n".join(lines).strip() + if not cleaned.startswith("{"): + raise ActionError("policy action must be one JSON object") + decoder = json.JSONDecoder() + try: + value, end = decoder.raw_decode(cleaned) + except JSONDecodeError as exc: + raise ActionError(f"policy action is invalid JSON: {exc.msg}") from exc + if cleaned[end:].strip(): + raise ActionError("policy action contains trailing content") + if not isinstance(value, Mapping): + raise ActionError("policy action must be one Tool Call JSON object") + return value + + +def parse_policy_action(text: str) -> ToolCall: + """Parse exactly one Tool Call and reject Programs or plain stop tokens.""" + + if isinstance(text, str) and text.strip().lower().rstrip(".!" ) in { + "stop", + "done", + "finish", + "finished", + "terminate", + }: + raise EarlyStopError("policy stopped without a final result") + payload = _decode_one_object(text) + if set(payload) != {"tool", "arguments"}: + raise ActionError("policy action must be one Tool Call with tool and arguments") + try: + return ToolCall.from_mapping(payload) + except (TypeError, ValueError) as exc: + raise ActionError(f"invalid Tool Call: {exc}") from exc + + +def _has_semantic_state(state: Mapping[str, Any]) -> bool: + return any( + name in state + for name in ( + "field_semantics", + "field_abbreviation_semantics", + "disambiguated_field_semantics", + "domain_semantics", + "field_metadata", + ) + ) + + +_SEMANTIC_EVIDENCE_KEYS = ( + "domain_semantics", + "field_abbreviation_semantics", + "field_semantics", + "disambiguated_field_semantics", + "field_description_semantics", + "table_description_semantics", + "standard_comparison", +) + +_TWO_STAGE_EVIDENCE_KEYS = ( + "domain_semantics", + "field_abbreviation_semantics", + "field_semantics", + "disambiguated_field_semantics", + "field_metadata", + "table_fields", + "field_description_semantics", + "table_semantics", + "table_description_semantics", + "context_field_semantics", + "metadata_semantics", + "standard_comparison", +) + + +def _two_stage_evidence(record: MetadataRecord, state: Mapping[str, Any]) -> dict[str, Any]: + return { + "domain": record.domain, + "field_name": record.field_name, + "observations": {key: state[key] for key in _TWO_STAGE_EVIDENCE_KEYS if key in state}, + } + + +def build_final_evidence_state(state: Mapping[str, Any]) -> dict[str, Any] | None: + """Create the auditable evidence bundle required by the terminal Skill.""" + + category = state.get("category") + level = state.get("grading_level") + aligned = state.get("aligned_standard_entry") + candidates = state.get("standard_candidates") + if ( + not isinstance(category, str) + or not category + or not isinstance(level, str) + or not level + or not isinstance(aligned, Mapping) + or not isinstance(candidates, Sequence) + or isinstance(candidates, (str, bytes)) + or not candidates + ): + return None + semantic_evidence = { + key: state[key] + for key in _SEMANTIC_EVIDENCE_KEYS + if key in state and state[key] not in (None, "") + } + return { + "category": category, + "grading_level": level, + "aligned_standard_entry": dict(aligned), + "standard_candidates": [ + dict(item) if isinstance(item, Mapping) else item for item in candidates + ], + "semantic_evidence": semantic_evidence, + } + + +def _input_available(name: str, record: MetadataRecord, state: Mapping[str, Any]) -> bool: + if name == "current_record": + return True + if name in {"domain", "field_name"}: + return record.available(name) + if name in state: + return state[name] is not None and state[name] != "" + if name == "semantic_query": + return _has_semantic_state(state) + if name in {"semantic_evidence", "current_evidence", "all_available_evidence"}: + return bool(state) + if name == "candidate_standard_categories": + return bool(state.get("standard_candidates")) + if name == "aligned_category": + return bool(state.get("category")) + if name == "final_evidence_state": + return isinstance(state.get("final_evidence_state"), Mapping) + return False + + +def derive_callable_skills( + record: MetadataRecord, + state: LoopState | Mapping[str, Any], + catalog: Mapping[str, SkillSpec] | None = None, +) -> dict[str, SkillSpec]: + """Return Skills whose required semantic inputs currently exist.""" + + named = state.named_state if isinstance(state, LoopState) else state + active = catalog or dynamic_loop_catalog() + return { + name: spec + for name, spec in active.items() + if all(_input_available(required, record, named) for required in spec.required_inputs) + } + + +def _catalog_payload(catalog: Mapping[str, SkillSpec]) -> list[dict[str, Any]]: + return [ + { + "name": spec.name, + "required_inputs": list(spec.required_inputs), + "optional_inputs": list(spec.optional_inputs), + "produces": list(spec.produces), + "terminal": spec.terminal, + } + for spec in catalog.values() + ] + + +def build_policy_messages( + record: MetadataRecord, + state: LoopState, + callable_skills: Mapping[str, SkillSpec], + *, + feedback: str | None = None, + forced_finalize: bool = False, +) -> list[dict[str, str]]: + """Build a model prompt from the minimum visible input and current states.""" + + payload: dict[str, Any] = { + "task": "Choose the next Skill from the callable catalog or finalize when evidence is sufficient.", + "current": state.to_model_dict(), + "callable_skills": _catalog_payload(callable_skills), + "forced_finalize": forced_finalize, + "argument_reference_grammar": [ + "$record", + "$domain", + "$metadata.field_name", + "$state.", + ], + "output_schema": {"tool": "SKILL_NAME", "arguments": {}}, + } + if feedback: + payload["validation_feedback"] = feedback + return [ + {"role": "system", "content": POLICY_SYSTEM_PROMPT}, + {"role": "user", "content": json.dumps(payload, ensure_ascii=False, indent=2)}, + ] + + +class ScriptedLoopPolicy: + """Deterministic policy used to test controller behavior without a model.""" + + def __init__(self, responses: Iterable[str | ToolCall | Mapping[str, Any]]) -> None: + self.responses = deque(responses) + self.seen_states: list[dict[str, Any]] = [] + self.seen_feedback: list[str | None] = [] + self.seen_forced: list[bool] = [] + + def select_action( + self, + record: MetadataRecord, + state: LoopState, + callable_skills: Mapping[str, SkillSpec], + *, + feedback: str | None = None, + forced_finalize: bool = False, + ) -> PolicyDecision: + del record, callable_skills + self.seen_states.append(state.to_model_dict()) + self.seen_feedback.append(feedback) + self.seen_forced.append(forced_finalize) + if not self.responses: + raise ActionError("scripted policy has no response remaining") + response = self.responses.popleft() + if isinstance(response, ToolCall): + call = response + raw = json.dumps(call.to_dict(), ensure_ascii=False) + elif isinstance(response, Mapping): + raw = json.dumps(dict(response), ensure_ascii=False) + call = parse_policy_action(raw) + else: + raw = response + call = parse_policy_action(raw) + return PolicyDecision(call=call, raw_response=raw) + + +class FixedPipelinePolicy: + """Reference policy that follows the same five evidence steps for every row.""" + + def select_action( + self, + record: MetadataRecord, + state: LoopState, + callable_skills: Mapping[str, SkillSpec], + *, + feedback: str | None = None, + forced_finalize: bool = False, + ) -> PolicyDecision: + del record, feedback, forced_finalize + named = state.named_state + if "field_semantics" not in named: + call = ToolCall( + "INTERPRET_FIELD_NAME", + {"field_name": "$metadata.field_name", "domain": "$domain"}, + ) + elif "standard_candidates" not in named: + call = ToolCall("SEARCH_STANDARD", {"semantic_query": "$state.field_semantics"}) + elif "category" not in named: + call = ToolCall( + "INFER_CATEGORY", + { + "candidate_standard_categories": "$state.standard_candidates", + "all_available_evidence": "$state.field_semantics", + }, + ) + elif "grading_level" not in named: + call = ToolCall( + "INFER_GRADING_LEVEL", + { + "aligned_category": "$state.category", + "aligned_standard_entry": "$state.aligned_standard_entry", + }, + ) + else: + call = ToolCall( + "FINALIZE_CLASSIFICATION", + {"final_evidence_state": "$state.final_evidence_state"}, + ) + if call.tool not in callable_skills: + raise ActionError( + f"fixed pipeline requires {call.tool}, but it is not callable from the current state" + ) + raw = json.dumps(call.to_dict(), ensure_ascii=False, separators=(",", ":")) + return PolicyDecision(call=call, raw_response=raw) + + +class ScriptedSkillExecutor: + """Return predefined Observations while recording executed tools.""" + + def __init__(self, outputs: Mapping[str, Sequence[Observation]]) -> None: + self.outputs = {name: deque(values) for name, values in outputs.items()} + self.executed_tools: list[str] = [] + + def execute( + self, + call: ToolCall, + record: MetadataRecord, + state: Mapping[str, Any], + contract: SkillContract, + ) -> Observation: + del record, state, contract + self.executed_tools.append(call.tool) + queue = self.outputs.get(call.tool) + if not queue: + return Observation(call.tool, "error", "no scripted output", {}) + return queue.popleft() + + +def _canonical_call(call: ToolCall) -> str: + return json.dumps(call.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _validate_call_arguments( + call: ToolCall, + spec: SkillSpec, + record: MetadataRecord, + state: Mapping[str, Any], +) -> str | None: + allowed = set(spec.required_inputs) | set(spec.optional_inputs) + unknown = sorted(set(call.arguments) - allowed) + if unknown: + return f"unknown arguments for {call.tool}: {unknown}" + for name in spec.required_inputs: + if name not in call.arguments: + return f"missing required argument {name!r}" + if (call.tool == "INFER_CATEGORY_FROM_TOPK" + and call.arguments.get("candidate_standard_categories") + != "$state.standard_candidates"): + return ( + "candidate_standard_categories must be the exact reference " + "$state.standard_candidates; do not rewrite prior candidate IDs" + ) + for name, value in call.arguments.items(): + found, _ = resolve_reference(value, record, state) + if not found: + return f"argument {name!r} cannot be resolved" + return None + + +class AgentLoopController: + """Run one validated Skill at a time with controller-owned safeguards.""" + + def __init__( + self, + *, + policy: LoopPolicy, + executor: SkillExecutor, + catalog: Mapping[str, SkillSpec] | None = None, + skill_loader: SkillLoader, + config: LoopConfig | None = None, + registry_categories: Iterable[str] = (), + registry_entries: Mapping[str, Mapping[str, Any]] | None = None, + approved_levels: Iterable[str] = ("L1", "L2", "L3", "L4"), + callable_derivation=None, + state_publication=None, + ) -> None: + self.policy = policy + self.executor = executor + self.catalog = dict(catalog or dynamic_loop_catalog()) + self.skill_loader = skill_loader + self.config = config or LoopConfig() + self.registry_entries = { + str(category): dict(entry) + for category, entry in (registry_entries or {}).items() + } + self.registry_categories = frozenset( + {str(item) for item in registry_categories} | set(self.registry_entries) + ) + self.approved_levels = frozenset(str(item) for item in approved_levels) + self.callable_derivation = callable_derivation or derive_callable_skills + self.state_publication = state_publication + + def _normalize_policy_decision(self, decision, record, state, spec): + """Extension point for controller-owned transport normalization.""" + return decision + + def _rejected_policy_metadata(self, decision, error): + return {} + + def _valid_terminal(self, result: Any, state: Mapping[str, Any]) -> bool: + if not isinstance(result, Mapping): + return False + category = result.get("category") + level = result.get("grading_level") + if not isinstance(category, str) or not category: + return False + if self.registry_categories and category not in self.registry_categories: + return False + if not isinstance(level, str) or level not in self.approved_levels: + return False + + frozen_entry = self.registry_entries.get(category) + if frozen_entry is not None and frozen_entry.get("grading_level") != level: + return False + + aligned = state.get("aligned_standard_entry") + if not isinstance(aligned, Mapping): + return False + aligned_category = aligned.get("category") or aligned.get("category_id") + if aligned_category != category or aligned.get("grading_level") != level: + return False + + candidates = state.get("standard_candidates") + if not isinstance(candidates, Sequence) or isinstance(candidates, (str, bytes)): + return False + if not any( + isinstance(candidate, Mapping) + and (candidate.get("category") or candidate.get("category_id")) == category + for candidate in candidates + ): + return False + + evidence = state.get("final_evidence_state") + return ( + isinstance(evidence, Mapping) + and evidence.get("category") == category + and evidence.get("grading_level") == level + ) + + def _normalize_observation( + self, + call: ToolCall, + observation: Observation, + state: Mapping[str, Any], + ) -> Observation: + """Bind model-selected evidence back to exact frozen controller objects.""" + + if observation.status != "success": + return observation + if call.tool in {"INFER_CATEGORY", "INFER_CATEGORY_FROM_TOPK"}: + category = observation.data.get("category") + candidates = state.get("standard_candidates") + matched = None + if isinstance(category, str) and isinstance(candidates, Sequence): + matched = next( + ( + candidate + for candidate in candidates + if isinstance(candidate, Mapping) + and (candidate.get("category") or candidate.get("category_id")) + == category + ), + None, + ) + if not isinstance(matched, Mapping): + return Observation( + call.tool, + "error", + "model-selected category is not one of the retrieved candidates", + {}, + metadata=dict(observation.metadata), + ) + data = dict(observation.data) + original = data.get("aligned_standard_entry") + if call.tool == "INFER_CATEGORY_FROM_TOPK": + frozen = self.registry_entries.get(category) + if not isinstance(frozen, Mapping): + return Observation(call.tool, "error", "selected category is absent from frozen registry", {}) + data["aligned_standard_entry"] = {"category": category, **dict(frozen)} + else: + data["aligned_standard_entry"] = dict(matched) + metadata = dict(observation.metadata) + metadata["controller_normalized_aligned_entry"] = original != data["aligned_standard_entry"] + return Observation( + observation.skill, + observation.status, + observation.summary, + data, + metadata=metadata, + ) + if call.tool == "INFER_GRADING_LEVEL": + level = observation.data.get("grading_level") + if not isinstance(level, str) or level not in self.approved_levels: + return Observation( + call.tool, + "error", + "model-produced grading level is empty or not approved", + {}, + metadata=dict(observation.metadata), + ) + return observation + + def _ready_finalization(self, state: Mapping[str, Any]) -> bool: + """Require the current bundle to agree with exact frozen registry evidence.""" + evidence = state.get("final_evidence_state") + if not self._valid_terminal(evidence, state): + return False + category = state.get("category") + frozen = self.registry_entries.get(category) if isinstance(category, str) else None + return ( + isinstance(frozen, Mapping) + and evidence.get("category") == category + and evidence.get("grading_level") == state.get("grading_level") + and state.get("aligned_standard_entry") == {"category": category, **dict(frozen)} + and evidence == self._final_evidence_state(None, state) + ) + + def _available_evidence(self, record, state): + return _two_stage_evidence(record, state) + + def _final_evidence_state(self, record, state): + return build_final_evidence_state(state) + + def run(self, record: MetadataRecord) -> LoopTrace: + visible_record = MetadataRecord.from_mapping( + {"domain": record.domain, "field_name": record.field_name, + "source_id": record.source_id} + ) + state = LoopState.from_record(visible_record, max_turns=self.config.max_turns) + trace = LoopTrace(input=dict(state.input), stop_reason="running") + identical_counts: Counter[str] = Counter() + repetition_rejections = 0 + invalid_terminal_count = 0 + feedback: str | None = None + recovery_attempted = False + + while state.turns < self.config.max_turns: + forced = state.remaining_turns == 1 + recovering = False + callable_skills = self.callable_derivation(visible_record, state, self.catalog) + decision: PolicyDecision | None = None + if forced: + if "FINALIZE_CLASSIFICATION" not in callable_skills: + trace.events.append( + LoopEvent( + "controller_intervention", + state.turns + 1, + {"reason": "forced_finalization_missing_evidence"}, + ) + ) + trace.stop_reason = "budget_exhausted_no_result" + break + call = ToolCall( + "FINALIZE_CLASSIFICATION", + {"final_evidence_state": "$state.final_evidence_state"}, + ) + raw = json.dumps(call.to_dict(), ensure_ascii=False, separators=(",", ":")) + decision = PolicyDecision(call=call, raw_response=raw) + callable_skills = { + "FINALIZE_CLASSIFICATION": callable_skills["FINALIZE_CLASSIFICATION"] + } + trace.events.append( + LoopEvent( + "controller_intervention", + state.turns + 1, + {"reason": "forced_finalization", "call": call.to_dict()}, + ) + ) + else: + invalid_reason = "invalid_action" + for attempt in range(self.config.action_retries + 1): + try: + decision = self.policy.select_action( + visible_record, + state, + callable_skills, + feedback=feedback, + forced_finalize=False, + ) + if decision.call.tool not in callable_skills: + invalid_reason = "uncallable_skill" + raise ActionError( + f"{decision.call.tool} is not callable; choices={list(callable_skills)}" + ) + spec = callable_skills[decision.call.tool] + decision = self._normalize_policy_decision( + decision, visible_record, state.named_state, spec + ) + argument_error = _validate_call_arguments( + decision.call, spec, visible_record, state.named_state + ) + if argument_error: + invalid_reason = "invalid_arguments" + raise ActionError(argument_error) + break + except ActionError as exc: + if isinstance(exc, EarlyStopError): + invalid_reason = "stop_without_final_result" + rejected_data: dict[str, Any] = { + "reason": invalid_reason, + "error": str(exc), + "attempt": attempt + 1, + } + if decision is not None: + rejected_data["raw_response"] = decision.raw_response + rejected_data["messages"] = [dict(m) for m in decision.messages] + rejected_data["metadata"] = dict(decision.metadata) + elif isinstance(getattr(exc, "metadata", None), Mapping): + rejected_data["metadata"] = dict(exc.metadata) + rejection_metadata = self._rejected_policy_metadata(decision, exc) + if rejection_metadata: + rejected_data.setdefault("metadata", {}).update(rejection_metadata) + decision = None + feedback = str(exc) + trace.events.append( + LoopEvent( + "retry" if attempt < self.config.action_retries else "rejected_action", + state.turns + 1, + rejected_data, + ) + ) + if attempt == self.config.action_retries: + if ( + self.config.recover_ready_finalization + and not recovery_attempted + and "FINALIZE_CLASSIFICATION" in callable_skills + and self._ready_finalization(state.named_state) + ): + recovery_attempted = True + recovering = True + forced = True + call = ToolCall( + "FINALIZE_CLASSIFICATION", + {"final_evidence_state": "$state.final_evidence_state"}, + ) + decision = PolicyDecision( + call=call, + raw_response=json.dumps(call.to_dict(), ensure_ascii=False), + ) + trace.events.append(LoopEvent( + "controller_intervention", state.turns + 1, + {"reason": "invalid_action_finalization_recovery", "call": call.to_dict()}, + )) + break + if invalid_reason == "stop_without_final_result": + trace.stop_reason = "stop_without_final_result" + else: + trace.stop_reason = "invalid_action" + trace.turns = state.turns + trace.state = dict(state.named_state) + return trace + + assert decision is not None + if forced: + decision = self._normalize_policy_decision( + decision, visible_record, state.named_state, self.catalog[decision.call.tool] + ) + call = decision.call + canonical = _canonical_call(call) + if identical_counts[canonical] >= self.config.max_identical_calls: + repetition_rejections += 1 + recoverable = ( + not recovering + and repetition_rejections <= self.config.repetition_retries + ) + trace.events.append( + LoopEvent( + "controller_intervention", + state.turns + 1, + { + "reason": "repetition_limit", + "call": call.to_dict(), + "recovered": recoverable, + "attempt": repetition_rejections, + "messages": [dict(m) for m in decision.messages], + "raw_response": decision.raw_response, + "metadata": dict(decision.metadata), + }, + ) + ) + if recoverable: + feedback = ( + f"Repeated call {call.tool} was blocked; choose a different callable Skill." + ) + continue + trace.stop_reason = "repetition_limit" + trace.turns = state.turns + trace.state = dict(state.named_state) + return trace + identical_counts[canonical] += 1 + + if decision.metadata.get("fixed_metadata_prefetch"): + trace.events.append( + LoopEvent( + "controller_intervention", state.turns + 1, + {"reason": "metadata_prefetch", "call": call.to_dict()}, + ) + ) + trace.events.append( + LoopEvent( + "policy_action", + state.turns + 1, + { + "call": call.to_dict(), + "raw_response": decision.raw_response, + "messages": [dict(message) for message in decision.messages], + "metadata": dict(decision.metadata), + "forced_finalize": forced, + }, + ) + ) + contract = self.skill_loader.load(call.tool) + try: + observation = self.executor.execute( + call, visible_record, dict(state.named_state), contract + ) + except Exception as exc: + metadata = getattr(exc, "metadata", {}) + observation = Observation( + call.tool, "error", str(exc), {}, + metadata=dict(metadata) if isinstance(metadata, Mapping) else {}, + ) + observation = self._normalize_observation( + call, observation, state.named_state + ) + + state.turns += 1 + state.call_history.append(call.to_dict()) + trace.events.append( + LoopEvent( + "observation", + state.turns, + { + "skill": observation.skill, + "status": observation.status, + "summary": observation.summary, + "data": dict(observation.data), + "metadata": dict(observation.metadata), + }, + ) + ) + feedback = None + + if observation.status != "success": + feedback = f"Skill {call.tool} failed: {observation.summary}" + if recovering: + trace.stop_reason = "recovery_finalization_failed" + break + if forced or state.turns >= self.config.max_turns: + trace.stop_reason = "budget_exhausted_no_result" + break + continue + + state.named_state.update(observation.data) + if self.state_publication is not None: + self.state_publication(visible_record, state.named_state, observation.data, call.tool) + if call.tool == "GET_FIELD_METADATA" and "INTERPRET_FIELD_DESCRIPTION" in self.catalog: + metadata = observation.data.get("field_metadata") + if isinstance(metadata, Mapping): + for key in ("field_description", "table_description"): + value = metadata.get(key) + if isinstance(value, str) and value.strip(): + state.named_state[key] = value + if "INFER_CATEGORY_FROM_TOPK" in self.catalog: + state.named_state["available_evidence"] = self._available_evidence( + visible_record, state.named_state + ) + evidence = self._final_evidence_state(visible_record, state.named_state) + if evidence is None: + state.named_state.pop("final_evidence_state", None) + else: + state.named_state["final_evidence_state"] = evidence + if call.tool != "FINALIZE_CLASSIFICATION": + continue + + result = observation.data.get("final_result") + if self._valid_terminal(result, state.named_state): + trace.terminated = True + trace.final_result = dict(result) + trace.stop_reason = "completed" + break + + invalid_terminal_count += 1 + feedback = "terminal result must contain one registry category and approved grading level" + trace.events.append( + LoopEvent( + "retry", + state.turns, + {"reason": "invalid_terminal", "result": result}, + ) + ) + if recovering: + trace.stop_reason = "invalid_terminal" + break + if invalid_terminal_count > self.config.forced_finalize_retries: + trace.stop_reason = "invalid_terminal" + break + if forced: + trace.stop_reason = "budget_exhausted_no_result" + break + + if trace.stop_reason == "running": + trace.stop_reason = "budget_exhausted_no_result" + trace.turns = state.turns + trace.state = dict(state.named_state) + return trace + + +__all__ = [ + "ActionError", + "AgentLoopController", + "EarlyStopError", + "FixedPipelinePolicy", + "LoopConfig", + "LoopEvent", + "LoopPolicy", + "LoopState", + "PrefetchNeighborFieldsPolicy", + "PrefetchFieldMetadataPolicy", + "LoopTrace", + "POLICY_SYSTEM_PROMPT", + "PolicyDecision", + "ScriptedLoopPolicy", + "ScriptedSkillExecutor", + "SkillExecutor", + "build_final_evidence_state", + "build_policy_messages", + "derive_callable_skills", + "parse_policy_action", +] diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/artifact_projection.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/artifact_projection.py new file mode 100644 index 0000000..e2b1250 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/artifact_projection.py @@ -0,0 +1,67 @@ +"""Project in-memory Agent Loop traces into label-isolated run artifacts.""" + +from __future__ import annotations + +from typing import Any, Mapping + + +def project_trace(row: Mapping[str, Any]) -> dict[str, Any]: + """Split one scored row into observation, model-log, trajectory, and evaluation views.""" + + source_id = row.get("source_id") + trace = row.get("trace") or {} + observations: list[dict[str, Any]] = [] + model_calls: list[dict[str, Any]] = [] + actions: list[dict[str, Any]] = [] + for event in trace.get("events", []): + if not isinstance(event, Mapping): + continue + kind = event.get("kind") + turn = event.get("turn") + data = event.get("data") or {} + if not isinstance(data, Mapping): + continue + if kind == "policy_action": + call = data.get("call") + if isinstance(call, Mapping): + actions.append({"turn": turn, "tool": call.get("tool"), + "arguments": dict(call.get("arguments") or {})}) + model_calls.append({ + "source_id": source_id, "turn": turn, "kind": "policy_action", + "skill": (call or {}).get("tool") if isinstance(call, Mapping) else None, + "messages": data.get("messages"), "raw_response": data.get("raw_response"), + "metadata": data.get("metadata") or {}, + }) + elif kind == "observation": + observations.append({ + "source_id": source_id, "turn": turn, "skill": data.get("skill"), + "status": data.get("status"), "summary": data.get("summary"), + "data": dict(data.get("data") or {}), + }) + metadata = data.get("metadata") or {} + if metadata: + model_calls.append({ + "source_id": source_id, "turn": turn, "kind": "skill_execution", + "skill": data.get("skill"), "metadata": dict(metadata), + }) + trajectory = { + "source_id": source_id, + "input": dict(trace.get("input") or {}), + "actions": actions, + "observations": [{key: value for key, value in item.items() if key != "source_id"} + for item in observations], + "final_result": trace.get("final_result"), + "stop_reason": trace.get("stop_reason"), + } + evaluation = { + "source_id": source_id, + "ground_truth": row.get("ground_truth"), + "ground_truth_level": row.get("ground_truth_level"), + "predicted_category": row.get("predicted_category"), + "predicted_level": row.get("predicted_level"), + "category_correct": row.get("category_correct"), + "level_correct": row.get("level_correct"), + "joint_correct": row.get("joint_correct"), + } + return {"observations": observations, "model_calls": model_calls, + "trajectory": trajectory, "evaluation": evaluation} diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/catalog.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/catalog.py new file mode 100644 index 0000000..da77ce4 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/catalog.py @@ -0,0 +1,105 @@ +"""First-version Skill catalog used by the local planner/validator.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Mapping + + +@dataclass(frozen=True) +class SkillSpec: + name: str + role: str + required_inputs: tuple[str, ...] + optional_inputs: tuple[str, ...] = () + produces: tuple[str, ...] = () + max_calls: int = 1 + terminal: bool = False + + +def default_catalog() -> dict[str, SkillSpec]: + """Return a fresh ordered catalog for the first local experiment.""" + + specs = ( + SkillSpec("CHECK_METADATA_AVAILABILITY", "metadata_acquisition", ("current_record",), produces=("metadata_availability",)), + SkillSpec("GET_TABLE_FIELDS", "metadata_acquisition", ("table_name",), produces=("table_fields",)), + SkillSpec("SELECT_CONTEXT_FIELDS", "metadata_acquisition", ("target_field", "table_fields"), optional_inputs=("current_semantic_state",), produces=("selected_context_fields",)), + SkillSpec("INTERPRET_DOMAIN", "semantic_interpretation", ("domain",), produces=("domain_semantics",)), + SkillSpec("EXPAND_FIELD_ABBREVIATION", "semantic_interpretation", ("field_name",), optional_inputs=("domain",), produces=("field_abbreviation_semantics",)), + SkillSpec("INTERPRET_FIELD_NAME", "semantic_interpretation", ("field_name",), optional_inputs=("domain", "field_abbreviation_semantics"), produces=("field_semantics",)), + SkillSpec("DISAMBIGUATE_FIELD_SEMANTICS", "semantic_interpretation", ("field_semantics",), optional_inputs=("domain", "standard_candidates"), produces=("disambiguated_field_semantics",)), + SkillSpec("INTERPRET_FIELD_DESCRIPTION", "semantic_interpretation", ("field_description",), optional_inputs=("field_name", "domain"), produces=("field_description_semantics",)), + SkillSpec("INTERPRET_TABLE_NAME", "semantic_interpretation", ("table_name",), optional_inputs=("domain", "table_description"), produces=("table_semantics",)), + SkillSpec("INTERPRET_TABLE_DESCRIPTION", "semantic_interpretation", ("table_description",), optional_inputs=("table_name", "domain"), produces=("table_description_semantics",)), + SkillSpec("INTERPRET_CONTEXT_FIELDS", "semantic_interpretation", ("context_fields",), optional_inputs=("domain", "table_name"), produces=("context_field_semantics",)), + SkillSpec("SYNTHESIZE_METADATA_SEMANTICS", "semantic_interpretation", ("field_semantics",), optional_inputs=("domain_semantics", "table_semantics", "context_field_semantics", "field_description_semantics"), produces=("metadata_semantics",)), + SkillSpec("SEARCH_STANDARD", "standard_alignment", ("semantic_query",), produces=("standard_candidates",)), + SkillSpec("ALIGN_FIELD_TO_STANDARD", "standard_alignment", ("field_semantics", "standard_candidates"), produces=("field_standard_alignment",)), + SkillSpec("COMPARE_STANDARD_CANDIDATES", "standard_alignment", ("standard_candidates", "semantic_evidence"), produces=("standard_comparison", "aligned_standard_entry")), + SkillSpec("CHECK_EVIDENCE_SUFFICIENCY", "evidence_reasoning", ("current_evidence",), produces=("evidence_sufficiency",)), + SkillSpec("INFER_CATEGORY", "evidence_reasoning", ("candidate_standard_categories", "all_available_evidence"), produces=("category", "aligned_standard_entry")), + SkillSpec("INFER_GRADING_LEVEL", "evidence_reasoning", ("aligned_category", "aligned_standard_entry"), produces=("grading_level",)), + SkillSpec("FINALIZE_CLASSIFICATION", "evidence_reasoning", ("final_evidence_state",), produces=("final_result",), terminal=True), + ) + return {spec.name: spec for spec in specs} + + +DYNAMIC_LOOP_SKILLS = ( + "CHECK_METADATA_AVAILABILITY", + "INTERPRET_DOMAIN", + "EXPAND_FIELD_ABBREVIATION", + "INTERPRET_FIELD_NAME", + "DISAMBIGUATE_FIELD_SEMANTICS", + "SEARCH_STANDARD", + "COMPARE_STANDARD_CANDIDATES", + "CHECK_EVIDENCE_SUFFICIENCY", + "INFER_CATEGORY", + "INFER_GRADING_LEVEL", + "FINALIZE_CLASSIFICATION", +) + + +def dynamic_loop_catalog( + *, include_field_metadata: bool = False, + include_table_fields: bool = False, + include_description_skills: bool = False, + two_stage: bool = False, +) -> dict[str, SkillSpec]: + """Return the approved ordered Skill subset for the dynamic pilot.""" + + catalog = default_catalog() + names = tuple( + "SELECT_CATEGORY_TOPK" if name == "SEARCH_STANDARD" else + "INFER_CATEGORY_FROM_TOPK" if name == "INFER_CATEGORY" else name + for name in DYNAMIC_LOOP_SKILLS + ) if two_stage else DYNAMIC_LOOP_SKILLS + if two_stage: + catalog.update({ + "SELECT_CATEGORY_TOPK": SkillSpec( + "SELECT_CATEGORY_TOPK", "standard_alignment", ("semantic_query",), + produces=("standard_candidates",), + ), + "INFER_CATEGORY_FROM_TOPK": SkillSpec( + "INFER_CATEGORY_FROM_TOPK", "evidence_reasoning", + ("candidate_standard_categories", "all_available_evidence"), + produces=("category", "aligned_standard_entry"), + ), + }) + selected = {name: catalog[name] for name in names} + if include_field_metadata: + selected["GET_FIELD_METADATA"] = SkillSpec( + "GET_FIELD_METADATA", "metadata_acquisition", ("current_record",), + produces=("field_metadata",), + ) + if include_table_fields: + selected["GET_TABLE_FIELDS"] = SkillSpec( + "GET_TABLE_FIELDS", "metadata_acquisition", ("current_record",), + produces=("table_fields",), + ) + if include_description_skills: + for name in ("INTERPRET_FIELD_DESCRIPTION", "INTERPRET_TABLE_DESCRIPTION"): + selected[name] = catalog[name] + return selected + + +__all__ = ["DYNAMIC_LOOP_SKILLS", "SkillSpec", "default_catalog", "dynamic_loop_catalog"] diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/category_catalog.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/category_catalog.py new file mode 100644 index 0000000..9696dd4 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/category_catalog.py @@ -0,0 +1,100 @@ +"""Frozen Registry views with a strict label-only Top-K information boundary.""" + +from __future__ import annotations + +from typing import Any, Mapping, Sequence + + +class CategoryCatalog: + """Expose labels first and descriptions only for validated candidate IDs.""" + + def __init__(self, registry_entries: Mapping[str, Mapping[str, Any]]) -> None: + self._entries = {str(key): dict(value) for key, value in registry_entries.items()} + if not self._entries: + raise ValueError("category registry is empty") + width = max(3, len(str(len(self._entries)))) + self._option_to_category = { + f"C{index:0{width}d}": category + for index, category in enumerate(sorted(self._entries), start=1) + } + self._category_to_option = { + category: option_id for option_id, category in self._option_to_category.items() + } + + def selection_options(self) -> list[dict[str, str]]: + """Return stable short choices and full paths, without canonical IDs or details.""" + + labels = {entry["category"]: entry["label"] for entry in self.labels()} + return [ + {"option_id": option_id, "label": labels[category]} + for option_id, category in self._option_to_category.items() + ] + + def resolve_option_ids(self, option_ids: Sequence[str], k: int) -> tuple[str, ...]: + """Validate ranked model choices before translating to frozen category IDs.""" + + if k < 1 or k > len(self._entries): + raise ValueError("top_k must be between 1 and registry size") + if not isinstance(option_ids, (list, tuple)) or len(option_ids) != k: + raise ValueError(f"top_k result must contain exactly {k} option IDs") + if any(not isinstance(item, str) or item not in self._option_to_category + for item in option_ids): + raise ValueError("top_k result contains an unknown option ID") + if len(set(option_ids)) != len(option_ids): + raise ValueError("top_k result contains duplicate option IDs") + return tuple(self._option_to_category[item] for item in option_ids) + + def option_details(self, ids: Sequence[str]) -> list[dict[str, str]]: + """Expose descriptions only for selected choices, still without long IDs.""" + + return [ + { + "option_id": self._category_to_option[item["category"]], + "label": item["label"], + "description": item["description"], + } + for item in self.details(ids) + ] + + def labels(self) -> list[dict[str, str]]: + result: list[dict[str, str]] = [] + for category, entry in self._entries.items(): + path = entry.get("path") + if isinstance(path, (list, tuple)) and path: + label = " > ".join(str(part) for part in path) + else: + label = str(entry.get("label") or category.partition(":")[2] or entry.get("name") or category) + result.append({"category": category, "label": label}) + return result + + def validate_top_k(self, ids: Sequence[str], k: int) -> tuple[str, ...]: + if k < 1 or k > len(self._entries): + raise ValueError("top_k must be between 1 and registry size") + if not isinstance(ids, (list, tuple)) or len(ids) != k: + raise ValueError(f"top_k result must contain exactly {k} IDs") + if any(not isinstance(item, str) or item not in self._entries for item in ids): + raise ValueError("top_k result contains an unknown category ID") + if len(set(ids)) != len(ids): + raise ValueError("top_k result contains duplicate category IDs") + return tuple(ids) + + def details(self, ids: Sequence[str]) -> list[dict[str, str]]: + if not isinstance(ids, (list, tuple)) or not ids or len(set(ids)) != len(ids): + raise ValueError("candidate IDs must be a nonempty distinct list") + labels = {entry["category"]: entry["label"] for entry in self.labels()} + result: list[dict[str, str]] = [] + for category in ids: + if category not in self._entries: + raise ValueError(f"unknown category ID: {category}") + entry = self._entries[category] + result.append({ + "category": category, + "label": labels[category], + "description": str(entry.get("description") or entry.get("definition") or ""), + }) + return result + + def frozen_entry(self, category: str) -> dict[str, Any]: + if category not in self._entries: + raise ValueError(f"unknown category ID: {category}") + return {"category": category, **self._entries[category]} diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/contracts.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/contracts.py new file mode 100644 index 0000000..d7634ae --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/contracts.py @@ -0,0 +1,120 @@ +"""JSON-compatible contracts for the DSL orchestration layer.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Mapping + + +OPTIONAL_METADATA_FIELDS = ( + "database_name", + "database_description", + "table_name", + "table_description", + "field_description", + "field_type", + "value", +) + + +def _text(value: Any) -> str: + return "" if value is None else str(value).strip() + + +@dataclass(frozen=True) +class MetadataRecord: + """One classification instance with required and optional metadata.""" + + domain: str + field_name: str + metadata: Mapping[str, Any] = field(default_factory=dict) + source_id: str | None = None + + @classmethod + def from_mapping(cls, raw: Mapping[str, Any]) -> "MetadataRecord": + nested = raw.get("metadata") + nested_metadata = dict(nested) if isinstance(nested, Mapping) else {} + merged = dict(nested_metadata) + for key in ("field_name", *OPTIONAL_METADATA_FIELDS): + if key in raw and raw[key] is not None: + merged[key] = raw[key] + + domain = _text(raw.get("domain", merged.pop("domain", None))) + field_name = _text(merged.get("field_name")) + if not domain: + raise ValueError("domain is required and must be non-empty") + if not field_name: + raise ValueError("field_name is required and must be non-empty") + merged["field_name"] = field_name + return cls( + domain=domain, + field_name=field_name, + metadata=merged, + source_id=_text(raw.get("source_id")) or None, + ) + + def get(self, name: str, default: Any = None) -> Any: + if name == "domain": + return self.domain + if name == "field_name": + return self.field_name + return self.metadata.get(name, default) + + def available(self, name: str) -> bool: + value = self.get(name) + return bool(_text(value)) + + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = {"domain": self.domain, "metadata": dict(self.metadata)} + if self.source_id: + result["source_id"] = self.source_id + return result + + +@dataclass(frozen=True) +class ToolCall: + tool: str + arguments: Mapping[str, Any] = field(default_factory=dict) + + @classmethod + def from_mapping(cls, raw: Mapping[str, Any]) -> "ToolCall": + tool = _text(raw.get("tool")) + if not tool: + raise ValueError("tool is required and must be non-empty") + arguments = raw.get("arguments", {}) + if not isinstance(arguments, Mapping): + raise ValueError("arguments must be an object") + return cls(tool=tool, arguments=dict(arguments)) + + def to_dict(self) -> dict[str, Any]: + return {"tool": self.tool, "arguments": dict(self.arguments)} + + +@dataclass(frozen=True) +class Program: + steps: tuple[ToolCall, ...] + + @classmethod + def from_mapping(cls, raw: Mapping[str, Any]) -> "Program": + steps = raw.get("program") + if not isinstance(steps, list): + raise ValueError("program must be a list") + return cls(steps=tuple(ToolCall.from_mapping(item) for item in steps)) + + def to_dict(self) -> dict[str, Any]: + return {"program": [step.to_dict() for step in self.steps]} + + +@dataclass(frozen=True) +class StandardEntry: + category: str + description: str + grading_level: str + + def to_dict(self) -> dict[str, str]: + return { + "category": self.category, + "description": self.description, + "grading_level": self.grading_level, + } + diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/metadata_lookup.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/metadata_lookup.py new file mode 100644 index 0000000..7de1ab4 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/metadata_lookup.py @@ -0,0 +1,56 @@ +"""Label-isolated lookup of real field metadata for an explicit DSL call.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Mapping + +from .contracts import MetadataRecord + + +RETRIEVABLE_FIELDS = ( + "database_name", "table_name", "field_description", "table_description" +) + + +class FieldMetadataLookup: + def __init__(self, rows: Mapping[str, Mapping[str, Any]]) -> None: + self._rows = {key: dict(value) for key, value in rows.items()} + + @classmethod + def from_canonical(cls, source: str | Path) -> "FieldMetadataLookup": + raw = json.loads(Path(source).read_text(encoding="utf-8")) + if not isinstance(raw, list): + raise ValueError("canonical source must be a list") + projected: dict[str, dict[str, Any]] = {} + for row in raw: + if not isinstance(row, Mapping): + raise ValueError("canonical row must be an object") + source_id = row.get("id") + if not isinstance(source_id, str) or not source_id: + raise ValueError("canonical row must have id") + if source_id in projected: + raise ValueError(f"duplicate source id: {source_id}") + metadata = row.get("metadata") + if not isinstance(metadata, Mapping): + raise ValueError(f"canonical metadata missing for {source_id}") + projected[source_id] = { + "domain": row.get("domain"), + "field_name": metadata.get("field_name"), + "fields": { + key: str(metadata[key]).strip() + for key in RETRIEVABLE_FIELDS + if metadata.get(key) not in (None, "") + and str(metadata[key]).strip().casefold() not in ('', 'none', 'null', 'n/a') + }, + } + return cls(projected) + + def get(self, record: MetadataRecord) -> dict[str, str]: + if not record.source_id or record.source_id not in self._rows: + raise ValueError("source id is absent from metadata lookup") + entry = self._rows[record.source_id] + if entry["domain"] != record.domain or entry["field_name"] != record.field_name: + raise ValueError("lookup source does not match domain and field_name") + return dict(entry["fields"]) diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/metrics.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/metrics.py new file mode 100644 index 0000000..d3ba05b --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/metrics.py @@ -0,0 +1,48 @@ +"""Auditable metrics for local Program execution traces.""" + +from __future__ import annotations + +from collections import Counter +from typing import Any, Iterable + +from .runtime import ExecutionTrace, Observation + + +def _value(item: Observation | dict[str, Any], key: str, default: Any = None) -> Any: + if isinstance(item, Observation): + return getattr(item, key, default) + return item.get(key, default) + + +def summarize_trace(trace: ExecutionTrace) -> dict[str, Any]: + names = [_value(item, "skill", "") for item in trace.observations] + statuses = [_value(item, "status", "error") for item in trace.observations] + counts = Counter(names) + return { + "tool_calls": len(names), + "successful_tool_calls": sum(status == "success" for status in statuses), + "failed_tool_calls": sum(status != "success" for status in statuses), + "tool_call_success_rate": (sum(status == "success" for status in statuses) / len(statuses)) if statuses else 1.0, + "program_length": len(names), + "duplicate_calls": sum(max(count - 1, 0) for count in counts.values()), + "terminated": bool(trace.terminated), + "final_result_valid": isinstance(trace.final_result, dict) and bool(trace.final_result), + } + + +def summarize_traces(traces: Iterable[ExecutionTrace]) -> dict[str, Any]: + rows = [summarize_trace(trace) for trace in traces] + count = len(rows) + calls = sum(row["tool_calls"] for row in rows) + successful = sum(row["successful_tool_calls"] for row in rows) + return { + "traces": count, + "termination_rate": (sum(row["terminated"] for row in rows) / count) if count else 0.0, + "average_program_length": (sum(row["program_length"] for row in rows) / count) if count else 0.0, + "tool_call_success_rate": (successful / calls) if calls else 1.0, + "average_duplicate_calls": (sum(row["duplicate_calls"] for row in rows) / count) if count else 0.0, + } + + +__all__ = ["summarize_trace", "summarize_traces"] + diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/model_skill_executor.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/model_skill_executor.py new file mode 100644 index 0000000..f61f797 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/model_skill_executor.py @@ -0,0 +1,837 @@ +"""Model-independent adapters for policy generation and semantic Skill execution.""" + +from __future__ import annotations + +import json +from dataclasses import replace +from json import JSONDecodeError +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +from .agent_loop import ( + ActionError, + LoopState, + PolicyDecision, + _two_stage_evidence, + build_policy_messages, + parse_policy_action, +) +from .catalog import SkillSpec +from .contracts import OPTIONAL_METADATA_FIELDS, MetadataRecord, ToolCall +from .metadata_lookup import FieldMetadataLookup +from .table_field_lookup import TableFieldLookup +from .category_catalog import CategoryCatalog +from .runtime import KnowledgeBase, Observation +from .skill_loader import SkillContract, SkillLoader +from .validator import resolve_reference +from .topk_selection import SELECTION_POLICY, aggregate_rankings, balanced_options, ranking_scores + + +TextGenerator = Callable[[list[dict[str, str]]], str] +DeterministicHandler = Callable[[ToolCall, MetadataRecord, Mapping[str, Any]], Observation] + + +def _generation_metadata(generate: TextGenerator) -> dict[str, Any]: + metadata = getattr(generate, "last_generation_metadata", None) + return dict(metadata) if isinstance(metadata, Mapping) else {} + + +def _attempt_metadata(attempts: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Keep every Skill generation while preserving the existing aggregate fields.""" + + last = attempts[-1] + metadata: dict[str, Any] = { + "attempts": [dict(item) for item in attempts], + "model_call_count": len(attempts), + "messages": last["messages"], + "raw_response": last.get("raw_response"), + } + generations = [item.get("generation", {}) for item in attempts] + usages = [item.get("token_usage") for item in generations if isinstance(item, Mapping)] + if any(isinstance(item, Mapping) for item in usages): + metadata["token_usage"] = { + key: sum(int(item.get(key) or 0) for item in usages if isinstance(item, Mapping)) + for key in ("prompt_tokens", "completion_tokens", "total_tokens") + } + elapsed = [item.get("elapsed_seconds") for item in generations if isinstance(item, Mapping)] + if any(isinstance(item, (int, float)) for item in elapsed): + metadata["elapsed_seconds"] = sum( + float(item) for item in elapsed if isinstance(item, (int, float)) + ) + return metadata + + +class SkillExecutionError(ValueError): + """Raised when a Skill cannot produce one valid Observation.""" + + def __init__(self, message: str, *, metadata: Mapping[str, Any] | None = None) -> None: + super().__init__(message) + self.metadata = dict(metadata or {}) + + +def metadata_availability_handler( + call: ToolCall, + record: MetadataRecord, + resolved: Mapping[str, Any], +) -> Observation: + """Report availability from the projected, model-visible record only.""" + + del resolved + available = [name for name in ("domain", "field_name") if record.available(name)] + missing = [name for name in OPTIONAL_METADATA_FIELDS if not record.available(name)] + return Observation( + call.tool, + "success", + f"Available metadata: {', '.join(available)}; missing: {', '.join(missing)}", + {"metadata_availability": {"available": available, "missing": missing}}, + ) + + +def make_standard_search_handler( + knowledge_base: KnowledgeBase, + *, + top_k: int = 5, +) -> DeterministicHandler: + """Create a frozen-registry Top-K retrieval handler.""" + + if top_k < 1: + raise ValueError("top_k must be positive") + + def handler( + call: ToolCall, + record: MetadataRecord, + resolved: Mapping[str, Any], + ) -> Observation: + del record + query = resolved.get("semantic_query") + candidates = knowledge_base.search_standard(query, top_k=top_k) + return Observation( + call.tool, + "success", + f"Retrieved {len(candidates)} standard candidates", + {"standard_candidates": candidates}, + ) + + return handler + + +def make_field_metadata_handler(lookup: FieldMetadataLookup) -> DeterministicHandler: + """Expose only allowlisted source metadata when the Agent invokes this Skill.""" + + def handler( + call: ToolCall, + record: MetadataRecord, + resolved: Mapping[str, Any], + ) -> Observation: + del resolved + fields = lookup.get(record) + return Observation( + call.tool, + "success", + f"Retrieved {len(fields)} approved metadata fields", + {"field_metadata": fields}, + ) + + return handler + + +def make_table_fields_handler(lookup: TableFieldLookup) -> DeterministicHandler: + """Expose bounded, label-free same-table fields only on an explicit call.""" + + def handler( + call: ToolCall, + record: MetadataRecord, + resolved: Mapping[str, Any], + ) -> Observation: + del resolved + fields = lookup.get(record) + return Observation( + call.tool, + "success", + f"Retrieved {len(fields['fields'])} of {fields['available_count']} neighboring fields", + {"table_fields": fields}, + ) + + return handler + + +def _decode_observation_object(text: str) -> Mapping[str, Any]: + if not isinstance(text, str) or not text.strip(): + raise SkillExecutionError("Skill response must be non-empty JSON") + cleaned = text.strip() + if cleaned.startswith("```"): + lines = cleaned.splitlines()[1:] + if lines and lines[-1].strip() == "```": + lines.pop() + cleaned = "\n".join(lines).strip() + if not cleaned.startswith("{"): + raise SkillExecutionError("Skill response must be one JSON object") + decoder = json.JSONDecoder() + try: + value, end = decoder.raw_decode(cleaned) + except JSONDecodeError as exc: + raise SkillExecutionError(f"Skill response is invalid JSON: {exc.msg}") from exc + if cleaned[end:].strip(): + raise SkillExecutionError("Skill response contains trailing content") + if not isinstance(value, Mapping): + raise SkillExecutionError("Skill response must be a JSON object") + return value + + +def parse_skill_observation(text: str, spec: SkillSpec) -> Observation: + """Parse one strict success Observation for the selected Skill.""" + + payload = _decode_observation_object(text) + if set(payload) != {"summary", "data"}: + raise SkillExecutionError("Skill response must contain exactly summary and data") + summary = payload.get("summary") + data = payload.get("data") + if not isinstance(summary, str) or not summary.strip(): + raise SkillExecutionError("Skill summary must be a non-empty string") + if not isinstance(data, Mapping): + raise SkillExecutionError("Skill data must be an object") + expected = set(spec.produces) + actual = set(data) + missing = sorted(expected - actual) + if missing: + raise SkillExecutionError(f"missing produced keys: {missing}") + extra = sorted(actual - expected) + if extra: + raise SkillExecutionError(f"unexpected produced keys: {extra}") + empty = sorted( + key + for key in expected + if data.get(key) is None or data.get(key) == "" + ) + if empty: + raise SkillExecutionError(f"produced keys must be non-empty: {empty}") + return Observation(spec.name, "success", summary.strip(), dict(data)) + + +def load_few_shot_messages(path: str | Path) -> tuple[dict[str, str], ...]: + """Load synthetic policy examples while rejecting evaluation-label leakage.""" + + source = Path(path) + payload = json.loads(source.read_text(encoding="utf-8")) + if not isinstance(payload, list) or not payload: + raise ValueError("few-shot file must contain a non-empty JSON list") + messages: list[dict[str, str]] = [] + forbidden = ("ground_truth", "ground truth", "source_id", "source id") + for index, item in enumerate(payload): + if not isinstance(item, Mapping) or set(item) != {"role", "content"}: + raise ValueError(f"few-shot message {index} must contain exactly role and content") + role = item.get("role") + content = item.get("content") + if role not in {"user", "assistant"} or not isinstance(content, str) or not content: + raise ValueError(f"few-shot message {index} has an invalid role or content") + lowered = content.lower() + if any(marker in lowered for marker in forbidden): + raise ValueError("few-shot examples must not contain ground-truth or source identifiers") + messages.append({"role": role, "content": content}) + return tuple(messages) + + +class ModelLoopPolicy: + """Adapt a text generator to the one-action dynamic policy contract.""" + + def __init__( + self, + generate: TextGenerator, + *, + few_shot_messages: Sequence[Mapping[str, str]] = (), + metadata_guidance: bool = False, + two_stage: bool = False, + description_skills: bool = False, + ) -> None: + self.generate = generate + self.few_shot_messages = tuple(dict(message) for message in few_shot_messages) + self.metadata_guidance = metadata_guidance + self.two_stage = two_stage + self.description_skills = description_skills + + def select_action( + self, + record: MetadataRecord, + state: LoopState, + callable_skills: Mapping[str, SkillSpec], + *, + feedback: str | None = None, + forced_finalize: bool = False, + ) -> PolicyDecision: + messages = build_policy_messages( + record, + state, + callable_skills, + feedback=feedback, + forced_finalize=forced_finalize, + ) + if self.two_stage: + messages[0]["content"] += ( + "\nUse only a Skill listed in callable_skills for this turn; " + "never call SELECT_CATEGORY_TOPK before it is callable. " + "If semantic evidence is absent, first call INTERPRET_FIELD_NAME " + "with field_name=$metadata.field_name or another callable semantic Skill. " + "When SELECT_CATEGORY_TOPK becomes callable, provide semantic_query=" + "$state.field_semantics if present, otherwise a different existing " + "$state.; never omit required arguments. " + "Then call INFER_CATEGORY_FROM_TOPK with candidate_standard_categories=" + "$state.standard_candidates and all_available_evidence=" + "$state.available_evidence; bare $state is invalid. " + "Only grade after selecting one category." + ) + if self.metadata_guidance: + messages[0]["content"] += ( + " If GET_FIELD_METADATA is callable and field_metadata is absent, " + "call it before category selection." + ) + if "table_fields" not in state.named_state: + if self.description_skills: + messages[0]["content"] += ( + " Once real field_metadata is available, if " + "INTERPRET_FIELD_DESCRIPTION is callable and " + "field_description_semantics is absent, call " + "INTERPRET_FIELD_DESCRIPTION before SELECT_CATEGORY_TOPK " + "with field_description=$state.field_description. " + "If no field description exists but " + "INTERPRET_TABLE_DESCRIPTION is callable, use its real " + "table_description instead. After interpretation, pass " + "semantic_query=$state.available_evidence to " + "SELECT_CATEGORY_TOPK and pass " + "all_available_evidence=$state.available_evidence to Top-1. " + "If neither description is available, use " + "semantic_query=$state.field_metadata." + ) + else: + messages[0]["content"] += ( + " Once field_metadata is available, pass semantic_query=" + "$state.field_metadata to SELECT_CATEGORY_TOPK and pass " + "$state.field_metadata to INFER_CATEGORY_FROM_TOPK." + ) + if "GET_TABLE_FIELDS" in callable_skills and "table_fields" not in state.named_state: + messages[0]["content"] += ( + " If the current field and table metadata leave the business " + "category ambiguous, you may call GET_TABLE_FIELDS with " + "current_record=$record before category selection; this is optional." + ) + if "table_fields" in state.named_state: + messages[0]["content"] += ( + " Neighboring table_fields are available. To use both target " + "metadata and neighbors in category selection, pass semantic_query=" + "$state.available_evidence to SELECT_CATEGORY_TOPK and " + "all_available_evidence=$state.available_evidence to " + "INFER_CATEGORY_FROM_TOPK." + ) + if self.metadata_guidance: + messages[0]["content"] += ( + " Finish grading and finalization without repeating diagnostic Skills." + ) + if self.description_skills and "field_metadata" in state.named_state: + messages[0]["content"] += ( + " Real descriptions in field_metadata may be interpreted through " + "INTERPRET_FIELD_DESCRIPTION or INTERPRET_TABLE_DESCRIPTION only " + "when these Skills appear in callable_skills. They are optional: " + "do not call a missing-description Skill or repeat one. " + "If description semantics were produced, use " + "semantic_query=$state.available_evidence for SELECT_CATEGORY_TOPK " + "and all_available_evidence=$state.available_evidence for Top-1." + ) + elif self.metadata_guidance: + messages[0]["content"] += ( + "\nIf GET_FIELD_METADATA is callable and field_metadata is absent, " + "call GET_FIELD_METADATA before semantic interpretation or retrieval. " + "When real field_metadata has been obtained and standard_candidates " + "are absent, call SEARCH_STANDARD with " + "{\"semantic_query\":\"$state.field_metadata\"}; do not search only " + "the guessed field_semantics. Once candidates are available, " + "proceed toward category, level, and finalization without " + "repeating diagnostic Skills." + ) + if self.few_shot_messages: + messages[1:1] = [dict(message) for message in self.few_shot_messages] + try: + raw = self.generate(messages) + except Exception as exc: # pragma: no cover - backend-specific + error = ActionError(f"policy generator failed: {exc}") + error.metadata = {"messages": messages, "raw_response": None, + "error": str(exc), "generation": _generation_metadata(self.generate)} + raise error from exc + if not isinstance(raw, str): + raise ActionError("policy generator must return a string") + try: + parsed = parse_policy_action(raw) + except ActionError as exc: + exc.metadata = {"messages": messages, "raw_response": raw, + "generation": _generation_metadata(self.generate)} + raise + return PolicyDecision( + call=parsed, + raw_response=raw, + messages=tuple(messages), + metadata=_generation_metadata(self.generate), + ) + + +class ModelSkillExecutor: + """Execute deterministic tools or one selected Markdown Skill via a generator.""" + + def __init__( + self, + generate: TextGenerator, + loader: SkillLoader, + *, + deterministic_handlers: Mapping[str, DeterministicHandler] | None = None, + category_catalog: CategoryCatalog | None = None, + top_k: int = 5, + guard_two_stage_evidence: bool = False, + topk_evidence_mode: str = "augmented", + topk_selection_mode: str = "single", + topk_selection_seed: int = 20260930, + top1_evidence_mode: str = "all", + top1_selection_mode: str = "model", + evidence_assembly=None, + include_comparison_explanation: bool = True, + ) -> None: + if type(include_comparison_explanation) is not bool: + raise ValueError('include_comparison_explanation must be bool') + self.include_comparison_explanation = include_comparison_explanation + if top1_evidence_mode not in {"all", "raw"}: + raise ValueError("top1_evidence_mode must be all or raw") + if top1_selection_mode not in {"model", "first"}: + raise ValueError("top1_selection_mode must be model or first") + self.top1_evidence_mode = top1_evidence_mode + self.top1_selection_mode = top1_selection_mode + if topk_evidence_mode not in {"augmented", "raw", "raw-flat"}: + raise ValueError("topk_evidence_mode must be augmented, raw or raw-flat") + if topk_selection_mode not in {"single", "balanced-vote"}: + raise ValueError("topk_selection_mode must be single or balanced-vote") + if topk_selection_mode == "balanced-vote" and category_catalog is None: + raise ValueError("balanced-vote requires a two-stage category catalog") + self.generate = generate + self.loader = loader + self.category_catalog = category_catalog + self.top_k = top_k + self.guard_two_stage_evidence = guard_two_stage_evidence + self.evidence_assembly = evidence_assembly + self.topk_evidence_mode = topk_evidence_mode + self.topk_selection_mode = topk_selection_mode + self.topk_selection_seed = topk_selection_seed + self.deterministic_handlers = { + "CHECK_METADATA_AVAILABILITY": metadata_availability_handler, + **dict(deterministic_handlers or {}), + } + + @staticmethod + def _resolved_arguments( + call: ToolCall, + record: MetadataRecord, + state: Mapping[str, Any], + ) -> dict[str, Any]: + resolved: dict[str, Any] = {} + for name, value in call.arguments.items(): + found, result = resolve_reference(value, record, state) + if not found: + raise SkillExecutionError(f"argument {name!r} cannot be resolved") + resolved[name] = result + return resolved + + @staticmethod + def _messages(contract: SkillContract, resolved: Mapping[str, Any]) -> list[dict[str, str]]: + system = ( + f"Execute only the selected Skill {contract.name}.\n" + f"DSL: {contract.dsl}\n" + f"Contract:\n{contract.body}\n" + "Use only the resolved inputs. Return exactly one JSON object with summary and data." + ) + payload = { + "skill": contract.name, + "resolved_inputs": dict(resolved), + "output_schema": { + "summary": "non-empty natural-language observation", + "data": {name: "required output" for name in contract.produces}, + }, + } + return [ + {"role": "system", "content": system}, + {"role": "user", "content": json.dumps(payload, ensure_ascii=False, indent=2)}, + ] + + @staticmethod + def _validate_deterministic(observation: Observation, spec: SkillSpec) -> Observation: + if observation.skill != spec.name: + raise SkillExecutionError( + f"deterministic handler returned Skill {observation.skill!r}, expected {spec.name!r}" + ) + if observation.status != "success": + return observation + actual = set(observation.data) + expected = set(spec.produces) + if actual != expected: + raise SkillExecutionError( + f"deterministic handler output keys {sorted(actual)} do not match {sorted(expected)}" + ) + return observation + + def execute( + self, + call: ToolCall, + record: MetadataRecord, + state: Mapping[str, Any], + contract: SkillContract, + ) -> Observation: + loaded = self.loader.load(call.tool) + if loaded != contract: + raise SkillExecutionError(f"contract mismatch for {call.tool}") + spec = self.loader.catalog[call.tool] + resolved = self._resolved_arguments(call, record, state) + if call.tool in {"SELECT_CATEGORY_TOPK", "INFER_CATEGORY_FROM_TOPK"}: + try: + return self._execute_two_stage(call, record, resolved, state, contract) + except SkillExecutionError as exc: + if call.tool == "INFER_CATEGORY_FROM_TOPK": + exc.metadata.update(top1_selection_mode=self.top1_selection_mode, + top1_evidence_mode=self.top1_evidence_mode) + raise + if call.tool == "FINALIZE_CLASSIFICATION" and self.guard_two_stage_evidence: + return self._finalize_guarded_two_stage(call, resolved, state) + handler = self.deterministic_handlers.get(call.tool) + if handler is not None: + return self._validate_deterministic(handler(call, record, resolved), spec) + + messages = self._messages(contract, resolved) + try: + raw = self.generate(messages) + except Exception as exc: # pragma: no cover - backend-specific + raise SkillExecutionError(f"Skill generator failed: {exc}", metadata={ + "messages": messages, "raw_response": None, "error": str(exc), + "generation": _generation_metadata(self.generate)}) from exc + if not isinstance(raw, str): + raise SkillExecutionError("Skill generator must return a string") + try: + observation = parse_skill_observation(raw, spec) + except SkillExecutionError as exc: + exc.metadata = {"messages": messages, "raw_response": raw, + "generation": _generation_metadata(self.generate)} + raise + return replace( + observation, + metadata={ + "messages": messages, + "raw_response": raw, + **_generation_metadata(self.generate), + }, + ) + + def _finalize_guarded_two_stage( + self, + call: ToolCall, + resolved: Mapping[str, Any], + state: Mapping[str, Any], + ) -> Observation: + if self.category_catalog is None: + raise SkillExecutionError("two-stage category catalog is not configured") + evidence = resolved.get("final_evidence_state") + if not isinstance(evidence, Mapping) or evidence != state.get("final_evidence_state"): + raise SkillExecutionError("finalizer must consume the frozen final evidence state") + category = evidence.get("category") + level = evidence.get("grading_level") + aligned = evidence.get("aligned_standard_entry") + candidates = evidence.get("standard_candidates") + if not isinstance(category, str) or not isinstance(level, str): + raise SkillExecutionError("final evidence is missing category or grading level") + if category != state.get("category") or level != state.get("grading_level"): + raise SkillExecutionError("final evidence differs from the selected category or level") + if ( + not isinstance(aligned, Mapping) + or (aligned.get("category") or aligned.get("category_id")) != category + or aligned.get("grading_level") != level + ): + raise SkillExecutionError("final evidence disagrees with the aligned entry") + if not isinstance(candidates, list) or not any( + isinstance(item, Mapping) + and (item.get("category") or item.get("category_id")) == category + for item in candidates + ): + raise SkillExecutionError("selected category is absent from frozen Top-K") + try: + registry_entry = self.category_catalog.frozen_entry(category) + except ValueError as exc: + raise SkillExecutionError(str(exc)) from exc + if registry_entry.get("grading_level") != level: + raise SkillExecutionError("selected level disagrees with frozen Registry") + return Observation( + call.tool, + "success", + "Bound final result to the selected category and frozen grading level", + {"final_result": {"category": category, "grading_level": level}}, + metadata={"controller_bound_terminal": True}, + ) + + def _execute_two_stage( + self, + call: ToolCall, + record: MetadataRecord, + resolved: Mapping[str, Any], + state: Mapping[str, Any], + contract: SkillContract, + ) -> Observation: + if self.category_catalog is None: + raise SkillExecutionError("two-stage category catalog is not configured") + evidence_assembled = ( + self.guard_two_stage_evidence + and call.tool == "SELECT_CATEGORY_TOPK" + and isinstance(state.get("field_metadata"), Mapping) + ) + if call.tool == "SELECT_CATEGORY_TOPK": + if evidence_assembled and self.evidence_assembly is None: + semantic_query = _two_stage_evidence(record, state) + if self.topk_evidence_mode == "raw": + semantic_query["observations"] = { + "field_metadata": state["field_metadata"], + } + elif self.topk_evidence_mode == "raw-flat": + semantic_query = { + **state["field_metadata"], + "domain": record.domain, "field_name": record.field_name, + } + else: + semantic_query = resolved["semantic_query"] + prompt_input = { + "record_input": {"domain": record.domain, "field_name": record.field_name}, + "semantic_query": semantic_query, + "top_k": self.top_k, + "category_options": self.category_catalog.selection_options(), + } + if self.evidence_assembly is not None: + # Opt-in pool callers preserve the selected native query and share the + # same validated publication/final evidence boundary. + prompt_input["available_evidence"] = self.evidence_assembly(record, state) + output_rule = ( + 'Return exactly one JSON object: {"option_ids":["C001",...],' + f'"summary":"reason"}}. The list MUST contain exactly {self.top_k} distinct ' + f'option IDs from category_options, ranked best first. If fewer than {self.top_k} ' + 'are strong matches, include weaker but plausible supplied options to fill every slot. ' + 'Copy the short option_id exactly. Do not output long category paths, canonical ' + 'category IDs, duplicates, invented option IDs, or a different top-N. ' + 'Limit summary to one short phrase (at most 20 characters); do not put ' + 'quotation marks, line breaks, or a long explanation inside summary. ' + 'Close the summary string and the JSON object.' + ) + else: + prior = state.get("standard_candidates") + supplied = resolved.get("candidate_standard_categories") + if not isinstance(prior, list) or supplied != prior: + raise SkillExecutionError("Top-1 must consume the actual prior Top-K observation") + if len(prior) != self.top_k or any(not isinstance(item, Mapping) for item in prior): + raise SkillExecutionError("Top-1 requires exactly top_k candidate objects") + ids = [item.get("category") for item in prior if isinstance(item, Mapping)] + try: + self.category_catalog.validate_top_k(ids, self.top_k) + details = self.category_catalog.option_details(ids) + except ValueError as exc: + raise SkillExecutionError(str(exc)) from exc + if self.top1_selection_mode == "first": + category = ids[0] + return Observation(call.tool, "success", "Selected actual first frozen candidate", + {"category": category, "aligned_standard_entry": self.category_catalog.frozen_entry(category)}, + metadata={"top1_selection_mode": "first", "top1_evidence_mode": self.top1_evidence_mode, + "controller_bound_selection": True, "model_call_count": 0, "attempts": []}) + evidence = resolved["all_available_evidence"] + if self.top1_evidence_mode == "raw": + evidence = {"domain": record.domain, "field_name": record.field_name, + "observations": {"field_metadata": dict(state["field_metadata"])} + if isinstance(state.get("field_metadata"), Mapping) else {}} + prompt_input = { + "all_available_evidence": evidence, + "candidate_details": details, + } + if self.evidence_assembly is not None: + prompt_input["available_evidence"] = self.evidence_assembly(record, state) + if not self.include_comparison_explanation: + from evidence_roles import project_top1_evidence + for key in ('all_available_evidence', 'available_evidence'): + if key in prompt_input: + prompt_input[key] = project_top1_evidence(prompt_input[key], state.get('standard_comparison')) + output_rule = ( + 'Return exactly one JSON object: {"option_id":"one supplied short option ID",' + '"summary":"reason"}. Do not choose outside candidate_details. ' + 'Limit summary to one short phrase (at most 20 characters); do not put ' + 'quotation marks, line breaks, or a long explanation inside summary. ' + 'Close the summary string and the JSON object.' + ) + messages = [ + {"role": "system", "content": f"Execute only the selected Skill {contract.name}.\n{contract.body}\n{output_rule}"}, + {"role": "user", "content": json.dumps(prompt_input, ensure_ascii=False, indent=2)}, + ] + if call.tool == "SELECT_CATEGORY_TOPK" and self.topk_selection_mode == "balanced-vote": + return self._execute_balanced_topk(record, prompt_input, messages[0], evidence_assembled) + attempts: list[dict[str, Any]] = [] + max_attempts = 2 + for attempt_index in range(max_attempts): + try: + raw = self.generate(messages) + except Exception as exc: + attempts.append({"messages": messages, "raw_response": None, "error": str(exc), + "generation": _generation_metadata(self.generate)}) + raise SkillExecutionError( + f"two-stage Skill generator failed: {exc}", + metadata=_attempt_metadata(attempts), + ) from exc + generation = _generation_metadata(self.generate) + try: + payload = _decode_observation_object(raw) + summary = payload.get("summary") + if not isinstance(summary, str) or not summary.strip(): + raise SkillExecutionError("two-stage Skill requires a nonempty summary") + if call.tool == "SELECT_CATEGORY_TOPK": + if set(payload) != {"option_ids", "summary"}: + raise SkillExecutionError("Top-K response must contain option_ids and summary only") + canonical_ids = self.category_catalog.resolve_option_ids( + payload["option_ids"], self.top_k + ) + label_map = {item["category"]: item for item in self.category_catalog.labels()} + data = {"standard_candidates": [label_map[item] for item in canonical_ids]} + else: + if set(payload) != {"option_id", "summary"}: + raise SkillExecutionError("Top-1 response must contain option_id and summary only") + category = self.category_catalog.resolve_option_ids([payload["option_id"]], 1)[0] + if category not in ids: + raise SkillExecutionError("Top-1 category is outside prior Top-K") + data = {"category": category, "aligned_standard_entry": self.category_catalog.frozen_entry(category)} + except (SkillExecutionError, ValueError, TypeError, KeyError) as exc: + error = str(exc) + attempts.append({"messages": messages, "raw_response": raw, + "error": error, "generation": generation}) + if attempt_index + 1 < max_attempts: + if call.tool == "SELECT_CATEGORY_TOPK": + feedback = ( + f"Your Top-K response failed validation: {error}. " + f"Return a complete new JSON object with exactly {self.top_k} " + "distinct option IDs copied from category_options. " + "Use option_ids and summary only; never use canonical paths or IDs. " + "Keep summary to one short phrase and close the JSON object." + ) + else: + feedback = ( + f"Your Top-1 response failed validation: {error}. " + "Return one complete JSON object with option_id copied from " + "candidate_details and a brief summary. Close the object with " + "a closing brace }. Keep summary to one short phrase. " + "Do not use any option outside candidate_details." + ) + messages = [ + *messages, + {"role": "assistant", "content": raw if isinstance(raw, str) else ""}, + {"role": "user", "content": feedback}, + ] + continue + raise SkillExecutionError( + f"invalid two-stage Skill response: {error}", + metadata=_attempt_metadata(attempts), + ) from exc + attempts.append({"messages": messages, "raw_response": raw, + "generation": generation}) + metadata = _attempt_metadata(attempts) + if call.tool == "INFER_CATEGORY_FROM_TOPK": + metadata.update({"top1_evidence_mode": self.top1_evidence_mode, + "top1_selection_mode": self.top1_selection_mode}) + if evidence_assembled: + metadata["controller_evidence_assembled"] = True + metadata["topk_evidence_mode"] = self.topk_evidence_mode + return Observation(call.tool, "success", summary.strip(), data, + metadata=metadata) + raise AssertionError("unreachable") + + def _execute_balanced_topk( + self, record: MetadataRecord, prompt_input: Mapping[str, Any], + system_message: Mapping[str, str], evidence_assembled: bool, + ) -> Observation: + """Make exactly three independent calls, retaining failures without retry.""" + assert self.category_catalog is not None + attempts: list[dict[str, Any]] = [] + rankings: list[list[str]] = [] + options = prompt_input["category_options"] + allowed = [item["option_id"] for item in options] + for index, presented in enumerate(balanced_options( + options, record.domain, record.field_name, self.topk_selection_seed + ), start=1): + messages = [dict(system_message), { + "role": "user", "content": json.dumps( + {**prompt_input, "category_options": presented}, ensure_ascii=False, indent=2 + ), + }] + attempt: dict[str, Any] = { + "round": index, "presented_option_ids": [item["option_id"] for item in presented], + "valid": False, "error": None, "raw_response": None, + "messages": messages, "generation": {}, + } + # Most generators keep a last-call attribute. Clear it before this call + # so a failing backend cannot be charged for the previous completion. + metadata_cleared = False + try: + if hasattr(self.generate, "last_generation_metadata"): + self.generate.last_generation_metadata = {} + metadata_cleared = True + except (AttributeError, TypeError): + pass + try: + raw = self.generate(messages) + except Exception as exc: + attempt["error"] = f"generator failed: {exc}" + if metadata_cleared: + attempt["generation"] = _generation_metadata(self.generate) + attempts.append(attempt) + continue + attempt["raw_response"] = raw + attempt["generation"] = _generation_metadata(self.generate) + try: + payload = _decode_observation_object(raw) + if set(payload) != {"option_ids", "summary"}: + raise SkillExecutionError("Top-K response must contain option_ids and summary only") + if not isinstance(payload["summary"], str) or not payload["summary"].strip(): + raise SkillExecutionError("two-stage Skill requires a nonempty summary") + self.category_catalog.resolve_option_ids(payload["option_ids"], self.top_k) + ranking = list(payload["option_ids"]) + rankings.append(ranking) + attempt.update({"valid": True, "option_ids": ranking, "votes": [ + {"option_id": item, "vote": 1, "reciprocal_rank_score": f"1/{rank}"} + for rank, item in enumerate(ranking, start=1) + ]}) + except (SkillExecutionError, ValueError, TypeError, KeyError) as exc: + attempt["error"] = str(exc) + attempts.append(attempt) + metadata = _attempt_metadata(attempts) + metadata["format_retry_count"] = 0 + selection = { + "mode": "balanced-vote", "policy": SELECTION_POLICY, + "seed": self.topk_selection_seed, "rounds": 3, "quorum": 2, + "valid_rounds": len(rankings), "selected_option_ids": [], "scores": [], + } + metadata["topk_selection"] = selection + if evidence_assembled: + metadata.update({"controller_evidence_assembled": True, "topk_evidence_mode": self.topk_evidence_mode}) + if len(rankings) < 2: + raise SkillExecutionError("balanced-vote requires at least two valid rankings", metadata=metadata) + selected = aggregate_rankings(rankings, allowed, record.domain, record.field_name, + self.topk_selection_seed, self.top_k) + selection["selected_option_ids"] = selected + selection["scores"] = ranking_scores(rankings, allowed, record.domain, record.field_name, + self.topk_selection_seed, self.top_k) + canonical = self.category_catalog.resolve_option_ids(selected, self.top_k) + labels = {item["category"]: item for item in self.category_catalog.labels()} + return Observation("SELECT_CATEGORY_TOPK", "success", "Balanced Top-K vote", + {"standard_candidates": [labels[item] for item in canonical]}, metadata=metadata) + + +__all__ = [ + "DeterministicHandler", + "ModelLoopPolicy", + "ModelSkillExecutor", + "SkillExecutionError", + "TextGenerator", + "make_standard_search_handler", + "make_field_metadata_handler", + "make_table_fields_handler", + "metadata_availability_handler", + "load_few_shot_messages", + "parse_skill_observation", +] diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/planner.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/planner.py new file mode 100644 index 0000000..9987d1d --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/planner.py @@ -0,0 +1,405 @@ +"""Prompt construction and model-independent Program planning adapters.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from json import JSONDecodeError +from pathlib import Path +from time import perf_counter +from typing import Any, Callable, Mapping, Sequence + +from .catalog import SkillSpec, default_catalog +from .contracts import MetadataRecord, Program +from .validator import DEFAULT_MAX_STEPS, ValidationIssue, validate_program + + +PLANNER_SYSTEM_PROMPT = """你是数据库字段分类分级任务的 DSL Program Planner。 +你的职责是根据当前字段元数据、已有状态和可用 Skill Catalog，生成一个可执行的 Program。 +只允许使用 Catalog 中出现的 Skill；只能引用当前真实存在的元数据或前一步已经产生的状态。 +引用语法必须严格遵守：`$record` 表示完整记录，`$domain` 表示域名，`$metadata.` 表示元数据字段，`$state.` 表示前一步产生的状态。 +禁止使用 `$metadata`、`$record.metadata.`、`$record.domain` 或其他未列出的路径；这些写法即使看起来等价，也不是合法 DSL 引用。 +不要补造缺失的表名、描述或样例值。Program 必须以 FINALIZE_CLASSIFICATION 结束。 +对于前一步 Skill 产生的结果，后续参数必须使用对应的 `$state.` 引用，不能把空数组、空对象或模型臆造的结果直接填入必需参数。 +Program 最多 10 步；如果已有信息足够，可以采用下面这个合法的最小骨架： +{"program":[ + {"tool":"INTERPRET_FIELD_NAME","arguments":{"field_name":"$metadata.field_name"}}, + {"tool":"SEARCH_STANDARD","arguments":{"semantic_query":"$state.field_semantics"}}, + {"tool":"INFER_CATEGORY","arguments":{"candidate_standard_categories":"$state.standard_candidates","all_available_evidence":"$state.field_semantics"}}, + {"tool":"INFER_GRADING_LEVEL","arguments":{"aligned_category":"$state.category","aligned_standard_entry":"$state.aligned_standard_entry"}}, + {"tool":"FINALIZE_CLASSIFICATION","arguments":{"final_evidence_state":"$state.final_evidence_state"}} +]} +只输出一个完整 JSON 对象，不要输出 Markdown、解释或额外文字，不能省略括号。""" + + +TWO_STAGE_PLANNER_SYSTEM_PROMPT = """你是数据库字段分类分级任务的 DSL Program Planner。 +每个 Program 步骤只包含一次 Tool Call；只能调用本轮 Skill Catalog 中的工具。 +仅使用可见的 domain 与 field_name，缺少其他元数据时通过可用的元数据 Skill 查询，不能编造。 +引用语法只允许 `$record`、`$domain`、`$metadata.`、`$state.`。 +先调用 SELECT_CATEGORY_TOPK 从完整类别路径目录选候选 ID，再调用 INFER_CATEGORY_FROM_TOPK 从这些候选中选一个类别。 +第二步的 candidate_standard_categories 必须引用 `$state.standard_candidates`，不能自己填写候选。 +只有 Top-1 类别确定后才调用 INFER_GRADING_LEVEL，最后必须调用 FINALIZE_CLASSIFICATION。 +合法的最小骨架： +{"program":[ + {"tool":"INTERPRET_FIELD_NAME","arguments":{"field_name":"$metadata.field_name"}}, + {"tool":"SELECT_CATEGORY_TOPK","arguments":{"semantic_query":"$state.field_semantics"}}, + {"tool":"INFER_CATEGORY_FROM_TOPK","arguments":{"candidate_standard_categories":"$state.standard_candidates","all_available_evidence":"$state.field_semantics"}}, + {"tool":"INFER_GRADING_LEVEL","arguments":{"aligned_category":"$state.category","aligned_standard_entry":"$state.aligned_standard_entry"}}, + {"tool":"FINALIZE_CLASSIFICATION","arguments":{"final_evidence_state":"$state.grading_level"}} +]} +只输出一个完整 JSON 对象，不输出解释或 Markdown。""" + + +class PlannerError(ValueError): + """Raised when generation, JSON parsing, or Program validation fails.""" + + def __init__( + self, + message: str, + *, + raw_response: str | None = None, + issues: Sequence[ValidationIssue] = (), + ) -> None: + super().__init__(message) + self.raw_response = raw_response + self.issues = tuple(issues) + + +@dataclass(frozen=True) +class PlannerResult: + """A validated Program together with the exact prompt and model response.""" + + program: Program + messages: tuple[dict[str, str], ...] + raw_response: str + + +def _catalog_payload(catalog: Mapping[str, SkillSpec]) -> list[dict[str, Any]]: + return [ + { + "name": spec.name, + "role": spec.role, + "required_inputs": list(spec.required_inputs), + "optional_inputs": list(spec.optional_inputs), + "produces": list(spec.produces), + "max_calls": spec.max_calls, + "terminal": spec.terminal, + } + for spec in catalog.values() + ] + + +def build_planner_messages( + record: MetadataRecord, + catalog: Mapping[str, SkillSpec] | None = None, + *, + state: Mapping[str, Any] | None = None, + max_steps: int = DEFAULT_MAX_STEPS, + two_stage: bool = False, +) -> list[dict[str, str]]: + """Build deterministic chat messages for a model that generates a Program.""" + + active_catalog = catalog or default_catalog() + available_metadata = [ + name + for name in ("domain", *record.metadata.keys()) + if record.available(name) + ] + payload = { + "task": "为当前数据库字段选择必要的元数据、语义、标准对齐和证据推理 Skill，并生成完整 Program。", + "record": ( + {"domain": record.domain, "metadata": dict(record.metadata)} + if two_stage else record.to_dict() + ), + "available_metadata": available_metadata, + "state": dict(state or {}), + "skill_catalog": _catalog_payload(active_catalog), + "constraints": { + "max_steps": max_steps, + "final_skill": "FINALIZE_CLASSIFICATION", + "output_schema": {"program": [{"tool": "SKILL_NAME", "arguments": {}}]}, + }, + } + return [ + {"role": "system", "content": ( + TWO_STAGE_PLANNER_SYSTEM_PROMPT if two_stage else PLANNER_SYSTEM_PROMPT + )}, + { + "role": "user", + "content": json.dumps(payload, ensure_ascii=False, indent=2), + }, + ] + + +def _decode_single_json_object(text: str) -> Mapping[str, Any]: + cleaned = text.strip() + if cleaned.startswith("```"): + lines = cleaned.splitlines() + if lines and lines[0].lstrip().startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + cleaned = "\n".join(lines).strip() + + start = cleaned.find("{") + if start < 0: + raise PlannerError("model response does not contain a JSON object", raw_response=text) + + decoder = json.JSONDecoder() + try: + value, end = decoder.raw_decode(cleaned[start:]) + except JSONDecodeError as exc: + raise PlannerError(f"invalid JSON response: {exc.msg}", raw_response=text) from exc + + trailing = cleaned[start + end :].strip() + if trailing: + if trailing.startswith("{"): + message = "model response contains multiple JSON objects" + else: + message = "model response contains trailing non-JSON content" + raise PlannerError(message, raw_response=text) + if not isinstance(value, Mapping): + raise PlannerError("model response JSON must be an object", raw_response=text) + return value + + +def parse_program_response(text: str) -> Program: + """Parse one model response into a Program without silently repairing it.""" + + if not isinstance(text, str) or not text.strip(): + raise PlannerError("model response must be a non-empty string", raw_response=text if isinstance(text, str) else None) + payload = _decode_single_json_object(text) + try: + return Program.from_mapping(payload) + except (TypeError, ValueError) as exc: + raise PlannerError(f"invalid Program schema: {exc}", raw_response=text) from exc + + +class PromptPlanner: + """Adapt any text generator to the validated DSL Program contract.""" + + def __init__( + self, + generate: Callable[[list[dict[str, str]]], str], + *, + catalog: Mapping[str, SkillSpec] | None = None, + max_steps: int = DEFAULT_MAX_STEPS, + max_retries: int = 1, + two_stage: bool = False, + ) -> None: + if max_retries < 0: + raise ValueError("max_retries must be non-negative") + self.generate = generate + self.catalog = dict(catalog or default_catalog()) + self.max_steps = max_steps + self.max_retries = max_retries + self.two_stage = two_stage + + @staticmethod + def _validation_feedback(issues: Sequence[ValidationIssue]) -> str: + rendered = "\n".join(f"- {issue}" for issue in issues) + return ( + "上一版 Program 未通过 DSL 校验，请修正后重新输出完整 JSON 对象。\n" + "校验错误：\n" + f"{rendered}\n" + "只允许使用 `$record`、`$domain`、`$metadata.` 和 `$state.`；" + "禁止使用 `$metadata` 或 `$record.metadata.`。" + "不要解释，不要输出 Markdown，只输出修正后的 JSON。" + ) + + def plan( + self, + record: MetadataRecord, + *, + state: Mapping[str, Any] | None = None, + ) -> PlannerResult: + messages = build_planner_messages( + record, + self.catalog, + state=state, + max_steps=self.max_steps, + two_stage=self.two_stage, + ) + attempt_messages = list(messages) + for attempt in range(self.max_retries + 1): + try: + raw_response = self.generate(attempt_messages) + except Exception as exc: # pragma: no cover - depends on external generator + raise PlannerError(f"planner generator failed: {exc}") from exc + if not isinstance(raw_response, str): + raise PlannerError("planner generator must return a string", raw_response=None) + + program = parse_program_response(raw_response) + issues = validate_program( + record, + program, + self.catalog, + initial_state=state, + max_steps=self.max_steps, + ) + if not issues: + return PlannerResult( + program=program, + messages=tuple(attempt_messages), + raw_response=raw_response, + ) + + if attempt == self.max_retries: + rendered = "; ".join(str(issue) for issue in issues) + raise PlannerError( + f"generated Program failed validation: {rendered}", + raw_response=raw_response, + issues=issues, + ) + + attempt_messages.extend( + [ + {"role": "assistant", "content": raw_response}, + {"role": "user", "content": self._validation_feedback(issues)}, + ] + ) + + raise AssertionError("unreachable") + + +class TransformersTextGenerator: + """Optional local Transformers backend; imports model dependencies lazily.""" + + def __init__( + self, + model_path: str | Path, + *, + max_new_tokens: int = 512, + local_files_only: bool = True, + device_map: str = "auto", + torch_dtype: str = "auto", + trust_remote_code: bool = False, + enable_thinking: bool = False, + ) -> None: + self.model_path = str(Path(model_path).expanduser()) + self.max_new_tokens = max_new_tokens + self.local_files_only = local_files_only + self.device_map = device_map + self.torch_dtype = torch_dtype + self.trust_remote_code = trust_remote_code + self.enable_thinking = enable_thinking + self._tokenizer = None + self._model = None + self._torch = None + self.last_generation_metadata: dict[str, Any] = {} + + def _load(self) -> None: + if self._model is not None: + return + try: + import torch + from transformers import ( + AutoConfig, + AutoModelForCausalLM, + AutoModelForImageTextToText, + AutoTokenizer, + ) + except ImportError as exc: # pragma: no cover - optional dependency + raise PlannerError( + "TransformersTextGenerator requires torch and transformers; " + "install them in the remote environment first" + ) from exc + self._torch = torch + + dtype: Any = self.torch_dtype + if dtype != "auto": + dtype = getattr(torch, dtype) + self._tokenizer = AutoTokenizer.from_pretrained( + self.model_path, + local_files_only=self.local_files_only, + trust_remote_code=self.trust_remote_code, + ) + model_config = AutoConfig.from_pretrained( + self.model_path, + local_files_only=self.local_files_only, + trust_remote_code=self.trust_remote_code, + ) + auto_model = ( + AutoModelForImageTextToText + if select_transformers_auto_model(model_config.to_dict()) == "image-text-to-text" + else AutoModelForCausalLM + ) + self._model = auto_model.from_pretrained( + self.model_path, + local_files_only=self.local_files_only, + device_map=self.device_map, + dtype=dtype, + trust_remote_code=self.trust_remote_code, + ) + self._model.eval() + + def __call__(self, messages: list[dict[str, str]]) -> str: + self._load() + assert self._tokenizer is not None + assert self._model is not None + tokenizer = self._tokenizer + model = self._model + if not hasattr(tokenizer, "apply_chat_template"): + raise PlannerError("tokenizer does not provide apply_chat_template") + try: + prompt = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + enable_thinking=self.enable_thinking, + ) + except TypeError: + # Generic chat templates may not expose Qwen's thinking switch. + prompt = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + ) + inputs = tokenizer(prompt, return_tensors="pt") + device = next(model.parameters()).device + inputs = {name: value.to(device) for name, value in inputs.items()} + prompt_length = inputs["input_ids"].shape[-1] + assert self._torch is not None + started = perf_counter() + with self._torch.no_grad(): + generated = model.generate( + **inputs, + max_new_tokens=self.max_new_tokens, + do_sample=False, + ) + new_tokens = generated[0][prompt_length:] + completion_tokens = int(new_tokens.shape[-1]) + self.last_generation_metadata = { + "token_usage": { + "prompt_tokens": int(prompt_length), + "completion_tokens": completion_tokens, + "total_tokens": int(prompt_length) + completion_tokens, + }, + "elapsed_seconds": perf_counter() - started, + } + return tokenizer.decode(new_tokens, skip_special_tokens=True).strip() + + +def select_transformers_auto_model(config: Mapping[str, Any]) -> str: + """Choose the compatible Transformers auto class from lightweight config data.""" + + architectures = config.get("architectures") + names = architectures if isinstance(architectures, list) else [] + model_type = str(config.get("model_type") or "") + if model_type == "qwen3_5" or any( + str(name).endswith("ForConditionalGeneration") for name in names + ): + return "image-text-to-text" + return "causal-lm" + + +__all__ = [ + "PLANNER_SYSTEM_PROMPT", + "PlannerError", + "PlannerResult", + "PromptPlanner", + "TransformersTextGenerator", + "build_planner_messages", + "parse_program_response", + "select_transformers_auto_model", +] diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/retrieval.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/retrieval.py new file mode 100644 index 0000000..df3e678 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/retrieval.py @@ -0,0 +1,335 @@ +"""Auditable Top-K retrieval backends for frozen standard registries.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import json +from json import JSONDecodeError +import math +from pathlib import Path +from time import perf_counter +from typing import Any, Callable, Mapping, Sequence + +from .runtime import KnowledgeBase + + +class RetrievalError(ValueError): + """Raised when a retriever cannot return one valid frozen-registry ranking.""" + + +@dataclass(frozen=True) +class RetrievalResult: + candidates: tuple[dict[str, Any], ...] + metadata: Mapping[str, Any] = field(default_factory=dict) + + +_QUERY_FIELD_ORDER = ( + "domain", + "field_name", + "field_abbreviation_semantics", + "field_semantics", +) + + +def format_retrieval_query(query: Any) -> str: + """Render allowed retrieval evidence deterministically for text encoders.""" + + if isinstance(query, Mapping): + parts = [ + f"{name}: {query[name]}" + for name in _QUERY_FIELD_ORDER + if name in query and query[name] not in (None, "") + ] + return "\n".join(parts) + return str(query) + + +def _canonical_registry(registry: Mapping[str, Mapping[str, Any]]) -> dict[str, dict[str, Any]]: + canonical: dict[str, dict[str, Any]] = {} + for key, raw in registry.items(): + category = str(raw.get("category") or raw.get("category_id") or key) + if category in canonical: + raise RetrievalError(f"duplicate Registry category: {category}") + entry = dict(raw) + entry["category"] = category + canonical[category] = entry + return canonical + + +def validate_ranked_candidates( + candidates: Sequence[Mapping[str, Any]], + registry: Mapping[str, Mapping[str, Any]], + *, + top_k: int, +) -> tuple[dict[str, Any], ...]: + """Return authoritative Registry entries for one exact, distinct ranking.""" + + if top_k < 1: + raise RetrievalError("top_k must be positive") + if len(candidates) != top_k: + raise RetrievalError(f"ranking must contain exactly {top_k} candidates") + frozen = _canonical_registry(registry) + ranked: list[dict[str, Any]] = [] + seen: set[str] = set() + for index, candidate in enumerate(candidates): + category = candidate.get("category") or candidate.get("category_id") + if not isinstance(category, str) or not category: + raise RetrievalError(f"candidate {index} must contain a non-empty category") + if category in seen: + raise RetrievalError(f"duplicate category in ranking: {category}") + if category not in frozen: + raise RetrievalError(f"unknown Registry category: {category}") + seen.add(category) + ranked.append(dict(frozen[category])) + return tuple(ranked) + + +class LexicalRetriever: + def __init__( + self, + knowledge_base: KnowledgeBase, + registry: Mapping[str, Mapping[str, Any]], + ) -> None: + self.knowledge_base = knowledge_base + self.registry = _canonical_registry(registry) + + def retrieve(self, query: Any, top_k: int = 5) -> RetrievalResult: + candidates = self.knowledge_base.search_standard(query, top_k=top_k) + return RetrievalResult( + validate_ranked_candidates(candidates, self.registry, top_k=top_k), + {"backend": "lexical"}, + ) + + +def _single_json_object(text: str) -> Mapping[str, Any]: + if not isinstance(text, str) or not text.strip(): + raise RetrievalError("model response must be non-empty JSON") + cleaned = text.strip() + if not cleaned.startswith("{"): + raise RetrievalError("model response must be one JSON object") + try: + value, end = json.JSONDecoder().raw_decode(cleaned) + except JSONDecodeError as exc: + raise RetrievalError(f"model response is invalid JSON: {exc.msg}") from exc + if cleaned[end:].strip() or not isinstance(value, Mapping): + raise RetrievalError("model response must be exactly one JSON object") + return value + + +def _last_generation_metadata(generate: Callable[..., Any]) -> dict[str, Any]: + metadata = getattr(generate, "last_generation_metadata", None) + return dict(metadata) if isinstance(metadata, Mapping) else {} + + +class CatalogModelRetriever: + """Ask a text generator to rank IDs from the complete frozen Registry.""" + + def __init__( + self, + generate: Callable[[list[dict[str, str]]], str], + registry: Mapping[str, Mapping[str, Any]], + ) -> None: + self.generate = generate + self.registry = _canonical_registry(registry) + + def _messages(self, query: Any, top_k: int) -> list[dict[str, str]]: + catalog = [ + { + "category_id": category, + "description": str(entry.get("description") or entry.get("definition") or category), + } + for category, entry in self.registry.items() + ] + payload = { + "semantic_query": query, + "top_k": top_k, + "category_catalog": catalog, + "output_schema": {"category_ids": ["exact Registry category ID"]}, + } + return [ + { + "role": "system", + "content": ( + "Rank the database-field categories supported by the semantic query. " + "Choose only exact category_id values from the supplied complete catalog. " + f"Return exactly {top_k} distinct IDs in descending relevance. " + "Return one JSON object only, with exactly the key category_ids." + ), + }, + {"role": "user", "content": json.dumps(payload, ensure_ascii=False)}, + ] + + def retrieve(self, query: Any, top_k: int = 5) -> RetrievalResult: + messages = self._messages(query, top_k) + raw = self.generate(messages) + payload = _single_json_object(raw) + if set(payload) != {"category_ids"}: + raise RetrievalError("model response must contain exactly category_ids") + ids = payload.get("category_ids") + if not isinstance(ids, list): + raise RetrievalError("category_ids must be a list") + candidates = [{"category": value} for value in ids] + ranked = validate_ranked_candidates(candidates, self.registry, top_k=top_k) + return RetrievalResult( + ranked, + { + "backend": "model-catalog", + "messages": messages, + "raw_response": raw, + **_last_generation_metadata(self.generate), + }, + ) + + +EmbeddingFunction = Callable[..., Sequence[Sequence[float]]] + + +def _normalized(vector: Sequence[float]) -> tuple[float, ...]: + values = tuple(float(value) for value in vector) + norm = math.sqrt(sum(value * value for value in values)) + if not values or norm == 0.0: + raise RetrievalError("embedding vector must be non-empty and non-zero") + return tuple(value / norm for value in values) + + +class EmbeddingRetriever: + """Rank a frozen Registry with cached normalized catalog embeddings.""" + + def __init__( + self, + encode: EmbeddingFunction, + registry: Mapping[str, Mapping[str, Any]], + ) -> None: + self.encode = encode + self.registry = _canonical_registry(registry) + self._categories = tuple(self.registry) + self._catalog_vectors: tuple[tuple[float, ...], ...] | None = None + + def _catalog_texts(self) -> list[str]: + return [ + "passage: " + + category + + " " + + str(entry.get("description") or entry.get("definition") or category) + for category, entry in self.registry.items() + ] + + def _ensure_catalog(self) -> tuple[tuple[float, ...], ...]: + if self._catalog_vectors is None: + encoded = self.encode(self._catalog_texts(), kind="passage") + if len(encoded) != len(self._categories): + raise RetrievalError("catalog embedding count does not match Registry") + self._catalog_vectors = tuple(_normalized(vector) for vector in encoded) + dimensions = {len(vector) for vector in self._catalog_vectors} + if len(dimensions) != 1: + raise RetrievalError("catalog embedding dimension mismatch") + return self._catalog_vectors + + def retrieve(self, query: Any, top_k: int = 5) -> RetrievalResult: + if top_k > len(self.registry): + raise RetrievalError("top_k exceeds Registry size") + started = perf_counter() + catalog_vectors = self._ensure_catalog() + query_rows = self.encode([f"query: {format_retrieval_query(query)}"], kind="query") + if len(query_rows) != 1: + raise RetrievalError("query encoder must return exactly one vector") + query_vector = _normalized(query_rows[0]) + if len(query_vector) != len(catalog_vectors[0]): + raise RetrievalError("query/catalog embedding dimension mismatch") + scored = [ + (sum(left * right for left, right in zip(query_vector, vector)), category) + for category, vector in zip(self._categories, catalog_vectors) + ] + scored.sort(key=lambda item: (-item[0], item[1])) + ranked = validate_ranked_candidates( + [{"category": category} for _, category in scored[:top_k]], + self.registry, + top_k=top_k, + ) + return RetrievalResult( + ranked, + { + "backend": "embedding", + "elapsed_seconds": perf_counter() - started, + "scores": [score for score, _ in scored[:top_k]], + }, + ) + + +class TransformersE5Encoder: + """Lazy Hugging Face E5 encoder with masked mean pooling.""" + + def __init__( + self, + model_path: str | Path, + *, + local_files_only: bool = True, + device_map: str = "auto", + batch_size: int = 32, + ) -> None: + self.model_path = str(Path(model_path).expanduser()) + self.local_files_only = local_files_only + self.device_map = device_map + self.batch_size = batch_size + self._tokenizer = None + self._model = None + self._torch = None + + def _load(self) -> None: + if self._model is not None: + return + try: + import torch + from transformers import AutoModel, AutoTokenizer + except ImportError as exc: # pragma: no cover - optional remote dependency + raise RetrievalError("TransformersE5Encoder requires torch and transformers") from exc + self._torch = torch + self._tokenizer = AutoTokenizer.from_pretrained( + self.model_path, local_files_only=self.local_files_only + ) + self._model = AutoModel.from_pretrained( + self.model_path, + local_files_only=self.local_files_only, + device_map=self.device_map, + dtype="auto", + ) + self._model.eval() + + def __call__(self, texts: Sequence[str], *, kind: str) -> list[list[float]]: + if kind not in {"query", "passage"}: + raise RetrievalError("embedding kind must be query or passage") + self._load() + assert self._tokenizer is not None + assert self._model is not None + assert self._torch is not None + rows: list[list[float]] = [] + device = next(self._model.parameters()).device + for start in range(0, len(texts), self.batch_size): + batch = list(texts[start : start + self.batch_size]) + inputs = self._tokenizer( + batch, + max_length=512, + padding=True, + truncation=True, + return_tensors="pt", + ) + inputs = {name: value.to(device) for name, value in inputs.items()} + with self._torch.no_grad(): + hidden = self._model(**inputs).last_hidden_state + mask = inputs["attention_mask"].unsqueeze(-1).to(hidden.dtype) + pooled = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1) + pooled = self._torch.nn.functional.normalize(pooled, p=2, dim=1) + rows.extend(pooled.detach().cpu().tolist()) + return rows + + +__all__ = [ + "CatalogModelRetriever", + "EmbeddingRetriever", + "LexicalRetriever", + "RetrievalError", + "RetrievalResult", + "TransformersE5Encoder", + "format_retrieval_query", + "validate_ranked_candidates", +] diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/runtime.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/runtime.py new file mode 100644 index 0000000..e90f120 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/runtime.py @@ -0,0 +1,173 @@ +"""CPU-only deterministic indexes and a mock Skill executor.""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass, field +import re +from typing import Any, Mapping, Sequence + +from .catalog import SkillSpec, default_catalog +from .contracts import MetadataRecord, Program, StandardEntry, ToolCall +from .validator import assert_valid_program, resolve_reference + + +@dataclass +class KnowledgeBase: + records: tuple[MetadataRecord, ...] + standards: tuple[StandardEntry, ...] + _table_index: dict[tuple[str | None, str], list[MetadataRecord]] = field(default_factory=dict) + + @classmethod + def from_mappings(cls, records: Sequence[Mapping[str, Any]], standards: Sequence[StandardEntry]) -> "KnowledgeBase": + normalized = tuple(MetadataRecord.from_mapping(record) for record in records) + index: dict[tuple[str | None, str], list[MetadataRecord]] = defaultdict(list) + for record in normalized: + table = record.get("table_name") + if table: + index[(record.get("database_name"), str(table))].append(record) + return cls(normalized, tuple(standards), dict(index)) + + def table_fields(self, table_name: str, database_name: str | None = None) -> list[str]: + rows = self._table_index.get((database_name, table_name), []) + if not rows and database_name is not None: + rows = [record for (db, table), values in self._table_index.items() if table == table_name for record in values] + return list(dict.fromkeys(record.field_name for record in rows)) + + def search_standard(self, query: Any, top_k: int = 5) -> list[dict[str, str]]: + tokens = _search_tokens(query) + scored = [] + for entry in self.standards: + haystack_tokens = _search_tokens(f"{entry.category} {entry.description}") + score = len(tokens.intersection(haystack_tokens)) + scored.append((score, entry)) + scored.sort(key=lambda item: (-item[0], item[1].category)) + return [entry.to_dict() for _, entry in scored[:top_k]] + + def level_for(self, category: str) -> str | None: + for entry in self.standards: + if entry.category == category: + return entry.grading_level + return None + + +def _flatten_text(value: Any) -> list[str]: + if isinstance(value, Mapping): + parts: list[str] = [] + for key, nested in value.items(): + parts.extend(_flatten_text(key)) + parts.extend(_flatten_text(nested)) + return parts + if isinstance(value, (list, tuple, set)): + parts = [] + for nested in value: + parts.extend(_flatten_text(nested)) + return parts + return [str(value)] if value is not None else [] + + +def _search_tokens(value: Any) -> set[str]: + """Extract identifier tokens and CJK n-grams from nested Skill inputs.""" + + text = " ".join(_flatten_text(value)).lower() + tokens = set(re.findall(r"[a-z0-9]+", text)) + for chunk in re.findall(r"[\u4e00-\u9fff]+", text): + if len(chunk) <= 3: + tokens.add(chunk) + tokens.update(chunk[index : index + 2] for index in range(len(chunk) - 1)) + tokens.update(chunk[index : index + 3] for index in range(len(chunk) - 2)) + return {token for token in tokens if token} + + +@dataclass(frozen=True) +class Observation: + skill: str + status: str + summary: str + data: Mapping[str, Any] = field(default_factory=dict) + metadata: Mapping[str, Any] = field(default_factory=dict) + + +@dataclass +class ExecutionTrace: + observations: list[Observation | Mapping[str, Any]] = field(default_factory=list) + terminated: bool = False + final_result: dict[str, Any] | None = None + state: dict[str, Any] = field(default_factory=dict) + + +class MockSkillExecutor: + """Run deterministic and placeholder semantic Skills without an LLM.""" + + def __init__(self, knowledge_base: KnowledgeBase, catalog: Mapping[str, SkillSpec] | None = None): + self.knowledge_base = knowledge_base + self.catalog = dict(catalog or default_catalog()) + + def execute(self, call: ToolCall, record: MetadataRecord, state: Mapping[str, Any]) -> Observation: + spec = self.catalog[call.tool] + resolved = { + key: resolve_reference(value, record, state)[1] + for key, value in call.arguments.items() + } + + if call.tool == "CHECK_METADATA_AVAILABILITY": + available = ["domain", "field_name"] + [name for name in record.metadata if record.available(name) and name != "field_name"] + missing = [name for name in ("database_name", "database_description", "table_name", "table_description", "field_description", "field_type", "value") if not record.available(name)] + data = {"metadata_availability": {"available": available, "missing": missing}} + elif call.tool == "GET_TABLE_FIELDS": + table_name = str(resolved["table_name"]) + data = {"table_fields": self.knowledge_base.table_fields(table_name, record.get("database_name"))} + elif call.tool == "SELECT_CONTEXT_FIELDS": + fields = list(resolved["table_fields"] or []) + target = str(resolved["target_field"]) + data = {"selected_context_fields": [field for field in fields if field != target][:3]} + elif call.tool == "SEARCH_STANDARD": + data = {"standard_candidates": self.knowledge_base.search_standard(resolved["semantic_query"])} + elif call.tool == "INFER_CATEGORY": + candidates = resolved["candidate_standard_categories"] + if isinstance(candidates, Mapping): + candidates = [candidates] + candidates = list(candidates or []) + first = dict(candidates[0]) if candidates else {} + category = first.get("category") or first.get("category_id") or first.get("name") + data = {"category": category, "aligned_standard_entry": first} + elif call.tool == "INFER_GRADING_LEVEL": + entry = resolved["aligned_standard_entry"] + if isinstance(entry, Mapping): + level = entry.get("grading_level") + else: + level = self.knowledge_base.level_for(str(resolved["aligned_category"])) + data = {"grading_level": level} + elif call.tool == "FINALIZE_CLASSIFICATION": + data = {"final_result": {"category": state.get("category"), "grading_level": state.get("grading_level")}} + else: + produced = spec.produces[0] if spec.produces else "observation" + data = {produced: {"input": resolved, "source_skill": call.tool}} + + return Observation(call.tool, "success", f"Mock observation from {call.tool}", data) + + +def execute_program( + record: MetadataRecord, + program: Program, + executor: MockSkillExecutor, + *, + initial_state: Mapping[str, Any] | None = None, +) -> ExecutionTrace: + assert_valid_program(record, program, executor.catalog, initial_state=initial_state) + state = dict(initial_state or {}) + trace = ExecutionTrace(state=state) + for call in program.steps: + observation = executor.execute(call, record, state) + trace.observations.append(observation) + if observation.status != "success": + break + state.update(observation.data) + if call.tool == "FINALIZE_CLASSIFICATION": + trace.terminated = True + trace.final_result = dict(observation.data["final_result"]) + break + return trace + + +__all__ = ["ExecutionTrace", "KnowledgeBase", "MockSkillExecutor", "Observation", "execute_program"] diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/skill_loader.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/skill_loader.py new file mode 100644 index 0000000..cb43a6a --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/skill_loader.py @@ -0,0 +1,159 @@ +"""Strict loader for one Markdown contract per orchestration Skill.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping + +from .catalog import SkillSpec + + +class SkillContractError(ValueError): + """Raised when Skill files are missing, duplicated, or inconsistent.""" + + +@dataclass(frozen=True) +class SkillContract: + name: str + dsl: str + required_inputs: tuple[str, ...] + optional_inputs: tuple[str, ...] + produces: tuple[str, ...] + terminal: bool + body: str + path: Path + + +def _parse_list(value: str, *, key: str, path: Path) -> tuple[str, ...]: + value = value.strip() + if not value.startswith("[") or not value.endswith("]"): + raise SkillContractError(f"{path}: {key} must use [a, b] syntax") + inner = value[1:-1].strip() + if not inner: + return () + items = tuple(item.strip() for item in inner.split(",")) + if any(not item for item in items): + raise SkillContractError(f"{path}: {key} contains an empty value") + return items + + +def _parse_contract(path: Path) -> SkillContract: + text = path.read_text(encoding="utf-8") + lines = text.splitlines() + if len(lines) < 3 or lines[0].strip() != "---": + raise SkillContractError(f"{path}: missing opening front matter marker") + try: + end = next(index for index, line in enumerate(lines[1:], start=1) if line.strip() == "---") + except StopIteration as exc: + raise SkillContractError(f"{path}: missing closing front matter marker") from exc + + header: dict[str, str] = {} + for line in lines[1:end]: + if ":" not in line: + raise SkillContractError(f"{path}: invalid front matter line {line!r}") + key, value = line.split(":", 1) + key = key.strip() + if key in header: + raise SkillContractError(f"{path}: duplicate front matter key {key!r}") + header[key] = value.strip() + + required = {"name", "dsl", "required_inputs", "optional_inputs", "produces", "terminal"} + if set(header) != required: + missing = sorted(required - set(header)) + extra = sorted(set(header) - required) + raise SkillContractError(f"{path}: front matter keys mismatch; missing={missing}, extra={extra}") + terminal_text = header["terminal"].lower() + if terminal_text not in {"true", "false"}: + raise SkillContractError(f"{path}: terminal must be true or false") + body = "\n".join(lines[end + 1 :]).strip() + if not body: + raise SkillContractError(f"{path}: Skill contract body is empty") + return SkillContract( + name=header["name"], + dsl=header["dsl"], + required_inputs=_parse_list(header["required_inputs"], key="required_inputs", path=path), + optional_inputs=_parse_list(header["optional_inputs"], key="optional_inputs", path=path), + produces=_parse_list(header["produces"], key="produces", path=path), + terminal=terminal_text == "true", + body=body, + path=path, + ) + + +class SkillLoader: + """Discover and validate Markdown Skill contracts against a catalog.""" + + def __init__( + self, + root: str | Path, + catalog: Mapping[str, SkillSpec], + *, + extra_roots: tuple[str | Path, ...] = (), + strict_catalog: bool = True, + ) -> None: + self.root = Path(root) + self.extra_roots = tuple(Path(path) for path in extra_roots) + self.strict_catalog = strict_catalog + self.catalog = dict(catalog) + self._contracts: dict[str, SkillContract] | None = None + + def validate_all(self) -> dict[str, SkillContract]: + discovered: dict[str, list[SkillContract]] = {} + for root in (self.root, *self.extra_roots): + if root.is_dir(): + for path in sorted(root.glob("*.md")): + contract = _parse_contract(path) + discovered.setdefault(contract.name, []).append(contract) + + validated: dict[str, SkillContract] = {} + for name, spec in self.catalog.items(): + matches = discovered.get(name, []) + if not matches: + raise SkillContractError(f"missing Skill contract for {name}") + if len(matches) > 1: + paths = ", ".join(str(item.path) for item in matches) + raise SkillContractError(f"duplicate Skill contracts for {name}: {paths}") + contract = matches[0] + expected_filename = f"{spec.role}_{spec.name}.md" + if contract.path.name != expected_filename: + raise SkillContractError( + f"{contract.path}: filename must be {expected_filename!r}" + ) + expected = { + "dsl": spec.role, + "required_inputs": spec.required_inputs, + "optional_inputs": spec.optional_inputs, + "produces": spec.produces, + "terminal": spec.terminal, + } + actual = { + "dsl": contract.dsl, + "required_inputs": contract.required_inputs, + "optional_inputs": contract.optional_inputs, + "produces": contract.produces, + "terminal": contract.terminal, + } + for key, value in expected.items(): + if actual[key] != value: + raise SkillContractError( + f"{contract.path}: {key}={actual[key]!r} does not match catalog {value!r}" + ) + validated[name] = contract + + unexpected = sorted(set(discovered) - set(self.catalog)) + if unexpected and self.strict_catalog: + raise SkillContractError(f"unexpected Skill contracts: {unexpected}") + self._contracts = validated + return dict(validated) + + def load(self, name: str) -> SkillContract: + if name not in self.catalog: + raise SkillContractError(f"unknown Skill {name!r}") + if self._contracts is None: + self.validate_all() + assert self._contracts is not None + return self._contracts[name] + + +__all__ = ["SkillContract", "SkillContractError", "SkillLoader"] diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/table_field_lookup.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/table_field_lookup.py new file mode 100644 index 0000000..2935086 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/table_field_lookup.py @@ -0,0 +1,89 @@ +"""Label-isolated, bounded lookup of neighboring fields in one real table.""" + +from __future__ import annotations + +from collections import defaultdict +import json +from pathlib import Path +from typing import Any, Mapping + +from .contracts import MetadataRecord + + +TableKey = tuple[str, str, str] + + +class TableFieldLookup: + def __init__( + self, + identities: Mapping[str, tuple[str, str, TableKey]], + tables: Mapping[TableKey, list[tuple[str, str]]], + *, + max_fields: int = 30, + ) -> None: + if max_fields < 1: + raise ValueError("max_fields must be positive") + self.max_fields = max_fields + self._identities = dict(identities) + self._tables = {key: tuple(values) for key, values in tables.items()} + + @classmethod + def from_canonical(cls, source: str | Path, *, max_fields: int = 30) -> "TableFieldLookup": + raw = json.loads(Path(source).read_text(encoding="utf-8")) + if not isinstance(raw, list): + raise ValueError("canonical source must be a list") + identities: dict[str, tuple[str, str, TableKey]] = {} + tables: dict[TableKey, list[tuple[str, str]]] = defaultdict(list) + for row in raw: + if not isinstance(row, Mapping): + raise ValueError("canonical row must be an object") + source_id = row.get("id") + if not isinstance(source_id, str) or not source_id.strip(): + raise ValueError("canonical row must have source id") + if source_id in identities: + raise ValueError(f"duplicate source id: {source_id}") + metadata = row.get("metadata") + if not isinstance(metadata, Mapping): + raise ValueError(f"canonical metadata missing for {source_id}") + domain = str(row.get("domain") or "").strip() + database = str(metadata.get("database_name") or "").strip() + table = str(metadata.get("table_name") or "").strip() + field_name = str(metadata.get("field_name") or "").strip() + if not domain or not database or not table or not field_name: + raise ValueError(f"table identity is incomplete for {source_id}") + key = (domain, database, table) + identities[source_id] = (domain, field_name, key) + description = str(metadata.get("field_description") or "").strip() + tables[key].append((field_name, description)) + return cls(identities, tables, max_fields=max_fields) + + def get(self, record: MetadataRecord) -> dict[str, Any]: + if not record.source_id or record.source_id not in self._identities: + raise ValueError("source id is absent from table field lookup") + domain, field_name, key = self._identities[record.source_id] + if domain != record.domain or field_name != record.field_name: + raise ValueError("lookup source does not match domain and field_name") + selected: dict[str, str] = {} + for name, description in self._tables[key]: + if name == record.field_name: + continue + previous = selected.get(name, "") + if description and (not previous or description < previous): + selected[name] = description + else: + selected.setdefault(name, previous) + names = sorted(selected, key=lambda value: (value.casefold(), value)) + fields: list[dict[str, str]] = [] + for name in names[: self.max_fields]: + item = {"field_name": name} + if selected[name]: + item["field_description"] = selected[name] + fields.append(item) + return { + "fields": fields, + "available_count": len(names), + "truncated": len(names) > self.max_fields, + } + + +__all__ = ["TableFieldLookup"] diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/topk_selection.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/topk_selection.py new file mode 100644 index 0000000..0a98919 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/topk_selection.py @@ -0,0 +1,61 @@ +"""Label-free, deterministic balanced presentation and exact vote aggregation.""" + +from __future__ import annotations + +import hashlib +import json +import random +from fractions import Fraction + + +SELECTION_POLICY = "canonical-sha256-shuffle-cyclic3-votes-rrf-hash-v1" + + +def _digest(seed: int, domain: str, field_name: str, option_id: str | None = None) -> str: + values = [seed, domain, field_name] + if option_id is not None: + values.append(option_id) + return hashlib.sha256(json.dumps(values, ensure_ascii=False, separators=(",", ":")).encode("utf-8")).hexdigest() + + +def balanced_options(options: list[dict], domain: str, field_name: str, seed: int) -> list[list[dict]]: + """Shuffle canonical IDs once, then present floor(i*N/3) cyclic offsets.""" + ids = [item.get("option_id") for item in options] + if not ids or any(not isinstance(item, str) or not item for item in ids) or len(set(ids)) != len(ids): + raise ValueError("options require distinct nonempty option IDs") + canonical = [dict(item) for item in sorted(options, key=lambda item: item["option_id"])] + random.Random(int(_digest(seed, domain, field_name), 16)).shuffle(canonical) + rounds = [] + for index in range(3): + offset = index * len(canonical) // 3 + rounds.append([dict(item) for item in canonical[offset:] + canonical[:offset]]) + return rounds + + +def ranking_scores(rankings: list[list[str]], allowed: list[str], domain: str, + field_name: str, seed: int, top_k: int = 5) -> list[dict]: + """Return exact score records in the fixed aggregation order for audit traces.""" + if top_k < 1 or top_k > len(allowed) or len(set(allowed)) != len(allowed): + raise ValueError("top_k must fit a distinct allowed catalog") + if len(rankings) < 2: + raise ValueError("balanced-vote requires at least two valid rankings") + for ranking in rankings: + if (not isinstance(ranking, list) or len(ranking) != top_k + or any(not isinstance(item, str) or item not in allowed for item in ranking) + or len(set(ranking)) != top_k): + raise ValueError("each ranking requires exactly top_k distinct in-catalog IDs") + scores = [] + for option_id in sorted(allowed): + ranks = [ranking.index(option_id) + 1 for ranking in rankings if option_id in ranking] + if ranks: + score = sum((Fraction(1, rank) for rank in ranks), Fraction()) + scores.append((option_id, len(ranks), score, _digest(seed, domain, field_name, option_id))) + scores.sort(key=lambda item: (-item[1], -item[2], item[3])) + return [{"option_id": option_id, "votes": votes, "reciprocal_rank_score": str(score), "tie_hash": tie_hash} + for option_id, votes, score, tie_hash in scores] + + +def aggregate_rankings(rankings: list[list[str]], allowed: list[str], domain: str, + field_name: str, seed: int, top_k: int = 5) -> list[str]: + """Require quorum, validate all rankings and select by votes, exact RRF, hash.""" + return [item["option_id"] for item in ranking_scores(rankings, allowed, domain, field_name, seed, top_k)[:top_k]] diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/validator.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/validator.py new file mode 100644 index 0000000..8c30690 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/agent/orchestration/validator.py @@ -0,0 +1,131 @@ +"""Static validation for planner-produced Programs.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Mapping + +from .catalog import SkillSpec +from .contracts import MetadataRecord, Program + + +DEFAULT_MAX_STEPS = 10 + + +@dataclass(frozen=True) +class ValidationIssue: + code: str + message: str + step_index: int | None = None + + def __str__(self) -> str: + location = "" if self.step_index is None else f" at step {self.step_index}" + return f"{self.code}{location}: {self.message}" + + +def resolve_reference(value: Any, record: MetadataRecord, state: Mapping[str, Any]) -> tuple[bool, Any]: + if not isinstance(value, str) or not value.startswith("$"): + return bool(value is not None and value != ""), value + if value == "$record": + # The controller may retain source_id privately for deterministic lookups. + # It is never included in resolved arguments sent to model-backed Skills. + return True, {"domain": record.domain, "metadata": dict(record.metadata)} + if value == "$domain": + return True, record.domain + if value.startswith("$metadata."): + name = value[len("$metadata.") :] + resolved = record.get(name) + return record.available(name), resolved + if value.startswith("$state."): + name = value[len("$state.") :] + return name in state, state.get(name) + return False, None + + +def validate_program( + record: MetadataRecord, + program: Program, + catalog: Mapping[str, SkillSpec], + *, + initial_state: Mapping[str, Any] | None = None, + max_steps: int = DEFAULT_MAX_STEPS, +) -> list[ValidationIssue]: + issues: list[ValidationIssue] = [] + state = dict(initial_state or {}) + seen: dict[str, int] = {} + finalized = False + + if len(program.steps) > max_steps: + issues.append(ValidationIssue("max_steps_exceeded", f"program has {len(program.steps)} steps; max is {max_steps}")) + + for index, call in enumerate(program.steps): + spec = catalog.get(call.tool) + if spec is None: + issues.append(ValidationIssue("unknown_skill", f"unknown Skill {call.tool!r}", index)) + continue + if call.tool == "INFER_CATEGORY_FROM_TOPK": + if not seen.get("SELECT_CATEGORY_TOPK") and "standard_candidates" not in (initial_state or {}): + issues.append(ValidationIssue( + "missing_topk_dependency", + "INFER_CATEGORY_FROM_TOPK requires a preceding SELECT_CATEGORY_TOPK", + index, + )) + if call.arguments.get("candidate_standard_categories") != "$state.standard_candidates": + issues.append(ValidationIssue( + "invalid_topk_reference", + "candidate_standard_categories must reference $state.standard_candidates", + index, + )) + if finalized: + issues.append(ValidationIssue("calls_after_finalizer", "no calls are allowed after FINALIZE_CLASSIFICATION", index)) + + seen[call.tool] = seen.get(call.tool, 0) + 1 + if seen[call.tool] > spec.max_calls: + issues.append(ValidationIssue("duplicate_skill", f"{call.tool} exceeds max_calls={spec.max_calls}", index)) + + for name in spec.required_inputs: + if name not in call.arguments: + issues.append(ValidationIssue("missing_required_argument", f"{call.tool} requires argument {name!r}", index)) + continue + found, _ = resolve_reference(call.arguments[name], record, state) + if not found: + value = call.arguments[name] + if isinstance(value, str) and value.startswith("$metadata."): + code = "unavailable_metadata" + elif isinstance(value, str) and value.startswith("$state."): + code = "unresolved_reference" + else: + code = "empty_required_argument" + issues.append(ValidationIssue(code, f"required argument {name!r} cannot be resolved", index)) + + for name in spec.optional_inputs: + if name in call.arguments: + resolve_reference(call.arguments[name], record, state) + + if spec.terminal: + finalized = True + if index != len(program.steps) - 1: + issues.append(ValidationIssue("finalizer_not_last", "FINALIZE_CLASSIFICATION must be the last step", index)) + + for produced in spec.produces: + state.setdefault(produced, {"source_skill": call.tool}) + + if not finalized: + issues.append(ValidationIssue("missing_finalizer", "program must end with FINALIZE_CLASSIFICATION")) + return issues + + +def assert_valid_program( + record: MetadataRecord, + program: Program, + catalog: Mapping[str, SkillSpec], + *, + initial_state: Mapping[str, Any] | None = None, + max_steps: int = DEFAULT_MAX_STEPS, +) -> None: + issues = validate_program(record, program, catalog, initial_state=initial_state, max_steps=max_steps) + if issues: + raise ValueError("; ".join(str(issue) for issue in issues)) + + +__all__ = ["DEFAULT_MAX_STEPS", "ValidationIssue", "assert_valid_program", "resolve_reference", "validate_program"] diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/script/orchestration/__init__.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/script/orchestration/__init__.py new file mode 100644 index 0000000..09f258a --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/script/orchestration/__init__.py @@ -0,0 +1,2 @@ +"""CPU-only helpers for exercising orchestration contracts locally.""" + diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/script/orchestration/run_agent_loop_batch.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/script/orchestration/run_agent_loop_batch.py new file mode 100644 index 0000000..c8c6963 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/code/script/orchestration/run_agent_loop_batch.py @@ -0,0 +1,1223 @@ +"""Run and evaluate CPU-scripted dynamic Agent Loops without loading a model.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path +import random +import subprocess +import tempfile +from typing import Any, Callable, Iterable, Mapping, Sequence + +from agent.orchestration.agent_loop import ( + AgentLoopController, + FixedPipelinePolicy, + LoopConfig, + LoopEvent, + LoopTrace, + PrefetchNeighborFieldsPolicy, + PrefetchFieldMetadataPolicy, + ScriptedLoopPolicy, + ScriptedSkillExecutor, +) +from agent.orchestration.catalog import dynamic_loop_catalog +from agent.orchestration.contracts import MetadataRecord, StandardEntry +from agent.orchestration.model_skill_executor import ( + ModelLoopPolicy, + ModelSkillExecutor, + TextGenerator, + load_few_shot_messages, + make_standard_search_handler, + make_field_metadata_handler, + make_table_fields_handler, +) +from agent.orchestration.metadata_lookup import FieldMetadataLookup +from agent.orchestration.table_field_lookup import TableFieldLookup +from agent.orchestration.category_catalog import CategoryCatalog +from agent.orchestration.artifact_projection import project_trace +from agent.orchestration.runtime import KnowledgeBase +from agent.orchestration.runtime import Observation +from agent.orchestration.skill_loader import SkillLoader + + +ControllerFactory = Callable[[Mapping[str, Any]], AgentLoopController] + + +@dataclass(frozen=True) +class ArtifactPaths: + manifest: Path + traces: Path + summary: Path + observations: Path | None = None + model_calls: Path | None = None + trajectory: Path | None = None + evaluation: Path | None = None + + +def _read_json(path: Path) -> Any: + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def _read_rows(path: Path) -> list[dict[str, Any]]: + if path.suffix.lower() == ".jsonl": + rows = [] + with path.open("r", encoding="utf-8") as handle: + for line_number, line in enumerate(handle, start=1): + if not line.strip(): + continue + value = json.loads(line) + if not isinstance(value, Mapping): + raise ValueError(f"row {line_number} in {path} must be an object") + rows.append(dict(value)) + return rows + value = _read_json(path) + if not isinstance(value, list) or any(not isinstance(row, Mapping) for row in value): + raise ValueError(f"{path} must contain a JSON list of objects") + return [dict(row) for row in value] + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _file_entry(path: Path) -> dict[str, Any]: + return { + "path": str(path.resolve()), + "sha256": _sha256(path), + "bytes": path.stat().st_size, + } + + +def _registry_categories(value: Any) -> list[str]: + if isinstance(value, Mapping): + entries = value.get("categories") + else: + entries = value + if not isinstance(entries, list): + raise ValueError("registry must be a list or contain a categories list") + names: list[str] = [] + for entry in entries: + if isinstance(entry, str): + name = entry + elif isinstance(entry, Mapping): + name = str( + entry.get("category_id") + or entry.get("category") + or entry.get("name") + or entry.get("level_4") + or "" + ) + else: + name = "" + if name: + names.append(name) + return list(dict.fromkeys(names)) + + +def _registry_entry_map(value: Any) -> dict[str, dict[str, Any]]: + entries = value.get("categories") if isinstance(value, Mapping) else value + if not isinstance(entries, list): + raise ValueError("registry must be a list or contain a categories list") + result: dict[str, dict[str, Any]] = {} + for entry in entries: + if isinstance(entry, str): + normalized = {"category": entry, "description": entry, "grading_level": ""} + category = entry + elif isinstance(entry, Mapping): + category = str( + entry.get("category_id") + or entry.get("category") + or entry.get("name") + or entry.get("level_4") + or "" + ) + normalized = dict(entry) + normalized["category"] = category + else: + continue + if category: + if category in result: + raise ValueError(f"duplicate registry category {category!r}") + result[category] = normalized + return result + + +def select_pilot_rows( + rows: Iterable[Mapping[str, Any]], + *, + count: int, + seed: int, +) -> list[dict[str, Any]]: + """Select a stable pilot independent of input ordering and predictions.""" + + if count < 1: + raise ValueError("count must be positive") + normalized = [dict(row) for row in rows] + if any(not row.get("source_id") for row in normalized): + raise ValueError("every pilot row must have source_id") + normalized.sort(key=lambda row: str(row["source_id"])) + source_ids = [str(row["source_id"]) for row in normalized] + if len(set(source_ids)) != len(source_ids): + raise ValueError("pilot source_id values must be unique") + if count > len(normalized): + raise ValueError(f"requested {count} pilot rows from only {len(normalized)} rows") + rng = random.Random(seed) + rng.shuffle(normalized) + return normalized[:count] + + +def pending_rows_for_resume( + rows: Iterable[Mapping[str, Any]], + traces_path: str | Path, +) -> list[dict[str, Any]]: + """Return only rows whose source IDs are absent from an existing trace file.""" + + path = Path(traces_path) + completed: set[str] = set() + if path.is_file(): + for row in _read_rows(path): + source_id = row.get("source_id") + if source_id is not None: + completed.add(str(source_id)) + return [ + dict(row) + for row in rows + if str(row.get("source_id")) not in completed + ] + + +def _trace_from_dict(value: Mapping[str, Any]) -> LoopTrace: + return LoopTrace( + input=dict(value.get("input") or {}), + events=[ + LoopEvent( + kind=str(event.get("kind")), + turn=int(event.get("turn") or 0), + data=dict(event.get("data") or {}), + ) + for event in value.get("events", []) + if isinstance(event, Mapping) + ], + terminated=bool(value.get("terminated")), + final_result=( + dict(value["final_result"]) + if isinstance(value.get("final_result"), Mapping) + else None + ), + stop_reason=str(value.get("stop_reason") or "unknown"), + turns=int(value.get("turns") or 0), + state=dict(value.get("state") or {}), + ) + + +def _recalculate_saved_rows(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Recompute a resumed summary from preserved traces without model calls.""" + + replay_rows: list[dict[str, Any]] = [] + traces: dict[str, LoopTrace] = {} + for index, raw in enumerate(rows): + trace_value = raw.get("trace") + if not isinstance(trace_value, Mapping): + raise ValueError("saved row is missing its trace object") + source_id = str(raw.get("source_id") or f"saved-{index}") + trace = _trace_from_dict(trace_value) + traces[source_id] = trace + replay_rows.append( + { + "source_id": source_id, + "domain": trace.input.get("domain"), + "field_name": trace.input.get("field_name"), + "ground_truth": raw.get("ground_truth"), + "ground_truth_level": raw.get("ground_truth_level"), + } + ) + + class ReplayController: + def __init__(self, trace: LoopTrace) -> None: + self.trace = trace + + def run(self, _record: MetadataRecord) -> LoopTrace: + return self.trace + + return evaluate_agent_rows( + replay_rows, + lambda row: ReplayController(traces[str(row["source_id"])]), + ) + + +def load_experiment_variants(path: str | Path) -> list[dict[str, Any]]: + """Expand and validate the checked-in A/B/C experiment contract.""" + + payload = _read_json(Path(path)) + if not isinstance(payload, Mapping): + raise ValueError("experiment config must be an object") + common = payload.get("common") + raw_variants = payload.get("variants") + if not isinstance(common, Mapping) or not isinstance(raw_variants, list): + raise ValueError("experiment config requires common and variants") + variants: list[dict[str, Any]] = [] + for raw in raw_variants: + if not isinstance(raw, Mapping): + raise ValueError("each experiment variant must be an object") + variant = dict(common) + variant.update(dict(raw)) + variants.append(variant) + if [item.get("id") for item in variants] != ["A", "B", "C"]: + raise ValueError("variants must be ordered A, B, C") + if variants[0].get("policy") != "fixed" or any( + variants[index].get("policy") != "dynamic" for index in (1, 2) + ): + raise ValueError("A must be fixed and B/C must be dynamic") + if variants[1].get("few_shot") is not None or not variants[2].get("few_shot"): + raise ValueError("B must omit few-shot and C must enable it") + return variants + + +def build_model_controller( + *, + generate: TextGenerator, + policy_mode: str, + skill_root: str | Path, + registry_entries: Mapping[str, Mapping[str, Any]], + config: LoopConfig | None = None, + few_shot_messages: Sequence[Mapping[str, str]] = (), + top_k: int = 5, + metadata_lookup: FieldMetadataLookup | None = None, + table_field_lookup: TableFieldLookup | None = None, + neighbor_prefetch: bool = False, + metadata_prefetch: bool = False, + include_description_skills: bool = False, + topk_evidence_mode: str = "augmented", + topk_selection_mode: str = "single", + topk_selection_seed: int = 20260930, + top1_evidence_mode: str = "all", + top1_selection_mode: str = "model", + classification_mode: str = "legacy", +) -> AgentLoopController: + """Assemble one model-backed controller without importing Transformers.""" + + if classification_mode not in {"legacy", "two-stage"}: + raise ValueError("classification_mode must be legacy or two-stage") + if classification_mode == "two-stage" and policy_mode == "fixed": + raise ValueError("two-stage mode currently requires the dynamic policy") + if neighbor_prefetch and table_field_lookup is None: + raise ValueError("neighbor_prefetch requires table_field_lookup") + if neighbor_prefetch and policy_mode != "dynamic": + raise ValueError("neighbor_prefetch requires the dynamic policy") + if metadata_prefetch and ( + policy_mode != "dynamic" or classification_mode != "two-stage" + or metadata_lookup is None or not include_description_skills + ): + raise ValueError("metadata_prefetch requires dynamic two-stage classification, metadata_lookup and description Skills") + if include_description_skills and metadata_lookup is None: + raise ValueError("description Skills require metadata_lookup") + if include_description_skills and (classification_mode != "two-stage" or policy_mode != "dynamic"): + raise ValueError("description Skills require dynamic two-stage classification") + if topk_selection_mode not in {"single", "balanced-vote"}: + raise ValueError("topk_selection_mode must be single or balanced-vote") + if topk_selection_mode == "balanced-vote" and (classification_mode != "two-stage" or policy_mode != "dynamic"): + raise ValueError("balanced-vote requires dynamic two-stage classification") + if topk_evidence_mode not in {"augmented", "raw", "raw-flat"}: + raise ValueError("topk_evidence_mode must be augmented, raw or raw-flat") + if topk_evidence_mode in {"raw", "raw-flat"} and not include_description_skills: + raise ValueError("raw Top-K evidence mode requires description Skills") + two_stage = classification_mode == "two-stage" + catalog = dynamic_loop_catalog( + include_field_metadata=metadata_lookup is not None, + include_table_fields=table_field_lookup is not None, + include_description_skills=include_description_skills, + two_stage=two_stage, + ) + loader = SkillLoader( + skill_root, catalog, + extra_roots=( + *((Path(skill_root) / "metadata_extra",) if metadata_lookup is not None else ()), + *((Path(skill_root) / "neighbor_extra",) if table_field_lookup is not None else ()), + *((Path(skill_root) / "semantic_extra",) if include_description_skills else ()), + *((Path(skill_root) / "two_stage",) if two_stage else ()), + ), + strict_catalog=not two_stage, + ) + standards = tuple( + StandardEntry( + category=str(category), + description=str(entry.get("description") or entry.get("definition") or category), + grading_level=str(entry.get("grading_level") or ""), + ) + for category, entry in registry_entries.items() + ) + knowledge_base = KnowledgeBase.from_mappings([], standards) + handlers = {"SEARCH_STANDARD": make_standard_search_handler(knowledge_base, top_k=top_k)} + if metadata_lookup is not None: + handlers["GET_FIELD_METADATA"] = make_field_metadata_handler(metadata_lookup) + if table_field_lookup is not None: + handlers["GET_TABLE_FIELDS"] = make_table_fields_handler(table_field_lookup) + executor = ModelSkillExecutor( + generate, + loader, + deterministic_handlers=handlers, + category_catalog=CategoryCatalog(registry_entries) if two_stage else None, + top_k=top_k, + guard_two_stage_evidence=include_description_skills, + topk_evidence_mode=topk_evidence_mode, + topk_selection_mode=topk_selection_mode, + topk_selection_seed=topk_selection_seed, + top1_evidence_mode=top1_evidence_mode, + top1_selection_mode=top1_selection_mode, + ) + if policy_mode == "fixed": + policy = FixedPipelinePolicy() + elif policy_mode == "dynamic": + policy = ModelLoopPolicy( + generate, + few_shot_messages=few_shot_messages, + metadata_guidance=metadata_lookup is not None, + two_stage=two_stage, + description_skills=include_description_skills, + ) + if neighbor_prefetch: + policy = PrefetchNeighborFieldsPolicy(policy) + if metadata_prefetch: + policy = PrefetchFieldMetadataPolicy(policy) + else: + raise ValueError("policy_mode must be fixed or dynamic") + return AgentLoopController( + policy=policy, + executor=executor, + catalog=catalog, + skill_loader=loader, + config=config or LoopConfig(), + registry_entries=registry_entries, + ) + + +def build_run_manifest( + *, + dataset_id: str, + input_path: str | Path, + registry_path: str | Path, + corpus_path: str | Path | None, + code_revision: str, + sample_count: int, + seed: int, + behavior_files: Mapping[str, str | Path] | None = None, + model: Mapping[str, Any] | None = None, + loop_config: Mapping[str, Any] | None = None, + working_tree_dirty: bool | None = None, +) -> dict[str, Any]: + """Build an auditable CPU-run manifest from exact files.""" + + input_file = Path(input_path) + registry_file = Path(registry_path) + corpus_file = Path(corpus_path) if corpus_path is not None else None + categories = _registry_categories(_read_json(registry_file)) + files = { + "input": _file_entry(input_file), + "registry": _file_entry(registry_file), + } + if corpus_file is not None: + files["corpus"] = _file_entry(corpus_file) + for name, path in (behavior_files or {}).items(): + files[str(name)] = _file_entry(Path(path)) + default_loop_config = { + "max_turns": 8, + "max_identical_calls": 2, + "action_retries": 1, + "forced_finalize_retries": 1, + "repetition_retries": 1, + } + default_loop_config.update(dict(loop_config or {})) + return { + "dataset_id": dataset_id, + "sample_count": sample_count, + "seed": seed, + "input_fields": ["domain", "field_name"], + "category_count": len(categories), + "approved_levels": ["L1", "L2", "L3", "L4"], + "code_revision": code_revision, + "files": files, + "model": dict(model or {"loaded": False, "backend": "scripted-cpu"}), + "loop_config": default_loop_config, + "working_tree_dirty": working_tree_dirty, + } + + +def _record_from_row(row: Mapping[str, Any]) -> MetadataRecord: + raw = row.get("record") + source = dict(raw) if isinstance(raw, Mapping) else dict(row) + return MetadataRecord.from_mapping( + { + "domain": source.get("domain"), + "field_name": source.get("field_name") + or (source.get("metadata") or {}).get("field_name"), + "source_id": row.get("source_id"), + } + ) + + +def _trace_tool_stats(trace: LoopTrace) -> tuple[int, int]: + observations = [event for event in trace.events if event.kind == "observation"] + successes = sum(event.data.get("status") == "success" for event in observations) + return len(observations), successes + + +def _macro_f1(scored: Sequence[Mapping[str, Any]]) -> float: + labels = sorted( + { + str(value) + for row in scored + for value in (row.get("ground_truth"), row.get("predicted_category")) + if value is not None + } + ) + if not labels: + return 0.0 + scores: list[float] = [] + for label in labels: + true_positive = sum( + row.get("ground_truth") == label and row.get("predicted_category") == label + for row in scored + ) + false_positive = sum( + row.get("ground_truth") != label and row.get("predicted_category") == label + for row in scored + ) + false_negative = sum( + row.get("ground_truth") == label and row.get("predicted_category") != label + for row in scored + ) + denominator = (2 * true_positive) + false_positive + false_negative + scores.append((2 * true_positive) / denominator if denominator else 0.0) + return sum(scores) / len(scores) + + +def evaluate_agent_rows( + rows: Iterable[Mapping[str, Any]], + controller_factory: ControllerFactory, +) -> dict[str, Any]: + """Run loops while keeping evaluation labels outside controller state.""" + + scored: list[dict[str, Any]] = [] + total_calls = 0 + successful_calls = 0 + for index, raw in enumerate(rows): + row = dict(raw) + record = _record_from_row(row) + trace = controller_factory(row).run(record) + calls, successes = _trace_tool_stats(trace) + total_calls += calls + successful_calls += successes + final = trace.final_result or {} + predicted_category = final.get("category") + predicted_level = final.get("grading_level") + expected_category = row.get("ground_truth") + expected_level = row.get("ground_truth_level") + category_correct = bool(trace.terminated and predicted_category == expected_category) + level_correct = bool(trace.terminated and predicted_level == expected_level) + scored.append( + { + "index": index, + "source_id": row.get("source_id"), + "ground_truth": expected_category, + "ground_truth_level": expected_level, + "predicted_category": predicted_category, + "predicted_level": predicted_level, + "category_correct": category_correct, + "level_correct": level_correct, + "joint_correct": category_correct and level_correct, + "stop_reason": trace.stop_reason, + "trace": trace.to_dict(), + } + ) + + total = len(scored) + terminal = sum(bool(row["trace"]["terminated"]) for row in scored) + category_correct = sum(bool(row["category_correct"]) for row in scored) + level_correct = sum(bool(row["level_correct"]) for row in scored) + joint_correct = sum(bool(row["joint_correct"]) for row in scored) + retry_count = sum( + event["kind"] == "retry" + for row in scored + for event in row["trace"]["events"] + ) + repetition_interventions = sum( + event["kind"] == "controller_intervention" + and event["data"].get("reason") == "repetition_limit" + for row in scored + for event in row["trace"]["events"] + ) + format_failures = sum( + event["kind"] in {"retry", "rejected_action"} + and event["data"].get("reason") == "invalid_action" + for row in scored + for event in row["trace"]["events"] + ) + legal_action_failures = sum( + event["kind"] in {"retry", "rejected_action"} + and event["data"].get("reason") in {"uncallable_skill", "invalid_arguments"} + for row in scored + for event in row["trace"]["events"] + ) + early_stop_attempts = sum( + event["kind"] in {"retry", "rejected_action"} + and event["data"].get("reason") == "stop_without_final_result" + for row in scored + for event in row["trace"]["events"] + ) + accepted_action_turns = sum( + event["kind"] == "policy_action" + or ( + event["kind"] == "controller_intervention" + and event["data"].get("reason") == "repetition_limit" + ) + for row in scored + for event in row["trace"]["events"] + ) + rejected_action_turns = sum( + event["kind"] == "rejected_action" + for row in scored + for event in row["trace"]["events"] + ) + action_turns = accepted_action_turns + rejected_action_turns + policy_actions = sum( + event["kind"] == "policy_action" + for row in scored + for event in row["trace"]["events"] + ) + parsed_but_rejected = legal_action_failures + repetition_interventions + policy_response_attempts = ( + policy_actions + + parsed_but_rejected + + format_failures + + early_stop_attempts + ) + parsed_policy_responses = policy_actions + parsed_but_rejected + terminal_validation_failures = sum( + event["kind"] == "retry" and event["data"].get("reason") == "invalid_terminal" + for row in scored + for event in row["trace"]["events"] + ) + forced_finalizations = sum( + event["kind"] == "policy_action" and bool(event["data"].get("forced_finalize")) + for row in scored + for event in row["trace"]["events"] + ) + forced_finalization_missing_evidence = sum( + event["kind"] == "controller_intervention" + and event["data"].get("reason") == "forced_finalization_missing_evidence" + for row in scored + for event in row["trace"]["events"] + ) + forced_terminations = 0 + normal_terminations = 0 + retrieval_hits = 0 + retrieval_rows = 0 + neighbor_field_call_rows = 0 + neighbor_field_calls = 0 + description_skill_call_rows = 0 + description_skill_calls = 0 + required_input_violations = 0 + total_tokens = 0 + measured_elapsed_seconds = 0.0 + measured_model_calls = 0 + skill_correction_attempts = 0 + for row in scored: + events = row["trace"]["events"] + row_neighbor_calls = sum( + event["kind"] == "observation" + and event["data"].get("skill") == "GET_TABLE_FIELDS" + for event in events + ) + neighbor_field_calls += row_neighbor_calls + neighbor_field_call_rows += row_neighbor_calls > 0 + row_description_calls = sum( + event["kind"] == "observation" + and event["data"].get("skill") in { + "INTERPRET_FIELD_DESCRIPTION", "INTERPRET_TABLE_DESCRIPTION" + } + for event in events + ) + description_skill_calls += row_description_calls + description_skill_call_rows += row_description_calls > 0 + if row["trace"]["terminated"]: + final_actions = [ + event + for event in events + if event["kind"] == "policy_action" + and event["data"].get("call", {}).get("tool") == "FINALIZE_CLASSIFICATION" + ] + if final_actions and final_actions[-1]["data"].get("forced_finalize"): + forced_terminations += 1 + else: + normal_terminations += 1 + retrieval = [ + event for event in events + if event["kind"] == "observation" + and event["data"].get("skill") in {"SEARCH_STANDARD", "SELECT_CATEGORY_TOPK"} + and event["data"].get("status") == "success" + ] + if retrieval: + retrieval_rows += 1 + candidates = retrieval[-1]["data"].get("data", {}).get("standard_candidates", []) + expected = row.get("ground_truth") + if any( + isinstance(candidate, Mapping) + and (candidate.get("category") or candidate.get("category_id")) == expected + for candidate in candidates + ): + retrieval_hits += 1 + for event in events: + metadata = event["data"].get("metadata", {}) + if event["kind"] == "observation" and metadata.get("required_input_violation"): + required_input_violations += 1 + if event["kind"] == "observation": + skill_correction_attempts += max( + int(metadata.get("format_retry_count", int(metadata.get("model_call_count") or 1) - 1)), 0 + ) + token_usage = metadata.get("token_usage") + if isinstance(token_usage, Mapping): + total_tokens += int(token_usage.get("total_tokens") or 0) + elapsed = metadata.get("elapsed_seconds") + if isinstance(elapsed, (int, float)): + measured_elapsed_seconds += float(elapsed) + measured_model_calls += int(metadata.get("model_call_count") or 1) + return { + "total": total, + "terminal_results": terminal, + "termination_rate": terminal / total if total else 0.0, + "tool_calls": total_calls, + "successful_tool_calls": successful_calls, + "tool_execution_success_rate": successful_calls / total_calls if total_calls else 1.0, + "category_correct": category_correct, + "level_correct": level_correct, + "joint_correct": joint_correct, + "category_accuracy": category_correct / total if total else 0.0, + "level_accuracy": level_correct / total if total else 0.0, + "joint_accuracy": joint_correct / total if total else 0.0, + "category_macro_f1": _macro_f1(scored), + "average_tool_calls": total_calls / total if total else 0.0, + "neighbor_field_calls": neighbor_field_calls, + "neighbor_field_call_rows": neighbor_field_call_rows, + "neighbor_field_call_rate": neighbor_field_call_rows / total if total else 0.0, + "description_skill_calls": description_skill_calls, + "description_skill_call_rows": description_skill_call_rows, + "description_skill_call_rate": description_skill_call_rows / total if total else 0.0, + "action_turns": action_turns, + "accepted_action_turns": accepted_action_turns, + "action_parse_rate_after_retry": ( + accepted_action_turns / action_turns if action_turns else 0.0 + ), + "policy_response_attempts": policy_response_attempts, + "json_object_parse_rate": ( + parsed_policy_responses / policy_response_attempts + if policy_response_attempts + else 0.0 + ), + "legal_action_rate_given_parsed": ( + policy_actions / parsed_policy_responses if parsed_policy_responses else 0.0 + ), + "retry_count": retry_count, + "format_failures": format_failures, + "legal_action_failures": legal_action_failures, + "early_stop_attempts": early_stop_attempts, + "terminal_validation_failures": terminal_validation_failures, + "early_stop_recoveries": sum( + bool(row["trace"]["terminated"]) + and any( + event["data"].get("reason") == "stop_without_final_result" + for event in row["trace"]["events"] + if event["kind"] == "retry" + ) + for row in scored + ), + "repetition_interventions": repetition_interventions, + "repetition_recoveries": sum( + bool(row["trace"]["terminated"]) + and any( + event["kind"] == "controller_intervention" + and event["data"].get("reason") == "repetition_limit" + and bool(event["data"].get("recovered")) + for event in row["trace"]["events"] + ) + for row in scored + ), + "forced_finalizations": forced_finalizations, + "forced_finalization_missing_evidence": forced_finalization_missing_evidence, + "normal_terminations": normal_terminations, + "forced_terminations": forced_terminations, + "retrieval_recall_at_5": retrieval_hits / retrieval_rows if retrieval_rows else 0.0, + "retrieval_evaluated_rows": retrieval_rows, + "required_input_violations_reaching_execution": required_input_violations, + "total_tokens": total_tokens if measured_model_calls else None, + "measured_model_calls": measured_model_calls, + "skill_correction_attempts": skill_correction_attempts, + "measured_elapsed_seconds": ( + measured_elapsed_seconds if measured_model_calls else None + ), + "stop_reasons": { + reason: sum(row["stop_reason"] == reason for row in scored) + for reason in sorted({str(row["stop_reason"]) for row in scored}) + }, + "rows": scored, + } + + +def write_run_artifacts( + output_dir: str | Path, + manifest: Mapping[str, Any], + report: Mapping[str, Any], + *, + classification_mode: str = "legacy", +) -> ArtifactPaths: + """Write manifest, one trace row per sample, and aggregate summary.""" + + root = Path(output_dir) + root.mkdir(parents=True, exist_ok=True) + paths = ArtifactPaths( + manifest=root / "manifest.json", + traces=root / "traces.jsonl", + summary=root / "summary.json", + observations=root / "observations.jsonl" if classification_mode == "two-stage" else None, + model_calls=root / "model_calls.jsonl" if classification_mode == "two-stage" else None, + trajectory=root / "trajectory.jsonl" if classification_mode == "two-stage" else None, + evaluation=root / "evaluation.jsonl" if classification_mode == "two-stage" else None, + ) + rows = report.get("rows", []) + projected = [project_trace(row) for row in rows] if classification_mode == "two-stage" else [] + _atomic_write_text( + paths.manifest, json.dumps(dict(manifest), ensure_ascii=False, indent=2) + "\n" + ) + _atomic_write_text(paths.traces, _jsonl_text(rows)) + if classification_mode == "two-stage": + for name, target in ( + ("observations", paths.observations), + ("model_calls", paths.model_calls), + ("trajectory", paths.trajectory), + ("evaluation", paths.evaluation), + ): + assert target is not None + records = [] + for item in projected: + payload = item[name] + records.extend(payload if isinstance(payload, list) else [payload]) + _atomic_write_text(target, _jsonl_text(records)) + summary = {key: value for key, value in report.items() if key != "rows"} + _atomic_write_text(paths.summary, json.dumps(summary, ensure_ascii=False, indent=2) + "\n") + return paths + + +def _jsonl_text(rows: Iterable[Mapping[str, Any]]) -> str: + return "".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in rows) + + +def _atomic_write_text(path: Path, content: str) -> None: + """Replace one completed artifact atomically within its output directory.""" + + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", newline="\n", dir=path.parent, + prefix=f".{path.name}.", suffix=".tmp", delete=False, + ) as handle: + temp_path = Path(handle.name) + try: + handle.write(content) + except BaseException: + temp_path.unlink(missing_ok=True) + raise + try: + os.replace(temp_path, path) + finally: + temp_path.unlink(missing_ok=True) + + +def _git_revision() -> str: + try: + return subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL + ).strip() + except (OSError, subprocess.CalledProcessError): + return "unknown" + + +def _git_dirty() -> bool | None: + try: + output = subprocess.check_output( + ["git", "status", "--porcelain"], text=True, stderr=subprocess.DEVNULL + ) + return bool(output.strip()) + except (OSError, subprocess.CalledProcessError): + return None + + +def _skill_behavior_files( + skill_root: Path, *, classification_mode: str = "legacy", + include_metadata_extra: bool = False, + include_table_fields: bool = False, + include_description_skills: bool = False, +) -> dict[str, Path]: + files = { + f"skill:{path.name}": path + for path in sorted(skill_root.glob("*.md")) + if path.is_file() + } + if classification_mode == "two-stage": + files.update({ + f"skill:two_stage/{path.name}": path + for path in sorted((skill_root / "two_stage").glob("*.md")) + if path.is_file() + }) + if include_metadata_extra: + files.update({ + f"skill:metadata_extra/{path.name}": path + for path in (skill_root / "metadata_extra" / "metadata_acquisition_GET_FIELD_METADATA.md",) + if path.is_file() + }) + if include_table_fields: + path = skill_root / "neighbor_extra" / "metadata_acquisition_GET_TABLE_FIELDS.md" + if path.is_file(): + files[f"skill:neighbor_extra/{path.name}"] = path + if include_description_skills: + for path in sorted((skill_root / "semantic_extra").glob("*.md")): + files[f"skill:semantic_extra/{path.name}"] = path + return files + + +def _observation(raw: Mapping[str, Any], tool: str) -> Observation: + return Observation( + skill=str(raw.get("skill") or tool), + status=str(raw.get("status") or "success"), + summary=str(raw.get("summary") or tool), + data=dict(raw.get("data") or {}), + ) + + +def _scripted_factory( + scripts: Mapping[str, Any], + skill_root: Path, + registry_categories: Sequence[str], +) -> ControllerFactory: + catalog = dynamic_loop_catalog() + + def factory(row: Mapping[str, Any]) -> AgentLoopController: + source_id = str(row.get("source_id")) + sample = scripts.get(source_id) + if not isinstance(sample, Mapping): + raise ValueError(f"missing scripted responses for source_id={source_id!r}") + raw_outputs = sample.get("observations", {}) + if not isinstance(raw_outputs, Mapping): + raise ValueError(f"observations for {source_id!r} must be an object") + outputs = { + str(tool): [_observation(item, str(tool)) for item in values] + for tool, values in raw_outputs.items() + } + config_raw = sample.get("config", {}) + return AgentLoopController( + policy=ScriptedLoopPolicy(sample.get("actions", [])), + executor=ScriptedSkillExecutor(outputs), + catalog=catalog, + skill_loader=SkillLoader(skill_root, catalog), + config=LoopConfig(**dict(config_raw)), + registry_categories=registry_categories, + ) + + return factory + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--backend", choices=("scripted", "transformers"), default="scripted") + parser.add_argument("--responses", type=Path) + parser.add_argument("--policy", choices=("fixed", "dynamic"), default="dynamic") + parser.add_argument("--classification-mode", choices=("legacy", "two-stage"), default="legacy") + parser.add_argument("--model", type=Path) + parser.add_argument("--few-shot", type=Path) + parser.add_argument("--max-new-tokens", type=int, default=512) + parser.add_argument("--enable-thinking", action="store_true") + parser.add_argument("--allow-remote-model-files", action="store_true") + parser.add_argument("--top-k", type=int, default=5) + parser.add_argument("--max-turns", type=int, default=8) + parser.add_argument("--max-identical-calls", type=int, default=2) + parser.add_argument("--action-retries", type=int, default=1) + parser.add_argument("--forced-finalize-retries", type=int, default=1) + parser.add_argument("--repetition-retries", type=int, default=1) + parser.add_argument("--resume", action="store_true") + parser.add_argument("--registry", required=True, type=Path) + parser.add_argument("--corpus", type=Path) + parser.add_argument("--metadata-source", type=Path) + parser.add_argument("--neighbor-fields", action="store_true") + parser.add_argument("--neighbor-prefetch", action="store_true") + parser.add_argument("--description-skills", action="store_true") + parser.add_argument("--topk-evidence-mode", choices=("augmented", "raw", "raw-flat"), default="augmented") + parser.add_argument("--topk-selection-mode", choices=("single", "balanced-vote"), default="single") + parser.add_argument("--topk-selection-seed", type=int, default=20260930) + parser.add_argument("--skill-root", type=Path, default=Path("skills")) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--dataset-id", required=True) + parser.add_argument("--seed", type=int, default=20260921) + parser.add_argument("--pilot-count", type=int) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) + if args.topk_selection_mode == "balanced-vote" and ( + args.classification_mode != "two-stage" or args.policy != "dynamic" or args.backend != "transformers" + ): + raise ValueError("balanced-vote requires transformers and dynamic two-stage classification") + if args.topk_evidence_mode in {"raw", "raw-flat"} and not args.description_skills: + raise ValueError("raw Top-K evidence mode requires --description-skills") + if args.neighbor_prefetch and not args.neighbor_fields: + raise ValueError("--neighbor-prefetch requires --neighbor-fields") + if args.neighbor_fields and args.metadata_source is None: + raise ValueError("--neighbor-fields requires --metadata-source") + if args.neighbor_fields and args.backend != "transformers": + raise ValueError("--neighbor-fields requires the transformers backend") + if args.neighbor_fields and args.policy != "dynamic": + raise ValueError("--neighbor-fields currently requires --policy dynamic") + if args.description_skills and ( + args.metadata_source is None or args.backend != "transformers" + or args.policy != "dynamic" or args.classification_mode != "two-stage" + ): + raise ValueError("--description-skills requires transformers, dynamic two-stage, and --metadata-source") + if args.classification_mode == "two-stage" and args.backend != "transformers": + raise ValueError("two-stage requires the transformers backend") + if args.classification_mode == "two-stage" and args.policy != "dynamic": + raise ValueError("two-stage currently supports only the dynamic policy") + prior_manifest = args.output / "manifest.json" + if args.resume and not prior_manifest.is_file() and args.output.is_dir(): + prior_results = ( + "traces.jsonl", "summary.json", "observations.jsonl", "model_calls.jsonl", + "trajectory.jsonl", "evaluation.jsonl", + ) + if any((args.output / name).is_file() and (args.output / name).stat().st_size > 0 + for name in prior_results): + raise ValueError("cannot resume existing results without manifest.json") + if args.resume and prior_manifest.is_file(): + previous = _read_json(prior_manifest) + model = previous.get("model", {}) if isinstance(previous, Mapping) else {} + prior_selection = model.get("topk_selection_mode", "single") if isinstance(model, Mapping) else "single" + prior_seed = model.get("topk_selection_seed", 20260930) if isinstance(model, Mapping) else 20260930 + if prior_selection != args.topk_selection_mode: + raise ValueError("cannot resume artifacts from a different Top-K selection mode") + if prior_seed != args.topk_selection_seed: + raise ValueError("cannot resume artifacts from a different Top-K selection seed") + prior_mode = model.get("classification_mode", "legacy") if isinstance(model, Mapping) else "legacy" + if prior_mode != args.classification_mode: + raise ValueError("cannot resume artifacts from a different classification mode") + prior_neighbors = bool(model.get("neighbor_fields_dsl_enabled", False)) if isinstance(model, Mapping) else False + if prior_neighbors != args.neighbor_fields: + raise ValueError("cannot resume artifacts from a different neighbor-fields mode") + prior_prefetch = model.get("neighbor_fields_mode") == "prefetch" if isinstance(model, Mapping) else False + if prior_prefetch != args.neighbor_prefetch: + raise ValueError("cannot resume artifacts from a different neighbor prefetch mode") + prior_descriptions = bool(model.get("description_skills_enabled", False)) if isinstance(model, Mapping) else False + if prior_descriptions != args.description_skills: + raise ValueError("cannot resume artifacts from a different description-Skill mode") + prior_topk_mode = model.get("topk_evidence_mode", "augmented") if isinstance(model, Mapping) else "augmented" + if prior_topk_mode != args.topk_evidence_mode: + raise ValueError("cannot resume artifacts from a different Top-K evidence mode") + files = previous.get("files", {}) if isinstance(previous, Mapping) else {} + prior_has_source = isinstance(files, Mapping) and "metadata_source" in files + current_has_source = args.metadata_source is not None + if prior_has_source != current_has_source: + raise ValueError("cannot resume artifacts from a different metadata source configuration") + if current_has_source: + source_entry = files.get("metadata_source", {}) if isinstance(files, Mapping) else {} + prior_hash = source_entry.get("sha256") if isinstance(source_entry, Mapping) else None + if not isinstance(prior_hash, str) or prior_hash != _sha256(args.metadata_source): + source_kind = "neighbor metadata" if args.neighbor_fields else "metadata" + raise ValueError(f"cannot resume artifacts from a different {source_kind} source") + rows = _read_rows(args.input) + if args.pilot_count is not None: + rows = select_pilot_rows(rows, count=args.pilot_count, seed=args.seed) + intended_sample_count = len(rows) + traces_path = args.output / "traces.jsonl" + if args.output.exists() and any(args.output.iterdir()) and not args.resume: + raise FileExistsError(f"output directory is not empty: {args.output}; use --resume") + existing_scored = _read_rows(traces_path) if args.resume and traces_path.is_file() else [] + if args.resume: + rows = pending_rows_for_resume(rows, traces_path) + registry_value = _read_json(args.registry) + registry_categories = _registry_categories(registry_value) + registry_entries = _registry_entry_map(registry_value) + loop_config = LoopConfig( + max_turns=args.max_turns, + max_identical_calls=args.max_identical_calls, + action_retries=args.action_retries, + forced_finalize_retries=args.forced_finalize_retries, + repetition_retries=args.repetition_retries, + ) + behavior_files = _skill_behavior_files( + args.skill_root, classification_mode=args.classification_mode, + include_metadata_extra=args.backend == "transformers" and args.metadata_source is not None, + include_table_fields=args.neighbor_fields, + include_description_skills=args.description_skills, + ) + if args.backend == "scripted": + if args.responses is None: + raise ValueError("--responses is required for the scripted backend") + response_payload = _read_json(args.responses) + if not isinstance(response_payload, Mapping) or not isinstance( + response_payload.get("samples"), Mapping + ): + raise ValueError("responses must contain a samples object") + behavior_files["scripted_responses"] = args.responses + factory = _scripted_factory( + response_payload["samples"], + args.skill_root, + registry_categories, + ) + model_manifest: dict[str, Any] = { + "loaded": False, + "backend": "scripted-cpu", + "sample_config_overrides": { + str(source_id): dict(sample.get("config") or {}) + for source_id, sample in response_payload["samples"].items() + if isinstance(sample, Mapping) and sample.get("config") + }, + } + else: + if args.model is None: + raise ValueError("--model is required for the transformers backend") + from agent.orchestration.planner import TransformersTextGenerator + + generate = TransformersTextGenerator( + args.model, + max_new_tokens=args.max_new_tokens, + local_files_only=not args.allow_remote_model_files, + enable_thinking=args.enable_thinking, + ) + few_shot_messages = load_few_shot_messages(args.few_shot) if args.few_shot else () + metadata_lookup = ( + FieldMetadataLookup.from_canonical(args.metadata_source) + if args.metadata_source else None + ) + table_field_lookup = ( + TableFieldLookup.from_canonical(args.metadata_source) + if args.neighbor_fields else None + ) + if args.metadata_source: + behavior_files["metadata_source"] = args.metadata_source + if args.few_shot: + behavior_files["few_shot"] = args.few_shot + + def factory(_row: Mapping[str, Any]) -> AgentLoopController: + return build_model_controller( + generate=generate, + policy_mode=args.policy, + skill_root=args.skill_root, + registry_entries=registry_entries, + config=loop_config, + few_shot_messages=few_shot_messages, + top_k=args.top_k, + metadata_lookup=metadata_lookup, + table_field_lookup=table_field_lookup, + neighbor_prefetch=args.neighbor_prefetch, + include_description_skills=args.description_skills, + topk_evidence_mode=args.topk_evidence_mode, + topk_selection_mode=args.topk_selection_mode, + topk_selection_seed=args.topk_selection_seed, + classification_mode=args.classification_mode, + ) + + model_manifest = { + "loaded": True, + "backend": "transformers", + "name_or_path": str(args.model), + "local_files_only": not args.allow_remote_model_files, + "enable_thinking": args.enable_thinking, + "max_new_tokens": args.max_new_tokens, + "do_sample": False, + "policy": args.policy, + "few_shot": bool(args.few_shot), + "retrieval": { + "backend": ( + "model-full-catalog-short-options" if args.classification_mode == "two-stage" + else "deterministic-lexical" + ), + "top_k": args.top_k, + "selection_id_scheme": ( + "sorted-canonical-c001-v1" + if args.classification_mode == "two-stage" else None + ), + }, + "classification_mode": args.classification_mode, + "metadata_dsl_enabled": metadata_lookup is not None, + "neighbor_fields_dsl_enabled": table_field_lookup is not None, + "description_skills_enabled": args.description_skills, + "topk_evidence_mode": args.topk_evidence_mode, + "neighbor_fields_mode": ( + "prefetch" if args.neighbor_prefetch else + "optional" if table_field_lookup else "off" + ), + "neighbor_fields_limit": table_field_lookup.max_fields if table_field_lookup else None, + "neighbor_fields_source_policy": ( + "cross-split-unlabeled-table-metadata" if table_field_lookup else None + ), + } + from agent.orchestration.topk_selection import SELECTION_POLICY + + model_manifest.update({ + "topk_selection_mode": args.topk_selection_mode, + "topk_selection_seed": args.topk_selection_seed, + "topk_selection_algorithm": SELECTION_POLICY if args.topk_selection_mode == "balanced-vote" else "single-with-one-format-retry", + "topk_selection_rounds": 3 if args.topk_selection_mode == "balanced-vote" else 1, + "topk_selection_quorum": 2 if args.topk_selection_mode == "balanced-vote" else 1, + }) + report = evaluate_agent_rows(rows, factory) + if existing_scored: + report = _recalculate_saved_rows([*existing_scored, *report["rows"]]) + manifest = build_run_manifest( + dataset_id=args.dataset_id, + input_path=args.input, + registry_path=args.registry, + corpus_path=args.corpus, + code_revision=_git_revision(), + sample_count=intended_sample_count, + seed=args.seed, + behavior_files=behavior_files, + model=model_manifest, + loop_config=loop_config.__dict__, + working_tree_dirty=_git_dirty(), + ) + paths = write_run_artifacts(args.output, manifest, report, + classification_mode=args.classification_mode) + print( + json.dumps( + { + "manifest": str(paths.manifest), + "traces": str(paths.traces), + "summary": str(paths.summary), + **{key: value for key, value in report.items() if key != "rows"}, + }, + ensure_ascii=False, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + + +__all__ = [ + "ArtifactPaths", + "build_model_controller", + "build_run_manifest", + "evaluate_agent_rows", + "load_experiment_variants", + "main", + "pending_rows_for_resume", + "select_pilot_rows", + "write_run_artifacts", +] diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_ASSESS_CATEGORY_SUPPORT.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_ASSESS_CATEGORY_SUPPORT.md new file mode 100644 index 0000000..0624414 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_ASSESS_CATEGORY_SUPPORT.md @@ -0,0 +1,17 @@ +--- +name: ASSESS_CATEGORY_SUPPORT +dsl: evidence_reasoning +required_inputs: [candidate_category, evidence] +optional_inputs: [] +produces: [category_support] +terminal: false +--- +# ASSESS_CATEGORY_SUPPORT + +Purpose: Assesses how strongly the current evidence supports a candidate category. + +Observation: 输出自然语言支持评估：支持证据、反对证据、薄弱点和总体判断。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["category_support"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_CHECK_EVIDENCE_SUFFICIENCY.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_CHECK_EVIDENCE_SUFFICIENCY.md new file mode 100644 index 0000000..cb91800 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_CHECK_EVIDENCE_SUFFICIENCY.md @@ -0,0 +1,19 @@ +--- +name: CHECK_EVIDENCE_SUFFICIENCY +dsl: evidence_reasoning +required_inputs: [current_evidence] +optional_inputs: [] +produces: [evidence_sufficiency] +terminal: false +--- +# Role +Judge whether the current evidence supports category and grading decisions. + +# Evidence boundaries +Use only the supplied evidence state. Missing evidence must remain missing; do not infer ground truth. + +# Output +Return JSON with `summary` and `data.evidence_sufficiency`, including a boolean `sufficient` and concise rationale. + +# Failure +Return an explicit error when no evidence is available. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_COMPARE_CATEGORY_HYPOTHESES.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_COMPARE_CATEGORY_HYPOTHESES.md new file mode 100644 index 0000000..b69eaaf --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_COMPARE_CATEGORY_HYPOTHESES.md @@ -0,0 +1,17 @@ +--- +name: COMPARE_CATEGORY_HYPOTHESES +dsl: evidence_reasoning +required_inputs: [candidate_categories, evidence] +optional_inputs: [] +produces: [category_comparison] +terminal: false +--- +# COMPARE_CATEGORY_HYPOTHESES + +Purpose: Compares competing standard-category hypotheses. + +Observation: 输出候选类别的自然语言排序与关键区分依据。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["category_comparison"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_DECIDE_NEXT_EVIDENCE_SOURCE.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_DECIDE_NEXT_EVIDENCE_SOURCE.md new file mode 100644 index 0000000..92d385f --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_DECIDE_NEXT_EVIDENCE_SOURCE.md @@ -0,0 +1,17 @@ +--- +name: DECIDE_NEXT_EVIDENCE_SOURCE +dsl: evidence_reasoning +required_inputs: [current_evidence_state, available_metadata] +optional_inputs: [] +produces: [next_evidence_source] +terminal: false +--- +# DECIDE_NEXT_EVIDENCE_SOURCE + +Purpose: Determines which available evidence source should be explored next. + +Observation: 输出自然语言下一步建议，例如解释字段、获取同表字段、解释表/库、查询/重查标准或停止；不得调用超出可用信息范围的数据源。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["next_evidence_source"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FINALIZE_CLASSIFICATION.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FINALIZE_CLASSIFICATION.md new file mode 100644 index 0000000..5e49b98 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FINALIZE_CLASSIFICATION.md @@ -0,0 +1,19 @@ +--- +name: FINALIZE_CLASSIFICATION +dsl: evidence_reasoning +required_inputs: [final_evidence_state] +optional_inputs: [] +produces: [final_result] +terminal: true +--- +# Role +Return the final category and grading level supported by the current evidence state. + +# Evidence boundaries +Use existing evidence only. The category must exist in the frozen registry and the level must be approved. Never use ground truth. + +# Output +Return exactly one JSON object with `summary` and `data.final_result`, where the result contains both `category` and `grading_level`. + +# Failure +Return an explicit error when a valid complete result cannot be produced. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_CONTEXT_EVIDENCE.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_CONTEXT_EVIDENCE.md new file mode 100644 index 0000000..bd49f03 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_CONTEXT_EVIDENCE.md @@ -0,0 +1,17 @@ +--- +name: FUSE_CONTEXT_EVIDENCE +dsl: evidence_reasoning +required_inputs: [context_field_evidence] +optional_inputs: [] +produces: [context_evidence] +terminal: false +--- +# FUSE_CONTEXT_EVIDENCE + +Purpose: Integrates semantic evidence from selected contextual fields. + +Observation: 输出自然语言总结上下文字段共同指向的业务场景及其对目标字段的支持。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["context_evidence"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_DATABASE_EVIDENCE.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_DATABASE_EVIDENCE.md new file mode 100644 index 0000000..9546a0b --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_DATABASE_EVIDENCE.md @@ -0,0 +1,17 @@ +--- +name: FUSE_DATABASE_EVIDENCE +dsl: evidence_reasoning +required_inputs: [available_database_evidence] +optional_inputs: [] +produces: [database_evidence] +terminal: false +--- +# FUSE_DATABASE_EVIDENCE + +Purpose: Integrates database-level semantic evidence. + +Observation: 输出数据库级自然语言背景及其对目标字段的高层约束。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["database_evidence"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_FIELD_EVIDENCE.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_FIELD_EVIDENCE.md new file mode 100644 index 0000000..97908d1 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_FIELD_EVIDENCE.md @@ -0,0 +1,17 @@ +--- +name: FUSE_FIELD_EVIDENCE +dsl: evidence_reasoning +required_inputs: [available_field_evidence] +optional_inputs: [] +produces: [field_evidence] +terminal: false +--- +# FUSE_FIELD_EVIDENCE + +Purpose: Integrates all available evidence directly associated with the target field. + +Observation: 输出字段级统一自然语言证据状态，区分直接证据、辅助证据和歧义。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["field_evidence"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_METADATA_EVIDENCE.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_METADATA_EVIDENCE.md new file mode 100644 index 0000000..f17c548 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_METADATA_EVIDENCE.md @@ -0,0 +1,17 @@ +--- +name: FUSE_METADATA_EVIDENCE +dsl: evidence_reasoning +required_inputs: [available_metadata_evidence] +optional_inputs: [] +produces: [metadata_evidence] +terminal: false +--- +# FUSE_METADATA_EVIDENCE + +Purpose: Integrates multi-granularity metadata evidence into a unified semantic state. + +Observation: 输出统一 metadata evidence state，并指出不同粒度证据的一致与冲突。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["metadata_evidence"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_STANDARD_EVIDENCE.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_STANDARD_EVIDENCE.md new file mode 100644 index 0000000..4f74a10 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_STANDARD_EVIDENCE.md @@ -0,0 +1,17 @@ +--- +name: FUSE_STANDARD_EVIDENCE +dsl: evidence_reasoning +required_inputs: [standard_alignment_evidence] +optional_inputs: [] +produces: [standard_evidence] +terminal: false +--- +# FUSE_STANDARD_EVIDENCE + +Purpose: Integrates multiple standard-alignment results into a unified standard-evidence state. + +Observation: 输出统一标准证据状态，明确主要支持类别和弱支持类别。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["standard_evidence"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_TABLE_EVIDENCE.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_TABLE_EVIDENCE.md new file mode 100644 index 0000000..9cb0a5f --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_TABLE_EVIDENCE.md @@ -0,0 +1,17 @@ +--- +name: FUSE_TABLE_EVIDENCE +dsl: evidence_reasoning +required_inputs: [available_table_evidence] +optional_inputs: [] +produces: [table_evidence] +terminal: false +--- +# FUSE_TABLE_EVIDENCE + +Purpose: Integrates all available table-level semantic evidence. + +Observation: 输出统一表级自然语言证据状态。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["table_evidence"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_IDENTIFY_MISSING_EVIDENCE.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_IDENTIFY_MISSING_EVIDENCE.md new file mode 100644 index 0000000..b36a14c --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_IDENTIFY_MISSING_EVIDENCE.md @@ -0,0 +1,17 @@ +--- +name: IDENTIFY_MISSING_EVIDENCE +dsl: evidence_reasoning +required_inputs: [current_evidence, candidate_categories] +optional_inputs: [] +produces: [missing_evidence] +terminal: false +--- +# IDENTIFY_MISSING_EVIDENCE + +Purpose: Identifies the most important missing evidence needed to discriminate among candidate categories. + +Observation: 输出自然语言说明缺少哪类证据（字段描述、表字段、上下文字段语义、表描述、数据库描述、标准描述等）及其必要性。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["missing_evidence"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY.md new file mode 100644 index 0000000..265b235 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY.md @@ -0,0 +1,19 @@ +--- +name: INFER_CATEGORY +dsl: evidence_reasoning +required_inputs: [candidate_standard_categories, all_available_evidence] +optional_inputs: [] +produces: [category, aligned_standard_entry] +terminal: false +--- +# Role +Select the best-supported category from the supplied registry candidates. + +# Evidence boundaries +The category and aligned entry must come from the supplied candidates. Do not invent categories or use ground truth. + +# Output +Return JSON with `summary`, `data.category`, and `data.aligned_standard_entry`. + +# Failure +Return an explicit error when no candidate is supported. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY_FROM_TOPK.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY_FROM_TOPK.md new file mode 100644 index 0000000..96a1155 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY_FROM_TOPK.md @@ -0,0 +1,16 @@ +--- +name: INFER_CATEGORY_FROM_TOPK +dsl: evidence_reasoning +required_inputs: [candidate_standard_categories, all_available_evidence] +optional_inputs: [] +produces: [category, aligned_standard_entry] +terminal: false +--- +# Role +Choose the single best-supported short option ID from the prior Top-K choices. + +# Evidence boundary +The executor adds descriptions for only those K candidates. Do not use any option outside the supplied set, grading level, evaluation label, or long canonical category ID. + +# Output +Return one supplied short option ID and a concise evidence summary. The executor maps it to the canonical category ID and binds the frozen Registry entry after the choice. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_GRADING_LEVEL.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_GRADING_LEVEL.md new file mode 100644 index 0000000..0f61b63 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_GRADING_LEVEL.md @@ -0,0 +1,19 @@ +--- +name: INFER_GRADING_LEVEL +dsl: evidence_reasoning +required_inputs: [aligned_category, aligned_standard_entry] +optional_inputs: [] +produces: [grading_level] +terminal: false +--- +# Role +Read the approved sensitivity level associated with the aligned registry entry. + +# Evidence boundaries +Use only the aligned category and frozen registry entry. Do not infer a level from ground truth or unrelated examples. + +# Output +Return JSON with `summary` and `data.grading_level`, restricted to an approved level. + +# Failure +Return an explicit error when the entry does not support a valid level. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_FIELD_CONTEXT_CONFLICT.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_FIELD_CONTEXT_CONFLICT.md new file mode 100644 index 0000000..746a767 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_FIELD_CONTEXT_CONFLICT.md @@ -0,0 +1,17 @@ +--- +name: RESOLVE_FIELD_CONTEXT_CONFLICT +dsl: evidence_reasoning +required_inputs: [field_evidence, context_evidence] +optional_inputs: [] +produces: [field_context_resolution] +terminal: false +--- +# RESOLVE_FIELD_CONTEXT_CONFLICT + +Purpose: Resolves conflicts between target-field evidence and contextual-field evidence. + +Observation: 输出冲突点、比较依据、保留的解释以及仍需补充的信息。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["field_context_resolution"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_METADATA_CONFLICT.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_METADATA_CONFLICT.md new file mode 100644 index 0000000..3593ec1 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_METADATA_CONFLICT.md @@ -0,0 +1,17 @@ +--- +name: RESOLVE_METADATA_CONFLICT +dsl: evidence_reasoning +required_inputs: [conflicting_metadata_evidence] +optional_inputs: [] +produces: [metadata_conflict_resolution] +terminal: false +--- +# RESOLVE_METADATA_CONFLICT + +Purpose: Resolves conflicting semantic evidence across metadata granularities. + +Observation: 输出自然语言冲突消解结论和依据。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["metadata_conflict_resolution"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_STANDARD_CONFLICT.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_STANDARD_CONFLICT.md new file mode 100644 index 0000000..5492796 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_STANDARD_CONFLICT.md @@ -0,0 +1,17 @@ +--- +name: RESOLVE_STANDARD_CONFLICT +dsl: evidence_reasoning +required_inputs: [conflicting_standard_alignments] +optional_inputs: [] +produces: [standard_conflict_resolution] +terminal: false +--- +# RESOLVE_STANDARD_CONFLICT + +Purpose: Resolves conflicts among competing standard alignments. + +Observation: 输出哪个标准候选更可信、为什么，以及仍存在的歧义。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["standard_conflict_resolution"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CATEGORY_WITH_STANDARD.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CATEGORY_WITH_STANDARD.md new file mode 100644 index 0000000..de3852f --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CATEGORY_WITH_STANDARD.md @@ -0,0 +1,17 @@ +--- +name: VERIFY_CATEGORY_WITH_STANDARD +dsl: evidence_reasoning +required_inputs: [candidate_category, standard_evidence, metadata_evidence] +optional_inputs: [] +produces: [category_verification] +terminal: false +--- +# VERIFY_CATEGORY_WITH_STANDARD + +Purpose: Verifies whether a candidate category is jointly supported by metadata and standard evidence. + +Observation: 输出自然语言验证结论，明确标准描述与元数据语义是否一致。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["category_verification"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CLASSIFICATION_GRADING_CONSISTENCY.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CLASSIFICATION_GRADING_CONSISTENCY.md new file mode 100644 index 0000000..482763b --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CLASSIFICATION_GRADING_CONSISTENCY.md @@ -0,0 +1,17 @@ +--- +name: VERIFY_CLASSIFICATION_GRADING_CONSISTENCY +dsl: evidence_reasoning +required_inputs: [category_candidate, grading_level_candidate, current_evidence] +optional_inputs: [] +produces: [grading_consistency] +terminal: false +--- +# VERIFY_CLASSIFICATION_GRADING_CONSISTENCY + +Purpose: Checks semantic consistency between the predicted category and Grading-level. + +Observation: 输出自然语言一致性检查结论及可能的矛盾点。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["grading_consistency"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_WEIGH_EVIDENCE.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_WEIGH_EVIDENCE.md new file mode 100644 index 0000000..c9649ca --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_WEIGH_EVIDENCE.md @@ -0,0 +1,17 @@ +--- +name: WEIGH_EVIDENCE +dsl: evidence_reasoning +required_inputs: [evidence_set] +optional_inputs: [] +produces: [evidence_weights] +terminal: false +--- +# WEIGH_EVIDENCE + +Purpose: Assesses the relative reliability and importance of available evidence. + +Observation: 输出自然语言说明哪些证据更强、哪些只是辅助，以及原因；不输出未经依据的数值权重。 + +Use only supplied evidence and actual standards. Explain agreement, conflict, missing evidence and relative strength. Do not invent metadata, standard entries, categories or grades, or overwrite normative state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["evidence_weights"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_ACQUIRE_MISSING_METADATA.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_ACQUIRE_MISSING_METADATA.md new file mode 100644 index 0000000..a8256f1 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_ACQUIRE_MISSING_METADATA.md @@ -0,0 +1,17 @@ +--- +name: ACQUIRE_MISSING_METADATA +dsl: metadata_acquisition +required_inputs: [missing_metadata_type, current_metadata] +optional_inputs: [] +produces: [acquired_metadata] +terminal: false +--- +# ACQUIRE_MISSING_METADATA + +Purpose: Acquires a specific type of missing metadata requested by the current reasoning state. + +Observation: 输出尝试获取的元数据类型、实际获得的信息及其自然语言摘要；若无法获取，明确失败原因。 + +Use deterministic canonical Val access only. Verify private record identity. Never invent missing descriptions, types, values or locators; explicitly report unavailable data. Context belongs to the actual domain/database/table and has bounded, disclosed coverage. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["acquired_metadata"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_CHECK_METADATA_AVAILABILITY.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_CHECK_METADATA_AVAILABILITY.md new file mode 100644 index 0000000..99385a6 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_CHECK_METADATA_AVAILABILITY.md @@ -0,0 +1,19 @@ +--- +name: CHECK_METADATA_AVAILABILITY +dsl: metadata_acquisition +required_inputs: [current_record] +optional_inputs: [] +produces: [metadata_availability] +terminal: false +--- +# Role +Report which approved metadata fields are present and which are absent. + +# Evidence boundaries +Inspect only the supplied record. Never reconstruct hidden descriptions, values, labels, or ground truth. + +# Output +Return JSON with `summary` and `data.metadata_availability`, containing `available` and `missing` lists. + +# Failure +Return an explicit error when the record is unavailable or malformed. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_EXPAND_SELECTED_CONTEXT.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_EXPAND_SELECTED_CONTEXT.md new file mode 100644 index 0000000..d6d9105 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_EXPAND_SELECTED_CONTEXT.md @@ -0,0 +1,17 @@ +--- +name: EXPAND_SELECTED_CONTEXT +dsl: metadata_acquisition +required_inputs: [selected_context_fields] +optional_inputs: [] +produces: [context_fields] +terminal: false +--- +# EXPAND_SELECTED_CONTEXT + +Purpose: Organizes selected contextual metadata into a usable state for subsequent semantic interpretation. + +Observation: 输出自然语言上下文摘要，保持字段名与其可用描述/类型/值之间的对应关系。 + +Use deterministic canonical Val access only. Verify private record identity. Never invent missing descriptions, types, values or locators; explicitly report unavailable data. Context belongs to the actual domain/database/table and has bounded, disclosed coverage. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["context_fields"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_DATABASE_CONTEXT.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_DATABASE_CONTEXT.md new file mode 100644 index 0000000..c119048 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_DATABASE_CONTEXT.md @@ -0,0 +1,17 @@ +--- +name: GET_COMPLETE_DATABASE_CONTEXT +dsl: metadata_acquisition +required_inputs: [database_name] +optional_inputs: [] +produces: [database_context] +terminal: false +--- +# GET_COMPLETE_DATABASE_CONTEXT + +Purpose: Retrieves the available high-level metadata context of the current database. + +Observation: 输出数据库级自然语言背景摘要，不引入跨库信息。 + +Use deterministic canonical Val access only. Verify private record identity. Never invent missing descriptions, types, values or locators; explicitly report unavailable data. Context belongs to the actual domain/database/table and has bounded, disclosed coverage. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["database_context"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_TABLE_CONTEXT.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_TABLE_CONTEXT.md new file mode 100644 index 0000000..0838b6e --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_TABLE_CONTEXT.md @@ -0,0 +1,17 @@ +--- +name: GET_COMPLETE_TABLE_CONTEXT +dsl: metadata_acquisition +required_inputs: [table_name] +optional_inputs: [] +produces: [table_context] +terminal: false +--- +# GET_COMPLETE_TABLE_CONTEXT + +Purpose: Retrieves the complete available metadata context of the current table. + +Observation: 输出当前表完整可用元数据的自然语言摘要，并明确哪些属性缺失。 + +Use deterministic canonical Val access only. Verify private record identity. Never invent missing descriptions, types, values or locators; explicitly report unavailable data. Context belongs to the actual domain/database/table and has bounded, disclosed coverage. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["table_context"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_DESCRIPTIONS.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_DESCRIPTIONS.md new file mode 100644 index 0000000..c6afadf --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_DESCRIPTIONS.md @@ -0,0 +1,17 @@ +--- +name: GET_CONTEXT_FIELD_DESCRIPTIONS +dsl: metadata_acquisition +required_inputs: [table_name, context_fields] +optional_inputs: [] +produces: [context_field_descriptions] +terminal: false +--- +# GET_CONTEXT_FIELD_DESCRIPTIONS + +Purpose: Retrieves descriptions for selected contextual fields. + +Observation: 输出所选上下文字段的描述及自然语言摘要；缺失描述需明确标记。 + +Use deterministic canonical Val access only. Verify private record identity. Never invent missing descriptions, types, values or locators; explicitly report unavailable data. Context belongs to the actual domain/database/table and has bounded, disclosed coverage. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["context_field_descriptions"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_TYPES.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_TYPES.md new file mode 100644 index 0000000..1515c03 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_TYPES.md @@ -0,0 +1,17 @@ +--- +name: GET_CONTEXT_FIELD_TYPES +dsl: metadata_acquisition +required_inputs: [table_name, context_fields] +optional_inputs: [] +produces: [context_field_types] +terminal: false +--- +# GET_CONTEXT_FIELD_TYPES + +Purpose: Retrieves data types of selected contextual fields as supporting evidence. + +Observation: 输出上下文字段类型的自然语言摘要，说明类型能提供的有限辅助信息。 + +Use deterministic canonical Val access only. Verify private record identity. Never invent missing descriptions, types, values or locators; explicitly report unavailable data. Context belongs to the actual domain/database/table and has bounded, disclosed coverage. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["context_field_types"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_VALUES.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_VALUES.md new file mode 100644 index 0000000..3566bb5 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_VALUES.md @@ -0,0 +1,17 @@ +--- +name: GET_CONTEXT_FIELD_VALUES +dsl: metadata_acquisition +required_inputs: [table_name, context_fields] +optional_inputs: [] +produces: [context_field_values] +terminal: false +--- +# GET_CONTEXT_FIELD_VALUES + +Purpose: Retrieves available values of contextual fields and summarizes their semantic patterns. + +Observation: 输出样例值模式的自然语言摘要；避免输出不必要的敏感原始值，重点描述模式和语义。 + +Use deterministic canonical Val access only. Verify private record identity. Never invent missing descriptions, types, values or locators; explicitly report unavailable data. Context belongs to the actual domain/database/table and has bounded, disclosed coverage. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["context_field_values"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_DATABASE_DESCRIPTION.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_DATABASE_DESCRIPTION.md new file mode 100644 index 0000000..a394829 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_DATABASE_DESCRIPTION.md @@ -0,0 +1,17 @@ +--- +name: GET_DATABASE_DESCRIPTION +dsl: metadata_acquisition +required_inputs: [database_name] +optional_inputs: [] +produces: [database_description] +terminal: false +--- +# GET_DATABASE_DESCRIPTION + +Purpose: Retrieves and summarizes the description of the current database. + +Observation: 输出数据库描述及自然语言摘要；不存在时明确说明。 + +Use deterministic canonical Val access only. Verify private record identity. Never invent missing descriptions, types, values or locators; explicitly report unavailable data. Context belongs to the actual domain/database/table and has bounded, disclosed coverage. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["database_description"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_DESCRIPTION.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_DESCRIPTION.md new file mode 100644 index 0000000..2015e2c --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_DESCRIPTION.md @@ -0,0 +1,17 @@ +--- +name: GET_FIELD_DESCRIPTION +dsl: metadata_acquisition +required_inputs: [table_name, field_name] +optional_inputs: [] +produces: [field_description] +terminal: false +--- +# GET_FIELD_DESCRIPTION + +Purpose: Retrieves the description of the target field from the available database metadata. + +Observation: 输出获取到的字段描述及自然语言摘要；若不存在，应明确说明未获得该信息。 + +Use deterministic canonical Val access only. Verify private record identity. Never invent missing descriptions, types, values or locators; explicitly report unavailable data. Context belongs to the actual domain/database/table and has bounded, disclosed coverage. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["field_description"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_METADATA.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_METADATA.md new file mode 100644 index 0000000..f8504a3 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_METADATA.md @@ -0,0 +1,21 @@ +--- +name: GET_FIELD_METADATA +dsl: metadata_acquisition +required_inputs: [current_record] +optional_inputs: [] +produces: [field_metadata] +terminal: false +--- +# Role +Retrieve approved real metadata for this exact source record from the external +metadata store. This is a deterministic lookup, not a language-model guess. + +# Evidence boundary +Only database name, table name, field description, and table description may be +returned. Match source ID, domain, and field name. Never expose the category, +grading level, split, or other target data. The Agent sees these fields only +after this Skill has been called and its Observation appended to state. + +# Output +Return `field_metadata` with the available approved fields. Missing values +are omitted. The result is an Observation for subsequent Skill calls. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_DESCRIPTION.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_DESCRIPTION.md new file mode 100644 index 0000000..2bf0689 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_DESCRIPTION.md @@ -0,0 +1,17 @@ +--- +name: GET_TABLE_DESCRIPTION +dsl: metadata_acquisition +required_inputs: [table_name] +optional_inputs: [] +produces: [table_description] +terminal: false +--- +# GET_TABLE_DESCRIPTION + +Purpose: Retrieves and summarizes the description of the current table. + +Observation: 输出表描述及自然语言摘要；不存在时明确说明。 + +Use deterministic canonical Val access only. Verify private record identity. Never invent missing descriptions, types, values or locators; explicitly report unavailable data. Context belongs to the actual domain/database/table and has bounded, disclosed coverage. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["table_description"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_FIELDS.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_FIELDS.md new file mode 100644 index 0000000..c1779ba --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_FIELDS.md @@ -0,0 +1,17 @@ +--- +name: GET_TABLE_FIELDS +dsl: metadata_acquisition +required_inputs: [table_name] +optional_inputs: [] +produces: [table_fields] +terminal: false +--- +# GET_TABLE_FIELDS + +Purpose: Retrieves other fields from the current table to provide local schema context. + +Observation: 输出同表字段名称列表的自然语言摘要，突出可能与目标字段相关的字段；不臆造不存在字段。 + +Use deterministic canonical Val access only. Verify private record identity. Never invent missing descriptions, types, values or locators; explicitly report unavailable data. Context belongs to the actual domain/database/table and has bounded, disclosed coverage. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["table_fields"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_SELECT_CONTEXT_FIELDS.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_SELECT_CONTEXT_FIELDS.md new file mode 100644 index 0000000..ca0decc --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_SELECT_CONTEXT_FIELDS.md @@ -0,0 +1,17 @@ +--- +name: SELECT_CONTEXT_FIELDS +dsl: metadata_acquisition +required_inputs: [target_field, table_fields] +optional_inputs: [current_semantic_state] +produces: [selected_context_fields] +terminal: false +--- +# SELECT_CONTEXT_FIELDS + +Purpose: Selects the most informative contextual fields for the target field. + +Observation: 输出自然语言说明选择了哪些上下文字段、为什么它们更有信息价值，以及哪些字段被忽略。 + +Use deterministic canonical Val access only. Verify private record identity. Never invent missing descriptions, types, values or locators; explicitly report unavailable data. Context belongs to the actual domain/database/table and has bounded, disclosed coverage. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["selected_context_fields"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_DISAMBIGUATE_FIELD_SEMANTICS.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_DISAMBIGUATE_FIELD_SEMANTICS.md new file mode 100644 index 0000000..1fc67c7 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_DISAMBIGUATE_FIELD_SEMANTICS.md @@ -0,0 +1,19 @@ +--- +name: DISAMBIGUATE_FIELD_SEMANTICS +dsl: semantic_interpretation +required_inputs: [field_semantics] +optional_inputs: [domain, standard_candidates] +produces: [disambiguated_field_semantics] +terminal: false +--- +# Role +Resolve competing interpretations using only the current semantic state and supplied candidates. + +# Evidence boundaries +Preserve uncertainty when evidence is insufficient. Do not use hidden metadata, labels, or ground truth. + +# Output +Return JSON with `summary` and `data.disambiguated_field_semantics` as concise natural language. + +# Failure +Return an explicit error when no field semantics are available. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_DATABASE_ABBREVIATION.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_DATABASE_ABBREVIATION.md new file mode 100644 index 0000000..76258c8 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_DATABASE_ABBREVIATION.md @@ -0,0 +1,17 @@ +--- +name: EXPAND_DATABASE_ABBREVIATION +dsl: semantic_interpretation +required_inputs: [database_name] +optional_inputs: [domain, database_description] +produces: [database_abbreviation_semantics] +terminal: false +--- +# EXPAND_DATABASE_ABBREVIATION + +Purpose: Expands an abbreviated database or system name into its likely semantic meaning. + +Observation: 输出数据库/系统缩写的自然语言展开及可能的业务范围。 + +Interpret only supplied evidence. Separate direct evidence, auxiliary evidence, and uncertain hypotheses. Do not retrieve standards or predict category/grade. Never turn inferred semantics into raw metadata. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["database_abbreviation_semantics"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_FIELD_ABBREVIATION.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_FIELD_ABBREVIATION.md new file mode 100644 index 0000000..98f9d07 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_FIELD_ABBREVIATION.md @@ -0,0 +1,19 @@ +--- +name: EXPAND_FIELD_ABBREVIATION +dsl: semantic_interpretation +required_inputs: [field_name] +optional_inputs: [domain] +produces: [field_abbreviation_semantics] +terminal: false +--- +# Role (extractive-grounded-source-v1 revision) +Report source-grounded field meaning without speculative abbreviation expansion. This version explicitly replaces free natural-language semantics with a typed evidence object. + +# Evidence boundaries +The executor supplies versioned grounding_context containing only initial field_name and already acquired/provenanced field_description, table_name and table_description. This is declared runtime context, not an extra Planner argument and not a hidden metadata lookup. No description is available until actual acquisition. Do not infer expansions, entities, types, values, labels, or consistency from an opaque name. + +# Output +Return exactly summary and data.field_abbreviation_semantics. The latter is an object with exactly schema="grounded-source-v1", status="observed_description" or "unknown", field_name=exact context field_name, description=entire exact acquired field_description or null, source_consistency="not_adjudicated". summary equals status. With actual description, copy it and use observed_description; without description use unknown/null. No guesses, free-form claims, translations or conflict adjudication. Source consistency still requires independent review. + +# Failure +Return an explicit error when the field name is empty. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_TABLE_ABBREVIATION.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_TABLE_ABBREVIATION.md new file mode 100644 index 0000000..1340fa2 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_TABLE_ABBREVIATION.md @@ -0,0 +1,17 @@ +--- +name: EXPAND_TABLE_ABBREVIATION +dsl: semantic_interpretation +required_inputs: [table_name] +optional_inputs: [domain, table_description] +produces: [table_abbreviation_semantics] +terminal: false +--- +# EXPAND_TABLE_ABBREVIATION + +Purpose: Expands an abbreviated table name into plausible business meanings. + +Observation: 输出表名缩写的候选展开，以及它可能表示的数据主题或业务对象。 + +Interpret only supplied evidence. Separate direct evidence, auxiliary evidence, and uncertain hypotheses. Do not retrieve standards or predict category/grade. Never turn inferred semantics into raw metadata. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["table_abbreviation_semantics"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_CONTEXT_FIELDS.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_CONTEXT_FIELDS.md new file mode 100644 index 0000000..c5a536a --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_CONTEXT_FIELDS.md @@ -0,0 +1,17 @@ +--- +name: INTERPRET_CONTEXT_FIELDS +dsl: semantic_interpretation +required_inputs: [context_fields] +optional_inputs: [domain, table_name] +produces: [context_field_semantics] +terminal: false +--- +# INTERPRET_CONTEXT_FIELDS + +Purpose: Interprets neighboring fields and summarizes the business context jointly implied by them. + +Observation: 输出各上下文字段的简要语义及它们共同指向的业务场景；不直接替目标字段做最终分类。 + +Interpret only supplied evidence. Separate direct evidence, auxiliary evidence, and uncertain hypotheses. Do not retrieve standards or predict category/grade. Never turn inferred semantics into raw metadata. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["context_field_semantics"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_DESCRIPTION.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_DESCRIPTION.md new file mode 100644 index 0000000..46c8b95 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_DESCRIPTION.md @@ -0,0 +1,17 @@ +--- +name: INTERPRET_DATABASE_DESCRIPTION +dsl: semantic_interpretation +required_inputs: [database_description] +optional_inputs: [database_name, domain] +produces: [database_description_semantics] +terminal: false +--- +# INTERPRET_DATABASE_DESCRIPTION + +Purpose: Interprets the semantic scope expressed by the database description. + +Observation: 输出数据库描述的自然语言摘要，并说明其对字段判断的约束。 + +Interpret only supplied evidence. Separate direct evidence, auxiliary evidence, and uncertain hypotheses. Do not retrieve standards or predict category/grade. Never turn inferred semantics into raw metadata. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["database_description_semantics"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_NAME.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_NAME.md new file mode 100644 index 0000000..a1fdeb0 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_NAME.md @@ -0,0 +1,17 @@ +--- +name: INTERPRET_DATABASE_NAME +dsl: semantic_interpretation +required_inputs: [database_name] +optional_inputs: [domain, database_description] +produces: [database_semantics] +terminal: false +--- +# INTERPRET_DATABASE_NAME + +Purpose: Interprets the domain and semantic scope implied by the database name. + +Observation: 输出数据库级自然语言语义，作为当前字段的高层背景。 + +Interpret only supplied evidence. Separate direct evidence, auxiliary evidence, and uncertain hypotheses. Do not retrieve standards or predict category/grade. Never turn inferred semantics into raw metadata. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["database_semantics"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DOMAIN.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DOMAIN.md new file mode 100644 index 0000000..23b3e93 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DOMAIN.md @@ -0,0 +1,19 @@ +--- +name: INTERPRET_DOMAIN +dsl: semantic_interpretation +required_inputs: [domain] +optional_inputs: [] +produces: [domain_semantics] +terminal: false +--- +# Role +Describe the semantic scope implied by the supplied domain. + +# Evidence boundaries +Use only the domain string. Do not replace it with a dataset name or infer hidden metadata or ground truth. + +# Output +Return JSON with `summary` and `data.domain_semantics` as concise natural language. + +# Failure +Return an explicit error when the domain is empty. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_DESCRIPTION.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_DESCRIPTION.md new file mode 100644 index 0000000..8a4c44f --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_DESCRIPTION.md @@ -0,0 +1,19 @@ +--- +name: INTERPRET_FIELD_DESCRIPTION +dsl: semantic_interpretation +required_inputs: [field_description] +optional_inputs: [field_name, domain] +produces: [field_description_semantics] +terminal: false +--- +# Role (extractive-grounded-source-v1 revision) +Report the observed description with exact source support rather than generating unverified interpretations or claiming that an opaque name contradicts it. + +# Evidence boundary +The executor supplies versioned grounding_context containing initial field_name and already acquired/provenanced field_description, table_name and table_description. This declared runtime context is not a Planner argument or hidden lookup. The required field_description must be the real acquired value. A source description is an observation, not proof of semantic truth or agreement with the name. Do not guess values, types, labels or conflict. + +# Output +Return exactly summary and data.field_description_semantics. The latter is an object with exactly schema="grounded-source-v1", status="observed_description", field_name=exact context field_name, description=entire exact acquired field_description, source_consistency="not_adjudicated". summary equals status. No extra fields, interpretations, translation or conflict adjudication. If the required description is absent, original input validation fails rather than fabricating a source. + +# Failure +Do not infer from an absent or empty field description. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_NAME.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_NAME.md new file mode 100644 index 0000000..8b64139 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_NAME.md @@ -0,0 +1,19 @@ +--- +name: INTERPRET_FIELD_NAME +dsl: semantic_interpretation +required_inputs: [field_name] +optional_inputs: [domain, field_abbreviation_semantics] +produces: [field_semantics] +terminal: false +--- +# Role (extractive-grounded-source-v1 revision) +Report source-grounded field meaning without speculative name interpretation. This revision replaces free natural-language semantics with a typed evidence object; it does not claim a source description proves name-description consistency. + +# Evidence boundaries +The executor supplies versioned grounding_context containing initial field_name and only already acquired/provenanced field_description, table_name and table_description. This declared runtime context is not an extra Planner argument or hidden lookup. No description is available before actual acquisition. An upstream model expansion cannot establish a verified business meaning. Do not infer missing meanings, values, types or labels. + +# Output +Return exactly summary and data.field_semantics. The latter is an object with exactly schema="grounded-source-v1", status="observed_description" or "unknown", field_name=exact context field_name, description=entire exact acquired field_description or null, source_consistency="not_adjudicated". summary equals status. With a real acquired description copy it/use observed_description; otherwise unknown/null. No guesses, free prose, translations, abbreviations or adjudicated conflict/consistency claims. Actual source consistency requires independent review. + +# Failure +Return an explicit error when the field name is empty. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_TYPE.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_TYPE.md new file mode 100644 index 0000000..f7faeb4 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_TYPE.md @@ -0,0 +1,17 @@ +--- +name: INTERPRET_FIELD_TYPE +dsl: semantic_interpretation +required_inputs: [field_type] +optional_inputs: [field_name, field_description] +produces: [field_type_semantics] +terminal: false +--- +# INTERPRET_FIELD_TYPE + +Purpose: Uses the field data type as supporting semantic evidence. + +Observation: 输出字段类型可提供的约束性自然语言判断；不得仅凭类型强行确定具体类别。 + +Interpret only supplied evidence. Separate direct evidence, auxiliary evidence, and uncertain hypotheses. Do not retrieve standards or predict category/grade. Never turn inferred semantics into raw metadata. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["field_type_semantics"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_VALUE.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_VALUE.md new file mode 100644 index 0000000..8fd4747 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_VALUE.md @@ -0,0 +1,17 @@ +--- +name: INTERPRET_FIELD_VALUE +dsl: semantic_interpretation +required_inputs: [value] +optional_inputs: [field_name, field_type, domain] +produces: [value_semantics] +terminal: false +--- +# INTERPRET_FIELD_VALUE + +Purpose: Infers possible semantics from the observed value pattern and content. + +Observation: 输出值模式的自然语言描述、可能对应的数据含义，以及这种证据的可靠性边界。 + +Interpret only supplied evidence. Separate direct evidence, auxiliary evidence, and uncertain hypotheses. Do not retrieve standards or predict category/grade. Never turn inferred semantics into raw metadata. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["value_semantics"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_DESCRIPTION.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_DESCRIPTION.md new file mode 100644 index 0000000..31d9fd8 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_DESCRIPTION.md @@ -0,0 +1,21 @@ +--- +name: INTERPRET_TABLE_DESCRIPTION +dsl: semantic_interpretation +required_inputs: [table_description] +optional_inputs: [table_name, domain] +produces: [table_description_semantics] +terminal: false +--- +# Role +Explain the business context of an observed table description, without assigning a category. + +# Evidence boundary +Use only the supplied real description and optional supplied context. The table's business +topic is context for the target field, not proof that all fields have one category. +Never invent values, labels or grading levels. + +# Output +Return JSON with a short `summary` and `data.table_description_semantics` as concise text. + +# Failure +Do not infer from an absent or empty table description. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_NAME.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_NAME.md new file mode 100644 index 0000000..799e31b --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_NAME.md @@ -0,0 +1,17 @@ +--- +name: INTERPRET_TABLE_NAME +dsl: semantic_interpretation +required_inputs: [table_name] +optional_inputs: [domain, table_description] +produces: [table_semantics] +terminal: false +--- +# INTERPRET_TABLE_NAME + +Purpose: Interprets the business semantics represented by a table name. + +Observation: 输出表级自然语言语义，强调该表可能描述什么业务数据。 + +Interpret only supplied evidence. Separate direct evidence, auxiliary evidence, and uncertain hypotheses. Do not retrieve standards or predict category/grade. Never turn inferred semantics into raw metadata. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["table_semantics"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_SYNTHESIZE_METADATA_SEMANTICS.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_SYNTHESIZE_METADATA_SEMANTICS.md new file mode 100644 index 0000000..18c218c --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_SYNTHESIZE_METADATA_SEMANTICS.md @@ -0,0 +1,17 @@ +--- +name: SYNTHESIZE_METADATA_SEMANTICS +dsl: semantic_interpretation +required_inputs: [field_semantics] +optional_inputs: [domain_semantics, table_semantics, database_semantics, context_field_semantics, field_description_semantics, field_type_semantics, value_semantics] +produces: [metadata_semantics] +terminal: false +--- +# SYNTHESIZE_METADATA_SEMANTICS + +Purpose: Synthesizes all available metadata interpretations into a unified natural-language semantic state. + +Observation: 输出一段紧凑但信息充分的自然语言 semantic state，区分强证据、辅助证据和仍存在的歧义。 + +Interpret only supplied evidence. Separate direct evidence, auxiliary evidence, and uncertain hypotheses. Do not retrieve standards or predict category/grade. Never turn inferred semantics into raw metadata. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["metadata_semantics"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_CONTEXT_TO_STANDARD.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_CONTEXT_TO_STANDARD.md new file mode 100644 index 0000000..dc5dae5 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_CONTEXT_TO_STANDARD.md @@ -0,0 +1,17 @@ +--- +name: ALIGN_CONTEXT_TO_STANDARD +dsl: standard_alignment +required_inputs: [context_semantics, standard_candidates] +optional_inputs: [] +produces: [align_context_to_standard] +terminal: false +--- +# ALIGN_CONTEXT_TO_STANDARD + +Purpose: Aligns contextual semantic evidence with candidate standard categories. + +Observation: 输出上下文语义对候选标准类别的支持、反对或弱相关关系。 + +Use only the actual frozen standard candidates and their disclosed descriptions. Explain support, contradictions, auxiliary context and unresolved distinctions. Never invent standard entries or overwrite canonical candidate/category/grade state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["align_context_to_standard"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DATABASE_TO_STANDARD.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DATABASE_TO_STANDARD.md new file mode 100644 index 0000000..c303142 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DATABASE_TO_STANDARD.md @@ -0,0 +1,17 @@ +--- +name: ALIGN_DATABASE_TO_STANDARD +dsl: standard_alignment +required_inputs: [database_semantics, standard_candidates] +optional_inputs: [] +produces: [align_database_to_standard] +terminal: false +--- +# ALIGN_DATABASE_TO_STANDARD + +Purpose: Aligns database-level semantics with candidate standard categories. + +Observation: Explain how database context supports, contradicts or weakly matches each actual standard candidate; context remains auxiliary to target-field evidence. + +Use only the actual frozen standard candidates and their disclosed descriptions. Explain support, contradictions, auxiliary context and unresolved distinctions. Never invent standard entries or overwrite canonical candidate/category/grade state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["align_database_to_standard"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DESCRIPTION_TO_STANDARD.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DESCRIPTION_TO_STANDARD.md new file mode 100644 index 0000000..cda05f1 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DESCRIPTION_TO_STANDARD.md @@ -0,0 +1,17 @@ +--- +name: ALIGN_DESCRIPTION_TO_STANDARD +dsl: standard_alignment +required_inputs: [field_description_semantics, standard_candidates] +optional_inputs: [] +produces: [align_description_to_standard] +terminal: false +--- +# ALIGN_DESCRIPTION_TO_STANDARD + +Purpose: Aligns field-description semantics with standard category descriptions. + +Observation: 输出字段描述与各标准候选的语义匹配理由，显式描述通常可作为较强证据。 + +Use only the actual frozen standard candidates and their disclosed descriptions. Explain support, contradictions, auxiliary context and unresolved distinctions. Never invent standard entries or overwrite canonical candidate/category/grade state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["align_description_to_standard"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_FIELD_TO_STANDARD.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_FIELD_TO_STANDARD.md new file mode 100644 index 0000000..0dde182 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_FIELD_TO_STANDARD.md @@ -0,0 +1,17 @@ +--- +name: ALIGN_FIELD_TO_STANDARD +dsl: standard_alignment +required_inputs: [field_semantics, standard_candidates] +optional_inputs: [] +produces: [field_standard_alignment] +terminal: false +--- +# ALIGN_FIELD_TO_STANDARD + +Purpose: Aligns the target-field semantics with candidate standard categories. + +Observation: 输出字段语义与各候选标准描述的匹配/不匹配理由，以及当前更优候选。 + +Use only the actual frozen standard candidates and their disclosed descriptions. Explain support, contradictions, auxiliary context and unresolved distinctions. Never invent standard entries or overwrite canonical candidate/category/grade state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["field_standard_alignment"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_TABLE_TO_STANDARD.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_TABLE_TO_STANDARD.md new file mode 100644 index 0000000..f20ed54 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_TABLE_TO_STANDARD.md @@ -0,0 +1,17 @@ +--- +name: ALIGN_TABLE_TO_STANDARD +dsl: standard_alignment +required_inputs: [table_semantics, standard_candidates] +optional_inputs: [] +produces: [align_table_to_standard] +terminal: false +--- +# ALIGN_TABLE_TO_STANDARD + +Purpose: Aligns table-level semantics with candidate standard categories. + +Observation: Explain how table context supports, contradicts or weakly matches each actual standard candidate; context remains auxiliary to target-field evidence. + +Use only the actual frozen standard candidates and their disclosed descriptions. Explain support, contradictions, auxiliary context and unresolved distinctions. Never invent standard entries or overwrite canonical candidate/category/grade state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["align_table_to_standard"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_VALUE_TO_STANDARD.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_VALUE_TO_STANDARD.md new file mode 100644 index 0000000..43a354f --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_VALUE_TO_STANDARD.md @@ -0,0 +1,17 @@ +--- +name: ALIGN_VALUE_TO_STANDARD +dsl: standard_alignment +required_inputs: [value_semantics, standard_candidates] +optional_inputs: [] +produces: [align_value_to_standard] +terminal: false +--- +# ALIGN_VALUE_TO_STANDARD + +Purpose: Aligns value-derived semantics with candidate standard categories. + +Observation: 输出字段值模式对标准候选的支持程度；强调值证据通常是辅助证据。 + +Use only the actual frozen standard candidates and their disclosed descriptions. Explain support, contradictions, auxiliary context and unresolved distinctions. Never invent standard entries or overwrite canonical candidate/category/grade state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["align_value_to_standard"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_COMPARE_STANDARD_CANDIDATES.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_COMPARE_STANDARD_CANDIDATES.md new file mode 100644 index 0000000..9443c8d --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_COMPARE_STANDARD_CANDIDATES.md @@ -0,0 +1,19 @@ +--- +name: COMPARE_STANDARD_CANDIDATES +dsl: standard_alignment +required_inputs: [standard_candidates, semantic_evidence] +optional_inputs: [] +produces: [standard_comparison, aligned_standard_entry] +terminal: false +--- +# Role +Compare retrieved candidates against the current semantic evidence and identify the best-supported entry. + +# Evidence boundaries +Compare only supplied candidates. Do not add categories, hidden metadata, labels, or ground truth. + +# Output +Return exactly one JSON object with exactly `decision`, `option_id`, and `reason`. For a supported candidate use `{"decision":"match","option_id":"C001","reason":"brief"}`, choosing an exact short `option_id` from the actual supplied `standard_candidate_details`. When no supplied candidate is supported use `{"decision":"no_match","option_id":null,"reason":"brief"}`. `reason` must be a nonempty brief string of at most 240 characters. Do not emit Markdown, duplicate fields, a summary/data wrapper, canonical category IDs, or full standard objects. The executor binds a successful short ID to the frozen standard and publishes the internal `standard_comparison` and `aligned_standard_entry`. + +# Failure +Absent inputs fail validation before generation. `no_match` is a model judgment and fails without publishing alignment or classification. Invalid format, actual output truncation, and backend generation errors are reported separately; do not invent a match. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_REFINE_STANDARD_QUERY.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_REFINE_STANDARD_QUERY.md new file mode 100644 index 0000000..29433fb --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_REFINE_STANDARD_QUERY.md @@ -0,0 +1,17 @@ +--- +name: REFINE_STANDARD_QUERY +dsl: standard_alignment +required_inputs: [previous_standard_results, current_semantic_state] +optional_inputs: [] +produces: [refined_semantic_query] +terminal: false +--- +# REFINE_STANDARD_QUERY + +Purpose: Refines the standard-search query using previous retrieval results and the current semantic state. + +Observation: 输出一段更聚焦的自然语言查询表达，并说明为何这样改写能区分当前候选。 + +Use only the actual frozen standard candidates and their disclosed descriptions. Explain support, contradictions, auxiliary context and unresolved distinctions. Never invent standard entries or overwrite canonical candidate/category/grade state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["refined_semantic_query"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_RESEARCH_STANDARD.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_RESEARCH_STANDARD.md new file mode 100644 index 0000000..421cda4 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_RESEARCH_STANDARD.md @@ -0,0 +1,17 @@ +--- +name: RESEARCH_STANDARD +dsl: standard_alignment +required_inputs: [refined_semantic_query] +optional_inputs: [] +produces: [standard_candidates] +terminal: false +--- +# RESEARCH_STANDARD + +Purpose: Performs a subsequent standard search after semantic refinement. + +Observation: 输出新一轮标准候选、与上一轮相比新增或变化的证据，以及是否更有区分度。 + +Use only the actual frozen standard candidates and their disclosed descriptions. Explain support, contradictions, auxiliary context and unresolved distinctions. Never invent standard entries or overwrite canonical candidate/category/grade state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["standard_candidates"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD.md new file mode 100644 index 0000000..79a5d12 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD.md @@ -0,0 +1,19 @@ +--- +name: SEARCH_STANDARD +dsl: standard_alignment +required_inputs: [semantic_query] +optional_inputs: [] +produces: [standard_candidates] +terminal: false +--- +# Role +Retrieve ranked category candidates from the frozen standard registry. + +# Evidence boundaries +Use only the supplied query and registry. Never consult evaluation labels or ground truth. + +# Output +Return JSON with `summary` and `data.standard_candidates` as a ranked list of registry entries. + +# Failure +Return an explicit error when the query is empty or the registry cannot be searched. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_CONTEXT.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_CONTEXT.md new file mode 100644 index 0000000..20055df --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_CONTEXT.md @@ -0,0 +1,17 @@ +--- +name: SEARCH_STANDARD_BY_CONTEXT +dsl: standard_alignment +required_inputs: [context_semantics] +optional_inputs: [] +produces: [standard_candidates] +terminal: false +--- +# SEARCH_STANDARD_BY_CONTEXT + +Purpose: Retrieves standard categories using contextual-field semantics. + +Observation: 输出上下文语义最相关的标准候选，并说明其对目标字段的辅助意义。 + +Use only the actual frozen standard candidates and their disclosed descriptions. Explain support, contradictions, auxiliary context and unresolved distinctions. Never invent standard entries or overwrite canonical candidate/category/grade state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["standard_candidates"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_DATABASE.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_DATABASE.md new file mode 100644 index 0000000..97b3bff --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_DATABASE.md @@ -0,0 +1,17 @@ +--- +name: SEARCH_STANDARD_BY_DATABASE +dsl: standard_alignment +required_inputs: [database_semantics] +optional_inputs: [] +produces: [standard_candidates] +terminal: false +--- +# SEARCH_STANDARD_BY_DATABASE + +Purpose: Uses database-level semantics to retrieve or constrain candidate standard categories. + +Observation: 输出数据库级语义可能支持的标准类别范围和依据。 + +Use only the actual frozen standard candidates and their disclosed descriptions. Explain support, contradictions, auxiliary context and unresolved distinctions. Never invent standard entries or overwrite canonical candidate/category/grade state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["standard_candidates"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_FIELD.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_FIELD.md new file mode 100644 index 0000000..5af62e3 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_FIELD.md @@ -0,0 +1,17 @@ +--- +name: SEARCH_STANDARD_BY_FIELD +dsl: standard_alignment +required_inputs: [field_semantics] +optional_inputs: [] +produces: [standard_candidates] +terminal: false +--- +# SEARCH_STANDARD_BY_FIELD + +Purpose: Retrieves standard categories relevant to the target-field semantics. + +Observation: 输出字段语义最相关的标准候选及匹配依据。 + +Use only the actual frozen standard candidates and their disclosed descriptions. Explain support, contradictions, auxiliary context and unresolved distinctions. Never invent standard entries or overwrite canonical candidate/category/grade state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["standard_candidates"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_TABLE.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_TABLE.md new file mode 100644 index 0000000..2ffdd24 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_TABLE.md @@ -0,0 +1,17 @@ +--- +name: SEARCH_STANDARD_BY_TABLE +dsl: standard_alignment +required_inputs: [table_semantics] +optional_inputs: [] +produces: [standard_candidates] +terminal: false +--- +# SEARCH_STANDARD_BY_TABLE + +Purpose: Retrieves candidate standard categories using table-level semantics. + +Observation: 输出表级语义相关的标准候选及其描述摘要。 + +Use only the actual frozen standard candidates and their disclosed descriptions. Explain support, contradictions, auxiliary context and unresolved distinctions. Never invent standard entries or overwrite canonical candidate/category/grade state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["standard_candidates"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SELECT_CATEGORY_TOPK.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SELECT_CATEGORY_TOPK.md new file mode 100644 index 0000000..99684b0 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SELECT_CATEGORY_TOPK.md @@ -0,0 +1,16 @@ +--- +name: SELECT_CATEGORY_TOPK +dsl: standard_alignment +required_inputs: [semantic_query] +optional_inputs: [] +produces: [standard_candidates] +terminal: false +--- +# Role +Select ranked Top-K short option IDs from the full frozen label catalog. The executor maps each selected option back to its canonical Registry category ID. + +# Evidence boundary +This stage receives deterministic short option IDs and full category-path labels only, never canonical long IDs, category descriptions, grading levels, or evaluation labels. Do not invent option IDs. + +# Output +Return exactly K distinct short option IDs from the supplied catalog. The executor validates them and normalizes them into label-only standard_candidates carrying canonical category IDs. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SUMMARIZE_STANDARD_EVIDENCE.md b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SUMMARIZE_STANDARD_EVIDENCE.md new file mode 100644 index 0000000..3318075 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SUMMARIZE_STANDARD_EVIDENCE.md @@ -0,0 +1,17 @@ +--- +name: SUMMARIZE_STANDARD_EVIDENCE +dsl: standard_alignment +required_inputs: [aligned_standard_candidates] +optional_inputs: [] +produces: [standard_evidence_summary] +terminal: false +--- +# SUMMARIZE_STANDARD_EVIDENCE + +Purpose: Summarizes multiple standard-search and alignment results into a unified standard-evidence state. + +Observation: 输出自然语言总结：当前标准证据主要支持哪些类别、哪些类别已被排除、还存在哪些冲突。 + +Use only the actual frozen standard candidates and their disclosed descriptions. Explain support, contradictions, auxiliary context and unresolved distinctions. Never invent standard entries or overwrite canonical candidate/category/grade state. + +Return exactly {"summary": "nonempty natural-language observation", "data": {...}}. Data keys must be exactly ["standard_evidence_summary"]. Semantic and reasoning outputs are nonempty strings; metadata/context/candidate outputs retain their typed validated structures. No extra keys. diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/skill_reference.json b/research/vehicle_program_v1/code/upstream/runtime/vendor/skill_reference.json new file mode 100644 index 0000000..dac74e8 --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/skill_reference.json @@ -0,0 +1,380 @@ +{ + "COMPARE_STANDARD_CANDIDATES": { + "required": "standard_candidates, semantic_evidence", + "output": "输出候选类别的排序、关键支持/反对依据及仍需区分的歧义点。", + "optional": "无", + "purpose": "Compares competing standard candidates based on accumulated semantic evidence." + }, + "GET_FIELD_DESCRIPTION": { + "required": "table_name, field_name", + "output": "输出获取到的字段描述及自然语言摘要；若不存在，应明确说明未获得该信息。", + "optional": "无", + "purpose": "Retrieves the description of the target field from the available database metadata." + }, + "GET_TABLE_DESCRIPTION": { + "required": "table_name", + "output": "输出表描述及自然语言摘要；不存在时明确说明。", + "optional": "无", + "purpose": "Retrieves and summarizes the description of the current table." + }, + "GET_DATABASE_DESCRIPTION": { + "required": "database_name", + "output": "输出数据库描述及自然语言摘要；不存在时明确说明。", + "optional": "无", + "purpose": "Retrieves and summarizes the description of the current database." + }, + "INTERPRET_FIELD_DESCRIPTION": { + "required": "field_description", + "output": "输出字段描述中最关键的数据对象、属性或业务语义，以及其与 field_name 的一致或冲突之处。", + "optional": "field_name, domain", + "purpose": "Extracts and interprets explicit semantic information from the field description." + }, + "SUMMARIZE_STANDARD_EVIDENCE": { + "required": "aligned_standard_candidates", + "output": "输出自然语言总结：当前标准证据主要支持哪些类别、哪些类别已被排除、还存在哪些冲突。", + "optional": "无", + "purpose": "Summarizes multiple standard-search and alignment results into a unified standard-evidence state." + }, + "GET_TABLE_FIELDS": { + "required": "table_name", + "output": "输出同表字段名称列表的自然语言摘要，突出可能与目标字段相关的字段；不臆造不存在字段。", + "optional": "无", + "purpose": "Retrieves other fields from the current table to provide local schema context." + }, + "FUSE_CONTEXT_EVIDENCE": { + "required": "context_field_evidence", + "output": "输出自然语言总结上下文字段共同指向的业务场景及其对目标字段的支持。", + "optional": "无", + "purpose": "Integrates semantic evidence from selected contextual fields." + }, + "ALIGN_DESCRIPTION_TO_STANDARD": { + "required": "field_description_semantics, standard_candidates", + "output": "输出字段描述与各标准候选的语义匹配理由，显式描述通常可作为较强证据。", + "optional": "无", + "purpose": "Aligns field-description semantics with standard category descriptions." + }, + "GET_COMPLETE_TABLE_CONTEXT": { + "required": "table_name", + "output": "输出当前表完整可用元数据的自然语言摘要，并明确哪些属性缺失。", + "optional": "无", + "purpose": "Retrieves the complete available metadata context of the current table." + }, + "RESEARCH_STANDARD": { + "required": "refined_semantic_query", + "output": "输出新一轮标准候选、与上一轮相比新增或变化的证据，以及是否更有区分度。", + "optional": "无", + "purpose": "Performs a subsequent standard search after semantic refinement." + }, + "SEARCH_STANDARD_BY_TABLE": { + "required": "table_semantics", + "output": "输出表级语义相关的标准候选及其描述摘要。", + "optional": "无", + "purpose": "Retrieves candidate standard categories using table-level semantics." + }, + "EXPAND_TABLE_ABBREVIATION": { + "required": "table_name", + "output": "输出表名缩写的候选展开，以及它可能表示的数据主题或业务对象。", + "optional": "domain, table_description", + "purpose": "Expands an abbreviated table name into plausible business meanings." + }, + "DISAMBIGUATE_FIELD_SEMANTICS": { + "required": "candidate_field_semantics, available_semantic_context", + "output": "输出候选解释之间的比较、保留/排除依据，以及当前更可信的字段语义。", + "optional": "无", + "purpose": "Disambiguates competing interpretations of the target field using available semantic context." + }, + "IDENTIFY_MISSING_EVIDENCE": { + "required": "current_evidence, candidate_categories", + "output": "输出自然语言说明缺少哪类证据（字段描述、表字段、上下文字段语义、表描述、数据库描述、标准描述等）及其必要性。", + "optional": "无", + "purpose": "Identifies the most important missing evidence needed to discriminate among candidate categories." + }, + "EXPAND_SELECTED_CONTEXT": { + "required": "selected_context_fields", + "output": "输出自然语言上下文摘要，保持字段名与其可用描述/类型/值之间的对应关系。", + "optional": "无", + "purpose": "Organizes selected contextual metadata into a usable state for subsequent semantic interpretation." + }, + "GET_CONTEXT_FIELD_VALUES": { + "required": "table_name, context_fields", + "output": "输出样例值模式的自然语言摘要；避免输出不必要的敏感原始值，重点描述模式和语义。", + "optional": "无", + "purpose": "Retrieves available values of contextual fields and summarizes their semantic patterns." + }, + "ACQUIRE_MISSING_METADATA": { + "required": "missing_metadata_type, current_metadata", + "output": "输出尝试获取的元数据类型、实际获得的信息及其自然语言摘要；若无法获取，明确失败原因。", + "optional": "无", + "purpose": "Acquires a specific type of missing metadata requested by the current reasoning state." + }, + "INTERPRET_DATABASE_DESCRIPTION": { + "required": "database_description", + "output": "输出数据库描述的自然语言摘要，并说明其对字段判断的约束。", + "optional": "database_name, domain", + "purpose": "Interprets the semantic scope expressed by the database description." + }, + "VERIFY_CATEGORY_WITH_STANDARD": { + "required": "candidate_category, standard_evidence, metadata_evidence", + "output": "输出自然语言验证结论，明确标准描述与元数据语义是否一致。", + "optional": "无", + "purpose": "Verifies whether a candidate category is jointly supported by metadata and standard evidence." + }, + "SEARCH_STANDARD_BY_FIELD": { + "required": "field_semantics", + "output": "输出字段语义最相关的标准候选及匹配依据。", + "optional": "无", + "purpose": "Retrieves standard categories relevant to the target-field semantics." + }, + "GET_CONTEXT_FIELD_TYPES": { + "required": "table_name, context_fields", + "output": "输出上下文字段类型的自然语言摘要，说明类型能提供的有限辅助信息。", + "optional": "无", + "purpose": "Retrieves data types of selected contextual fields as supporting evidence." + }, + "CHECK_METADATA_AVAILABILITY": { + "required": "current_record", + "output": "输出自然语言可用性摘要，列出当前可直接使用和缺失的元数据类型，并指出可行的下一步信息获取方向。", + "optional": "无", + "purpose": "Identifies which metadata attributes are currently available or missing." + }, + "RESOLVE_METADATA_CONFLICT": { + "required": "conflicting_metadata_evidence", + "output": "输出自然语言冲突消解结论和依据。", + "optional": "无", + "purpose": "Resolves conflicting semantic evidence across metadata granularities." + }, + "VERIFY_CLASSIFICATION_GRADING_CONSISTENCY": { + "required": "category_candidate, grading_level_candidate, current_evidence", + "output": "输出自然语言一致性检查结论及可能的矛盾点。", + "optional": "无", + "purpose": "Checks semantic consistency between the predicted category and Grading-level." + }, + "RESOLVE_FIELD_CONTEXT_CONFLICT": { + "required": "field_evidence, context_evidence", + "output": "输出冲突点、比较依据、保留的解释以及仍需补充的信息。", + "optional": "无", + "purpose": "Resolves conflicts between target-field evidence and contextual-field evidence." + }, + "FINALIZE_CLASSIFICATION": { + "required": "final_evidence_state", + "output": "最终输出结构化结果：category 与 grading_level；可附一段简短 final_reasoning，但不再继续获取新证据。", + "optional": "无", + "purpose": "Terminates the reasoning program and produces the final classification and grading prediction." + }, + "GET_CONTEXT_FIELD_DESCRIPTIONS": { + "required": "table_name, context_fields", + "output": "输出所选上下文字段的描述及自然语言摘要；缺失描述需明确标记。", + "optional": "无", + "purpose": "Retrieves descriptions for selected contextual fields." + }, + "SELECT_CONTEXT_FIELDS": { + "required": "target_field, table_fields", + "output": "输出自然语言说明选择了哪些上下文字段、为什么它们更有信息价值，以及哪些字段被忽略。", + "optional": "current_semantic_state", + "purpose": "Selects the most informative contextual fields for the target field." + }, + "FUSE_FIELD_EVIDENCE": { + "required": "available_field_evidence", + "output": "输出字段级统一自然语言证据状态，区分直接证据、辅助证据和歧义。", + "optional": "无", + "purpose": "Integrates all available evidence directly associated with the target field." + }, + "INTERPRET_TABLE_DESCRIPTION": { + "required": "table_description", + "output": "输出表描述的自然语言摘要，并说明其对目标字段语义理解的辅助作用。", + "optional": "table_name, domain", + "purpose": "Extracts and summarizes the semantic content of a table description." + }, + "FUSE_TABLE_EVIDENCE": { + "required": "available_table_evidence", + "output": "输出统一表级自然语言证据状态。", + "optional": "无", + "purpose": "Integrates all available table-level semantic evidence." + }, + "INFER_GRADING_LEVEL": { + "required": "aligned_category, aligned_standard_entry", + "output": "输出自然语言说明最终 category 对齐到哪个标准条目，以及该条目明确给出的 grading_level。", + "optional": "无", + "purpose": "Determines the Grading-level from the grading_level associated with the aligned standard category." + }, + "INTERPRET_FIELD_NAME": { + "required": "field_name", + "output": "输出对字段名整体语义的自然语言解释，包括主要候选含义与简要依据。", + "optional": "domain", + "purpose": "Interprets the overall semantic meaning of a field name." + }, + "DECIDE_NEXT_EVIDENCE_SOURCE": { + "required": "current_evidence_state, available_metadata", + "output": "输出自然语言下一步建议，例如解释字段、获取同表字段、解释表/库、查询/重查标准或停止；不得调用超出可用信息范围的数据源。", + "optional": "无", + "purpose": "Determines which available evidence source should be explored next." + }, + "ALIGN_CONTEXT_TO_STANDARD": { + "required": "context_semantics, standard_candidates", + "output": "输出上下文语义对候选标准类别的支持、反对或弱相关关系。", + "optional": "无", + "purpose": "Aligns contextual semantic evidence with candidate standard categories." + }, + "WEIGH_EVIDENCE": { + "required": "evidence_set", + "output": "输出自然语言说明哪些证据更强、哪些只是辅助，以及原因；不输出未经依据的数值权重。", + "optional": "无", + "purpose": "Assesses the relative reliability and importance of available evidence." + }, + "FUSE_METADATA_EVIDENCE": { + "required": "available_metadata_evidence", + "output": "输出统一 metadata evidence state，并指出不同粒度证据的一致与冲突。", + "optional": "无", + "purpose": "Integrates multi-granularity metadata evidence into a unified semantic state." + }, + "SYNTHESIZE_METADATA_SEMANTICS": { + "required": "field_semantics", + "output": "输出一段紧凑但信息充分的自然语言 semantic state，区分强证据、辅助证据和仍存在的歧义。", + "optional": "domain_semantics, table_semantics, database_semantics, context_field_semantics, field_description_semantics, field_type_semantics, value_semantics", + "purpose": "Synthesizes all available metadata interpretations into a unified natural-language semantic state." + }, + "EXPAND_DATABASE_ABBREVIATION": { + "required": "database_name", + "output": "输出数据库/系统缩写的自然语言展开及可能的业务范围。", + "optional": "domain, database_description", + "purpose": "Expands an abbreviated database or system name into its likely semantic meaning." + }, + "SEARCH_STANDARD_BY_CONTEXT": { + "required": "context_semantics", + "output": "输出上下文语义最相关的标准候选，并说明其对目标字段的辅助意义。", + "optional": "无", + "purpose": "Retrieves standard categories using contextual-field semantics." + }, + "INTERPRET_CONTEXT_FIELDS": { + "required": "context_fields", + "output": "输出各上下文字段的简要语义及它们共同指向的业务场景；不直接替目标字段做最终分类。", + "optional": "domain, table_name", + "purpose": "Interprets neighboring fields and summarizes the business context jointly implied by them." + }, + "FUSE_DATABASE_EVIDENCE": { + "required": "available_database_evidence", + "output": "输出数据库级自然语言背景及其对目标字段的高层约束。", + "optional": "无", + "purpose": "Integrates database-level semantic evidence." + }, + "ALIGN_VALUE_TO_STANDARD": { + "required": "value_semantics, standard_candidates", + "output": "输出字段值模式对标准候选的支持程度；强调值证据通常是辅助证据。", + "optional": "无", + "purpose": "Aligns value-derived semantics with candidate standard categories." + }, + "COMPARE_CATEGORY_HYPOTHESES": { + "required": "candidate_categories, evidence", + "output": "输出候选类别的自然语言排序与关键区分依据。", + "optional": "无", + "purpose": "Compares competing standard-category hypotheses." + }, + "INTERPRET_DOMAIN": { + "required": "domain", + "output": "输出一段自然语言，说明该领域通常涉及的数据对象、术语语境，以及它对当前元数据解释可提供的约束；不直接给出最终分类。", + "optional": "无", + "purpose": "Interprets the domain context and provides domain-specific semantic background for subsequent metadata understanding." + }, + "REFINE_STANDARD_QUERY": { + "required": "previous_standard_results, current_semantic_state", + "output": "输出一段更聚焦的自然语言查询表达，并说明为何这样改写能区分当前候选。", + "optional": "无", + "purpose": "Refines the standard-search query using previous retrieval results and the current semantic state." + }, + "SEARCH_STANDARD": { + "required": "semantic_query", + "output": "输出最相关的若干标准类别标签及其描述的自然语言摘要，说明为何相关；不额外引入标准库不存在的字段。", + "optional": "无", + "purpose": "Searches the standard corpus using the current semantic representation." + }, + "ASSESS_CATEGORY_SUPPORT": { + "required": "candidate_category, evidence", + "output": "输出自然语言支持评估：支持证据、反对证据、薄弱点和总体判断。", + "optional": "无", + "purpose": "Assesses how strongly the current evidence supports a candidate category." + }, + "INTERPRET_FIELD_TYPE": { + "required": "field_type", + "output": "输出字段类型可提供的约束性自然语言判断；不得仅凭类型强行确定具体类别。", + "optional": "field_name, field_description", + "purpose": "Uses the field data type as supporting semantic evidence." + }, + "RESOLVE_STANDARD_CONFLICT": { + "required": "conflicting_standard_alignments", + "output": "输出哪个标准候选更可信、为什么，以及仍存在的歧义。", + "optional": "无", + "purpose": "Resolves conflicts among competing standard alignments." + }, + "SEARCH_STANDARD_BY_DATABASE": { + "required": "database_semantics", + "output": "输出数据库级语义可能支持的标准类别范围和依据。", + "optional": "无", + "purpose": "Uses database-level semantics to retrieve or constrain candidate standard categories." + }, + "GET_COMPLETE_DATABASE_CONTEXT": { + "required": "database_name", + "output": "输出数据库级自然语言背景摘要，不引入跨库信息。", + "optional": "无", + "purpose": "Retrieves the available high-level metadata context of the current database." + }, + "INTERPRET_TABLE_NAME": { + "required": "table_name", + "output": "输出表级自然语言语义，强调该表可能描述什么业务数据。", + "optional": "domain, table_description", + "purpose": "Interprets the business semantics represented by a table name." + }, + "ALIGN_FIELD_TO_STANDARD": { + "required": "field_semantics, standard_candidates", + "output": "输出字段语义与各候选标准描述的匹配/不匹配理由，以及当前更优候选。", + "optional": "无", + "purpose": "Aligns the target-field semantics with candidate standard categories." + }, + "INTERPRET_DATABASE_NAME": { + "required": "database_name", + "output": "输出数据库级自然语言语义，作为当前字段的高层背景。", + "optional": "domain, database_description", + "purpose": "Interprets the domain and semantic scope implied by the database name." + }, + "EXPAND_FIELD_ABBREVIATION": { + "required": "field_name", + "output": "输出字段缩写的候选展开及依据；如果存在歧义，应保留多个候选并说明不确定来源。", + "optional": "domain", + "purpose": "Expands an abbreviated field name into one or more plausible natural-language meanings." + }, + "CHECK_EVIDENCE_SUFFICIENCY": { + "required": "current_evidence", + "output": "输出自然语言说明“足够/不足”的依据；不足时明确建议继续获取哪类既有信息。", + "optional": "无", + "purpose": "Determines whether the current evidence is sufficient for reliable classification." + }, + "INTERPRET_FIELD_VALUE": { + "required": "value", + "output": "输出值模式的自然语言描述、可能对应的数据含义，以及这种证据的可靠性边界。", + "optional": "field_name, field_type, domain", + "purpose": "Infers possible semantics from the observed value pattern and content." + }, + "INFER_CATEGORY": { + "required": "candidate_standard_categories, all_available_evidence", + "output": "输出自然语言说明为何选择某一 category_label 作为 category；此处仍可保留解释文本，最终结构化由 FINALIZE 完成。", + "optional": "无", + "purpose": "Infers the final Category category from candidate standard categories and accumulated evidence." + }, + "FUSE_STANDARD_EVIDENCE": { + "required": "standard_alignment_evidence", + "output": "输出统一标准证据状态，明确主要支持类别和弱支持类别。", + "optional": "无", + "purpose": "Integrates multiple standard-alignment results into a unified standard-evidence state." + }, + "ALIGN_TABLE_TO_STANDARD": { + "purpose": "Aligns table-level semantics with candidate standard categories.", + "required": "table_semantics, standard_candidates", + "optional": "无", + "output": "Explain how table context supports, contradicts or weakly matches each actual standard candidate; context remains auxiliary to target-field evidence." + }, + "ALIGN_DATABASE_TO_STANDARD": { + "purpose": "Aligns database-level semantics with candidate standard categories.", + "required": "database_semantics, standard_candidates", + "optional": "无", + "output": "Explain how database context supports, contradicts or weakly matches each actual standard candidate; context remains auxiliary to target-field evidence." + } +} diff --git a/research/vehicle_program_v1/code/upstream/runtime/vendor/study.py b/research/vehicle_program_v1/code/upstream/runtime/vendor/study.py new file mode 100644 index 0000000..82d329c --- /dev/null +++ b/research/vehicle_program_v1/code/upstream/runtime/vendor/study.py @@ -0,0 +1,323 @@ +"""Frozen full-pool study identity and independently reconstructible metrics.""" +from __future__ import annotations + +from collections import Counter +import hashlib +import json +import copy +from pathlib import Path + +from common import fingerprint, read_json, read_jsonl, sha256, write_json + +ROOT = Path(__file__).resolve().parent +ADAPTER_SKILLS = ('SELECT_CATEGORY_TOPK', 'INFER_CATEGORY_FROM_TOPK') +P14_SKILLS = frozenset(('CHECK_METADATA_AVAILABILITY','INTERPRET_DOMAIN','EXPAND_FIELD_ABBREVIATION', + 'INTERPRET_FIELD_NAME','DISAMBIGUATE_FIELD_SEMANTICS','SELECT_CATEGORY_TOPK', + 'COMPARE_STANDARD_CANDIDATES','CHECK_EVIDENCE_SUFFICIENCY','INFER_CATEGORY_FROM_TOPK', + 'INFER_GRADING_LEVEL','FINALIZE_CLASSIFICATION','GET_FIELD_METADATA', + 'INTERPRET_FIELD_DESCRIPTION','INTERPRET_TABLE_DESCRIPTION')) +BASE_CONFIG = { + 'raw_metadata_schema_version': 'initial-record-plus-acquired-raw-v2', + 'binding_rule_version': 'canonical-state-bindings-v1', + 'evidence_schema_version': 'typed-observed-evidence-v2', + 'schema': 1, 'pools': ['P14', 'F66'], 'conditions': ['base', 'sft'], + 'condition_order': 'all-base-before-any-sft', + 'selection': 'sha256(finance-followup-20261004: + source_id), ascending digest; ID tie-break', + 'pilot_size': 20, 'expanded_size': 60, 'smoke_size': 2, + 'generator_seed': 20260928, + 'generation': {'max_new_tokens': 512, 'enable_thinking': False, 'do_sample': False, + 'eos_token_ids': [248044, 248046], 'local_files_only': True}, + 'loop': {'max_turns': 8, 'max_identical_calls': 2, 'action_retries': 1, + 'forced_finalize_retries': 1, 'repetition_retries': 1, + 'recover_ready_finalization': True}, + 'native_standard_backend': 'model-catalog-top5-v1', + 'adapter_skills': list(ADAPTER_SKILLS), 'new_training': False, + 'primary_recall': 'last-successful-top5', + 'metadata_source_scope': 'frozen-Val-unlabelled-only', +} + +VARIANTS = {name: {'variant':name,'missing_awareness':aware,'max_turns':turns} + for name,aware,turns in [('A',False,8),('B',True,8),('C',False,12),('D',True,12)]} + +P14_ARMS = { + 'O': {'variant':'C','metadata_prefetch':False,'top1_selection_mode':'model'}, + 'D': {'variant':'D','metadata_prefetch':False,'top1_selection_mode':'model'}, + 'M': {'variant':'D','metadata_prefetch':True,'top1_selection_mode':'model'}, + 'E': {'variant':'D','metadata_prefetch':False,'top1_selection_mode':'first'}, + 'ME': {'variant':'D','metadata_prefetch':True,'top1_selection_mode':'first'}, +} + + +def load_config(root=ROOT): + path = Path(root)/'variant.json' + variant = read_json(path) if path.exists() else VARIANTS['A'] + if (not isinstance(variant,dict) or set(variant) != {'variant','missing_awareness','max_turns'} + or type(variant.get('missing_awareness')) is not bool + or type(variant.get('max_turns')) is not int + or variant.get('variant') not in VARIANTS + or variant != VARIANTS[variant['variant']]): + raise ValueError('invalid canonical variant configuration') + config = copy.deepcopy(BASE_CONFIG) + config.update({'variant':variant['variant'],'missing_awareness':variant['missing_awareness']}) + config['loop']['max_turns'] = variant['max_turns'] + control_path = Path(root)/'p14_controls.json' + if control_path.exists(): + control = read_json(control_path) + if (not isinstance(control,dict) or set(control) != {'arm','metadata_prefetch','top1_selection_mode'} + or type(control.get('arm')) is not str or control['arm'] not in P14_ARMS + or type(control.get('metadata_prefetch')) is not bool + or type(control.get('top1_selection_mode')) is not str): + raise ValueError('invalid P14 control declaration') + expected=P14_ARMS[control['arm']] + if (not path.is_file() or variant != VARIANTS[expected['variant']] + or {k:v for k,v in control.items() if k!='arm'} != {k:v for k,v in expected.items() if k!='variant'}): + raise ValueError('P14 control combination or variant differs') + config.update({'p14_arm':control['arm'],'metadata_prefetch':control['metadata_prefetch'], + 'top1_selection_mode':control['top1_selection_mode'],'pools':['P14'], + 'conditions':['sft'],'condition_order':'existing-adapter-only-no-new-training'}) + return config + + +CONFIG = load_config() + + +def select_cohorts(rows): + ids = [row.get('source_id') for row in rows] + if any(not isinstance(sid, str) or not sid for sid in ids) or len(ids) != len(set(ids)): + raise ValueError('source IDs must be nonempty unique strings') + size = CONFIG['pilot_size'] + CONFIG['expanded_size'] + CONFIG['smoke_size'] + if len(ids) < size: + raise ValueError('too few rows for disjoint Pilot, Expanded and smoke') + ordered = sorted(ids, key=lambda sid: (hashlib.sha256(('finance-followup-20261004:'+sid).encode()).hexdigest(), sid)) + return {'pilot': ordered[:20], 'expanded': ordered[20:80], 'smoke': ordered[80:82]} + + +def adapter_route(condition, messages): + if condition not in CONFIG['conditions']: + raise ValueError('unknown condition') + if condition == 'base' or not messages or messages[0].get('role') != 'system': + return None + header = messages[0].get('content', '') + return next((name for name in ADAPTER_SKILLS + if header.startswith(f'Execute only the selected Skill {name}.\n')), None) + + +def execution_files(root): + root = Path(root) + paths = list(root.glob('*.py')) + list(root.glob('*.sh')) + paths += list((root/'runtime/code').rglob('*.py')) + paths += list((root/'runtime/skills').rglob('*.md')) + paths += list((root/'runtime/tests').rglob('*.py')) + paths += [path for folder in ('runtime/config', 'runtime/tests/fixtures') + for path in (root/folder).rglob('*') if path.is_file()] + paths += list((root/'tests').rglob('*.py')) + for name in ('variant.json','p14_controls.json'): + if (root/name).exists(): + paths.append(root/name) + paths += [root/name for name in ( + 'data/val.jsonl', 'data/metadata-val.json', 'data/registry.json', + 'data/split_boundaries.json', 'reference_skills_63.json', 'skill_reference.json', 'model_artifacts.json', + 'adapter_completion_reference.json')] + if any(path.is_symlink() or not path.resolve().is_relative_to(root.resolve()) for path in paths): + raise ValueError('execution inventory path escape or symbolic link') + return {path.relative_to(root).as_posix(): sha256(path) for path in sorted(set(paths))} + + +def freeze_study(root=ROOT): + root = Path(root) + if (root/'frozen/manifest.json').exists(): + raise ValueError('refuse to overwrite frozen study') + cohorts = select_cohorts(read_jsonl(root/'data/val.jsonl')) + payload = {'config': load_config(root), 'cohorts': cohorts, 'files': execution_files(root), + 'expected_base_manifest': read_json(root/'model_artifacts.json'), + 'adapter': read_json(root/'adapter_completion_reference.json'), + 'scope': 'development validation; no new SFT; not untouched Test'} + manifest = {**payload, 'execution_signature': fingerprint(payload)} + write_json(root/'frozen/manifest.json', manifest) + return manifest + + +def check_study(root=ROOT): + root = Path(root) + manifest = read_json(root/'frozen/manifest.json') + if fingerprint({k:v for k,v in manifest.items() if k != 'execution_signature'}) != manifest.get('execution_signature'): + raise ValueError('frozen manifest signature mismatch') + if manifest.get('config') != load_config(root): + raise ValueError('frozen configuration changed') + if manifest.get('files') != execution_files(root): + raise ValueError('execution file inventory changed') + if manifest.get('cohorts') != select_cohorts(read_jsonl(root/'data/val.jsonl')): + raise ValueError('frozen cohort selection changed') + return manifest + + +def write_jsonl(path, rows): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name+'.tmp') + temporary.write_text(''.join(json.dumps(row, ensure_ascii=False, sort_keys=True)+'\n' for row in rows), encoding='utf-8') + temporary.replace(path) + + +def prepare_resume(output, identity, ids, resume): + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + manifest_path = output/'manifest.json' + if manifest_path.exists(): + previous = {k:v for k,v in read_json(manifest_path).items() if k != 'status'} + if not resume or previous != identity: + raise ValueError('refuse overwrite or changed resume identity') + elif any(output.iterdir()): + raise ValueError('unknown output artifacts without identity') + saved = read_jsonl(output/'traces.jsonl') if (output/'traces.jsonl').exists() else [] + completed_ids = [row.get('source_id') for row in saved] + if len(completed_ids) != len(set(completed_ids)): + raise ValueError('duplicate saved sample IDs') + if not set(completed_ids) <= set(ids): + raise ValueError('saved sample ID outside frozen cohort') + if not manifest_path.exists(): + write_json(manifest_path, identity) + return saved + + +def candidate_history(row, registry): + history = [] + for event in row['trace'].get('events', []): + data = event.get('data', {}) + outputs = data.get('data', {}) + if event.get('kind') != 'observation' or data.get('status') != 'success' or 'standard_candidates' not in outputs: + continue + candidates = outputs['standard_candidates'] + if not isinstance(candidates, list) or any(not isinstance(c, dict) for c in candidates): + raise ValueError('invalid candidate observation shape') + ids = [c.get('category') or c.get('category_id') for c in candidates] + if len(ids) != 5 or len(set(ids)) != 5 or not set(ids) <= set(registry): + raise ValueError('candidate observation is not five distinct frozen Registry IDs') + history.append(ids) + return history + + +def _macro_f1(golds, predictions, labels): + scores = [] + for label in labels: + tp = sum(g == label and p == label for g,p in zip(golds, predictions)) + fp = sum(g != label and p == label for g,p in zip(golds, predictions)) + fn = sum(g == label and p != label for g,p in zip(golds, predictions)) + scores.append(2*tp/(2*tp+fp+fn) if 2*tp+fp+fn else 0.0) + return sum(scores)/len(scores) if scores else 0.0 + + +def summarize(rows, registry, supported_classes): + n = len(rows) + if len({r['source_id'] for r in rows}) != n: + raise ValueError('duplicate sample ID in scoring') + totals = Counter() + skills = Counter() + successes = Counter() + failures = Counter() + interventions = Counter() + stop_reasons = Counter() + golds, predictions = [], [] + elapsed = 0.0 + new_sample_count = 0 + for row in rows: + trace = row['trace'] + final = trace.get('final_result') or {} + terminated = bool(trace.get('terminated')) + prediction = final.get('category') if terminated else None + predicted_level = final.get('grading_level') if terminated else None + gold, level = row['ground_truth'], row['ground_truth_level'] + if terminated and (prediction not in registry or registry[prediction].get('grading_level') != predicted_level): + raise ValueError('terminal result disagrees with frozen Registry') + golds.append(gold) + predictions.append(prediction) + correct = terminated and prediction == gold + level_correct = terminated and predicted_level == level + totals['terminated'] += terminated + totals['category_correct'] += correct + totals['level_correct'] += level_correct + totals['joint_correct'] += correct and level_correct + stop_reasons[trace.get('stop_reason', 'unknown')] += 1 + recovered = False + used_new = False + for event in trace.get('events', []): + data = event.get('data', {}) + kind = event.get('kind') + if kind == 'controller_intervention': + reason = data.get('reason', 'unknown') + interventions[reason] += 1 + recovered |= reason in {'forced_finalization', 'invalid_action_finalization_recovery'} + if kind == 'policy_action': + tool = data.get('call', data).get('tool') + if tool: + skills[tool] += 1 + totals['skill_calls'] += 1 + used_new |= tool not in P14_SKILLS + if kind == 'observation': + tool = data.get('skill', 'unknown') + if data.get('status') == 'success': + successes[tool] += 1 + totals['successful_skill_calls'] += 1 + else: + failures[tool] += 1 + totals['observation_failures'] += 1 + totals['unavailable_skill_calls'] += data.get('status') == 'unavailable' + if kind == 'retry': + totals['retry_events'] += 1 + if kind in {'retry', 'rejected_action'} and data.get('reason') in {'invalid_action','uncallable_skill','invalid_arguments','stop_without_final_result'}: + totals['invalid_policy_attempts'] += 1 + if data.get('reason') == 'stop_without_final_result': + totals['early_stop_attempts'] += 1 + if kind == 'policy_action' and data.get('messages') and not data.get('forced_finalize'): + totals['accepted_policy_actions'] += 1 + totals['recovered_terminated'] += terminated and recovered + totals['normal_terminated'] += terminated and not recovered + new_sample_count += used_new + history = candidate_history(row, registry) + totals['top5_first_hit'] += bool(history and gold in history[0]) + totals['top5_last_hit'] += bool(history and gold in history[-1]) + totals['top5_any_hit_diagnostic'] += any(gold in c for c in history) + totals['samples_with_retrieval'] += bool(history) + totals['multiple_retrieval_samples'] += len(history) > 1 + if not correct: + if not history: + totals['no_retrieval_failures'] += 1 + elif gold not in history[-1]: + totals['last_top5_miss_failures'] += 1 + elif not terminated: + totals['hit_but_nonterminal_failures'] += 1 + else: + totals['candidate_inside_wrong_choice'] += 1 + totals['runner_errors'] += bool(row.get('error')) + for call in row.get('generation_calls', []): + meta = call.get('generation', {}) + totals['model_calls'] += 1 + totals['backend_failed_calls'] += bool(call.get('error')) + totals['total_tokens'] += int(meta.get('token_usage', {}).get('total_tokens') or 0) + elapsed += float(meta.get('elapsed_seconds') or 0) + expected_counters = ('terminated','category_correct','level_correct','joint_correct', + 'normal_terminated','recovered_terminated','top5_first_hit','top5_last_hit', + 'top5_any_hit_diagnostic','model_calls','total_tokens','skill_calls', + 'successful_skill_calls','runner_errors','backend_failed_calls', + 'no_retrieval_failures','last_top5_miss_failures','hit_but_nonterminal_failures', + 'candidate_inside_wrong_choice','observation_failures','retry_events', + 'early_stop_attempts','samples_with_retrieval','multiple_retrieval_samples') + result = {'denominator': n, **{key: totals[key] for key in expected_counters}, **dict(totals), + 'model_elapsed_seconds': elapsed, + 'macro_f1_supported': _macro_f1(golds, predictions, supported_classes), + 'macro_f1_fixed12': _macro_f1(golds, predictions, sorted(registry)), + 'supported_class_count': len(supported_classes), 'fixed_class_count': len(registry), + 'distinct_skills_used': len(skills), 'skills_called': dict(sorted(skills.items())), + 'skills_succeeded': dict(sorted(successes.items())), 'skills_failed': dict(sorted(failures.items())), + 'controller_interventions': dict(sorted(interventions.items())), + 'stop_reasons': dict(sorted(stop_reasons.items()))} + result.update({'new_skills_called':{k:v for k,v in sorted(skills.items()) if k not in P14_SKILLS}, + 'samples_using_new_skills':new_sample_count, + 'unavailable_skill_calls':totals['unavailable_skill_calls'], + 'invalid_policy_attempts':totals['invalid_policy_attempts'], + 'accepted_policy_actions':totals['accepted_policy_actions'], + 'legal_action_rate':totals['accepted_policy_actions']/(totals['accepted_policy_actions']+totals['invalid_policy_attempts']) + if totals['accepted_policy_actions']+totals['invalid_policy_attempts'] else 0.0}) + for key in ('terminated','category_correct','level_correct','joint_correct','top5_first_hit','top5_last_hit'): + result[key+'_rate'] = result[key]/n if n else 0.0 + return result diff --git a/research/vehicle_program_v1/code/vendor_snapshot.py b/research/vehicle_program_v1/code/vendor_snapshot.py new file mode 100644 index 0000000..7cd0ade --- /dev/null +++ b/research/vehicle_program_v1/code/vendor_snapshot.py @@ -0,0 +1,47 @@ +"""Copy a clean pinned local research runtime, without network or data contents.""" +import argparse +from pathlib import Path +import shutil +import subprocess +from artifacts import ROOT, REVISION, TEAM_COMMIT, put, read, sha + + +def vendor(source, download_manifest): + source = Path(source) + commit = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=source, text=True).strip() + dirty = subprocess.check_output(['git', 'status', '--porcelain', '--', 'research/finance_program_v1'], cwd=source, text=True) + if commit != TEAM_COMMIT or dirty: + raise ValueError('source must be the clean pinned team research commit') + research = source / 'research/finance_program_v1' + target = ROOT / 'upstream' + target.mkdir(exist_ok=False) + for path in sorted((research / 'evaluation/runtime').rglob('*')): + if path.is_file() and path.suffix in ('.py', '.md', '.json') and '__pycache__' not in path.parts: + dest = target / 'runtime' / path.relative_to(research / 'evaluation/runtime') + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, dest) + for name in ('local_smoke.py', 'download_model.py'): + dest = target / 'backend' / name + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(research / 'evaluation/legacy' / name, dest) + put(ROOT / 'upstream.json', {'repository': 'https://github.com/DocAILab/DataClassifyGrading', + 'commit': TEAM_COMMIT, 'runtime_origin': 'research/finance_program_v1/evaluation/runtime', + 'files': {p.relative_to(ROOT).as_posix(): sha(p) for p in sorted(target.rglob('*')) if p.is_file()}}) + manifest = read(download_manifest) + if manifest['repository'] != 'DocAILab/DCG' or manifest['revision'] != REVISION: + raise ValueError('download manifest revision differs') + names = {f'DCG_VEHICLE/DCG_VEHICLE_{s}.json' for s in ('SFT', 'TRAIN', 'VAL', 'TEST', 'CORPUS')} + hashes = {r['path']: r['sha256'] for r in manifest['files'] if r['path'] in names} + if set(hashes) != names: + raise ValueError('missing pinned source files') + put(ROOT / 'pinned-inputs.json', {'repository': manifest['repository'], 'revision': REVISION, + 'download_manifest_sha256': sha(download_manifest), 'files': hashes}) + print('Pinned runtime and five input hashes saved; no dataset contents copied.') + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--source', type=Path, required=True) + p.add_argument('--download-manifest', type=Path, required=True) + a = p.parse_args() + vendor(a.source, a.download_manifest) diff --git a/research/vehicle_program_v1/publication_tests/test_publication.py b/research/vehicle_program_v1/publication_tests/test_publication.py new file mode 100644 index 0000000..6584898 --- /dev/null +++ b/research/vehicle_program_v1/publication_tests/test_publication.py @@ -0,0 +1,69 @@ +import copy +import importlib.util +import json +from pathlib import Path +import sys +import unittest + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tools')) +from redact_trajectories import public_trace + + +class PublicationTests(unittest.TestCase): + def project(self, trace): + return public_trace(trace,'val-0001','a'*64,'completed',{'GET_FIELD_METADATA'}, + {'private-standard':'standard-001'},{'private-grade':'grade-001'}) + + def test_no_free_text_or_dynamic_keys_escape(self): + trace={'input':{'id':'PRIVATE_SOURCE','field_name':'PRIVATE_FIELD'}, + 'terminated':True,'stop_reason':'validated_terminal', + 'planner_attempts':[{'attempt':1,'status':'accepted','raw_response':json.dumps({'program':[ + {'tool':'GET_FIELD_METADATA','arguments':{'PRIVATE_FIELD':'$state.PRIVATE_FIELD'}}, + {'tool':'PRIVATE_FIELD','arguments':{}}]})}], + 'programs':[{'program_index':1,'status':'terminated','terminated':True,'plan':{'program':[]}, + 'steps':[{'call':{'tool':'GET_FIELD_METADATA','arguments':{'x':'PRIVATE_SOURCE'}}, + 'status':'success','summary':'PRIVATE_FIELD','data':{'PRIVATE_FIELD':'SECRET_VALUE'}}]}], + 'final_result':{'category':'private-standard','grading_level':'private-grade','reason':'SECRET_VALUE'}, + 'budget':{'programs':1,'PRIVATE_FIELD':'SECRET_VALUE'}} + original=copy.deepcopy(trace) + result=self.project(trace) + self.assertEqual(trace,original) + serialized=json.dumps(result) + for token in ('PRIVATE_SOURCE','PRIVATE_FIELD','SECRET_VALUE','private-standard','private-grade'): + self.assertNotIn(token,serialized) + self.assertEqual(result['final_result']['category_alias'],'standard-001') + self.assertTrue(result['terminated']) + + def test_rejected_attempt_and_failure_preserved_without_raw_text(self): + result=self.project({'terminated':False,'stop_reason':'planner_budget_exhausted', + 'planner_attempts':[{'attempt':1,'status':'rejected','raw_response':'SECRET invalid JSON'}], + 'programs':[],'final_result':None}) + self.assertEqual(len(result['planner_attempts']),1) + self.assertTrue(result['planner_attempts'][0]['unparseable']) + self.assertNotIn('SECRET',json.dumps(result)) + self.assertFalse(result['terminated']) + + def test_unknown_status_stop_and_budget_strings_are_not_published(self): + result=self.project({'stop_reason':'PRIVATE_REASON','budget':{'programs':'PRIVATE_BUDGET'}, + 'planner_attempts':[{'status':'PRIVATE_STATUS','attempt':'PRIVATE_COUNTER'}]}) + self.assertNotIn('PRIVATE_',json.dumps(result)) + + def test_released_rows_and_metrics_agree(self): + rows=[json.loads(line) for line in (ROOT/'trajectories/val397.redacted.jsonl').read_text(encoding='utf-8').splitlines()] + self.assertEqual([r['public_id'] for r in rows],[f'val-{i:04d}' for i in range(1,398)]) + self.assertEqual(sum(r['terminated'] for r in rows),115) + self.assertEqual(sum(r['receipt_status']=='backend_error' for r in rows),17) + summary=json.loads((ROOT/'results/evidence-summary.json').read_text(encoding='utf-8')) + self.assertEqual(summary['metrics']['joint_correct'],26) + self.assertEqual(summary['metrics']['denominator'],397) + self.assertTrue(all(r['privacy']['not_training_data'] for r in rows)) + + def test_publication_code_has_only_declared_differences(self): + import hashlib + manifest=json.loads((ROOT/'results/code-publication.json').read_text(encoding='utf-8')) + changes=[name for name,digest in manifest['historical_runtime_files_sha256'].items() + if hashlib.sha256((ROOT/'code'/name).read_bytes()).hexdigest()!=digest] + self.assertEqual(sorted(changes),sorted(manifest['publication_changes'])) + +if __name__=='__main__': unittest.main() diff --git a/research/vehicle_program_v1/results/code-publication.json b/research/vehicle_program_v1/results/code-publication.json new file mode 100644 index 0000000..052e1b2 --- /dev/null +++ b/research/vehicle_program_v1/results/code-publication.json @@ -0,0 +1,157 @@ +{ + "upstream_commit": "9473b2c2a330b7f48dc0c1831bda11e36d428379", + "historical_runtime_files_sha256": { + "api_backend.py": "5b7d9e338e6040240ed45531e487f4a52619ef70ac54f67dd2aaa0aaeb11c1f2", + "api_check.py": "bca1d75b8d865f3501dc79e7e61d648e2b53899bc108cdf3f17eb729014992b0", + "api_discover.py": "3a47a1dfb2bd4ddb8bd0741a7ca7f3b96ca73e85ca62affd93de409f44f29e2b", + "api_offline_audit.py": "38b71f0a322008f85d72db2b3c9ea2057600889864c079db4a3e3ace2b30cb88", + "api_parallel.py": "997d9446a5079aa8617ba54dc28551ed70cf8ccbd00b902f5824eb7ebd5182fc", + "api_prepare.py": "d41bba6844a9a3d5fedee642dc842c2f8b2008fa151ae0ae02b4d09f73a97794", + "artifacts.py": "a275c2d4d855509aff039999e09d9015f75b4cc29583a53d659624232ba40f64", + "backends.py": "1de00b39e5f834a646ffb35c04bc80d233bd0ee281162d5dbfa5a10e7a9f39da", + "bundle.py": "35084ac081219e785db8aeb8c4a4fd54caa7d9dc1f8e7c823a209fe3408d3f6f", + "Night-Control.ps1": "c84e01dc7f6ad328d632078d73fc3ad842386901a0d16ac10f35405d444ca912", + "night_ops.py": "754e6fb1935a60893ed3ded35a2937d5856381e504464160fb55e40b09cf70b5", + "night_prepare.py": "3187efb36912f213e8e5b6f365db75044c170b1730bc5ab68a358baf28105096", + "night_resume.py": "17e9db5de8d9887483cb0215a555b76ed1cf15e76ed010011a9f8675cc1492b6", + "night_runner.py": "1680180706aa665dd58815100b9e173ae2f0f801549f9c1084b815778b1bdae2", + "night_supervisor.py": "108bc00c65020b9d63a60e28397bdd1ee93173b970eea818285e99d07301a08a", + "pinned-inputs.json": "b102eaa6b614e5246ddccf9eabe027117cbe27422da947065c14cb6fc4617e32", + "prepare.py": "6d7cf76783cf16a1a311ba1d702713ee023405be33706a000fb98c17fcdae426", + "README.md": "340c6085cbc767abcc5b3e791a5e4b8c5efa982fae6e34e17f9309de9d75e5ff", + "replay.py": "3815818e3b9b093e6d20c53aedc21863931974ac7a71b43dbdbafda7e4a18e07", + "runner.py": "06eac150c6864d1a051311a4783961f342fdd907efc54d21616392f44db48912", + "score.py": "18378a86f09893aa9aa2ef18246318fcd82687f32fad965521b0c086ae4bd2fc", + "tests/test_api.py": "ccd557fc0e43cfa3e07ea327d812560f0ba1105f3118e7d2451980c2f2fae6ce", + "tests/test_night.py": "0df57ab7835b40d418b37a4bb2c54c2bf1a66d63fc30d6708972567388bb6750", + "tests/test_night_recovery.py": "33bcc68cb78fcd69d2c2169e8a08cfa4f0b482a7f15ad716c2c8aef6ac852781", + "tests/test_parallel.py": "e7accf1a049be28f1c9dc57cbad7489fd7894676bff34f0c42c0d3617a53b342", + "tests/test_vehicle.py": "5c6bf49df62d1562c760398d98fc166c323da493dd193ab86a4af0c2f7097b7f", + "upstream/backend/download_model.py": "5e435d6f87d920d58ff6d305afe61be3a98fec8d12b6f79986072a9a208a8edd", + "upstream/backend/local_smoke.py": "6517f5bc9bbb8c5fa4bc7e9100d1c4cf65a7829b4d42543e837d5798ad81f8ef", + "upstream/runtime/program_loop/__init__.py": "8ebc1b9a3937a1848c199126edf97e5862741ede120a491e87a8473d6c0f7c46", + "upstream/runtime/program_loop/__main__.py": "935a1c1166b0c1ea35a82256345000bf2c73ded718d77773bc27a71ecce28f7d", + "upstream/runtime/program_loop/_vendor.py": "cfb187e38a0b8acd86e83476ee24f3aa02928c86fd3deb532f752ae8784bbf80", + "upstream/runtime/program_loop/catalog_context.py": "7e08a466d701eb8b67733629c7af4723f9baf1eedf16ba768a78ba4d6d0e10cb", + "upstream/runtime/program_loop/cli.py": "82d3841baa287c2d5e0cf519ae2532a4fd5a4d79e89d907e29ad029c17d59335", + "upstream/runtime/program_loop/contracts.py": "04f584b9acfb033fb1cbd38c8f08595f46bd3d4c57a55495fc282920bd9169a4", + "upstream/runtime/program_loop/controller.py": "a3a8ad54e30a0fb623cf9524bc766c4ee6d334de84648aaec77addda3f0c93e1", + "upstream/runtime/program_loop/execution.py": "960800db0c9f49ca7db61c9f20fa3967f6373d2df000da1a7eb253d150efa72e", + "upstream/runtime/program_loop/failure_policy.py": "92282fb6456a3df91deab50c141964abbad68741e948116ef0e4a3ad8057762f", + "upstream/runtime/program_loop/planner.py": "6edeab5d5164663a6e9f1ac5c31848836469f970b7accbeff937e033dc80e22b", + "upstream/runtime/program_loop/planning_contracts.py": "b0cd81f6c0aac6c0ef09bf460176522683f9612b908de06f0841abd16958e312", + "upstream/runtime/program_loop/prompt_encoding.py": "20862c2334a809ac06629058495eaaf4e94a601345dbcdbd5b6d573df07ebebc", + "upstream/runtime/program_loop/recovery_context.py": "0d67d47e51f55edc4bbffe1164327c7f74d7aff1b629dae6df3e5c9c201af2f7", + "upstream/runtime/program_loop/rejection_context.py": "fd71ad424ff4a201b01f4db7276f2d958fd94eba7aaf26deb16cc54dbcb9de1f", + "upstream/runtime/program_loop/validation.py": "c96220f85cc14375ce6171dc59c2ff5d46389017ca9549cb41ff00cda82b51eb", + "upstream/runtime/runtime_identity.py": "a183d837ec5f666ea8bd0c1965ff0004d63be74cd1c369505cd7e6ddcb2dccd1", + "upstream/runtime/tests/test_grounding.py": "8bc06990233ef500ea3b7cd572c65b0dd7ca14565c5269ec25f2a0ced649e716", + "upstream/runtime/tests/test_integration.py": "c38e9d6b4c0cae512637c050c951770d5bca35df519927ab2ce01905941946f9", + "upstream/runtime/vendor/common.py": "c79a02e987a169436796f438a415d7c309c0e0cf50bb447e493cd68cb9301fa4", + "upstream/runtime/vendor/comparison_output.py": "ed7b3bb552aeaff329ab4531bfa4fbd93772bf560475dc4d3fb848c12bee68e8", + "upstream/runtime/vendor/evidence_roles.py": "6b76fbc8696a70d9ed5723a4ba71edc4de540f29e883d5ee5f99d9f806536064", + "upstream/runtime/vendor/grounded_semantics.py": "4f2f764e2256ad1dda8716f42277e688fe710fbb607e310cdbe57995e43c254c", + "upstream/runtime/vendor/pool_availability.py": "c6fa8e23a678b4451ef78d301efb347507fb0513a9d18bfda76eeefc824d70aa", + "upstream/runtime/vendor/pool_bindings.py": "3c0c65e78f7bcdc5bb81003b455fd957dba5c4abdbfdaa20b8fddb3b155e07db", + "upstream/runtime/vendor/pool_catalog.py": "bd19db2c908d41120498eae4afc26e30726b60ddf7fbf834e77ca6251e8c8a82", + "upstream/runtime/vendor/pool_metadata.py": "0705318048785fd34edd970274145dec359269e5e30756a996eb156e168e3a8a", + "upstream/runtime/vendor/pool_runtime.py": "ae5f3927e4b4faa5a7808c9ae9975af9ae9ef7d7f2f347fd034f34cadf2e9f45", + "upstream/runtime/vendor/reference_skills_63.json": "a63b6ee85f983caf9f55d537fe85d24a283a860ccb0985c8a7fce8b1466e3caf", + "upstream/runtime/vendor/runtime/code/agent/__init__.py": "c2ee5b379bec6f6381e0651bee6078e38ad6ca294f497d40b6d27e583fd1492e", + "upstream/runtime/vendor/runtime/code/agent/orchestration/__init__.py": "64f470fa2d85450ebbd8819a66c63064bcdbf993c829d248c64315285e7aca73", + "upstream/runtime/vendor/runtime/code/agent/orchestration/agent_loop.py": "176bc72eebd17a7d2e45276654c49cbd5b1828306e11d2579faffe143eebcade", + "upstream/runtime/vendor/runtime/code/agent/orchestration/artifact_projection.py": "d71939b3e4b04b83d33f7aedbc3173cd44043b8a55d44c149c46f35d5aa3f7d8", + "upstream/runtime/vendor/runtime/code/agent/orchestration/catalog.py": "66ceb63dd02030cecb3965247a671da64c4d29724ece9a5eb52961a0647c8e6a", + "upstream/runtime/vendor/runtime/code/agent/orchestration/category_catalog.py": "baf7b0b234822060b48dded26f00d526654749f1384c3e23f9eb07ddc8fa0a58", + "upstream/runtime/vendor/runtime/code/agent/orchestration/contracts.py": "19ed69938bdc517be14625e0ef886945dc4483faeb6a8260fdfcacaffa83f233", + "upstream/runtime/vendor/runtime/code/agent/orchestration/metadata_lookup.py": "e338e0d804caa20218637074422b08f26b16fdbdd9faff39077a1cb73beb89fc", + "upstream/runtime/vendor/runtime/code/agent/orchestration/metrics.py": "fffb69bd3e8efa149e46e3413b4056457840b9cd8f6798fd5e0871cdb7e3a0b3", + "upstream/runtime/vendor/runtime/code/agent/orchestration/model_skill_executor.py": "6ad63e1de6cb77bbf6d3cc537f907b59e94147b7e58de885d0d0c43bdb31671b", + "upstream/runtime/vendor/runtime/code/agent/orchestration/planner.py": "5be67a873f8fa54c183c9fa381def425b73cc92181fab6079506eb2296fa53d8", + "upstream/runtime/vendor/runtime/code/agent/orchestration/retrieval.py": "fbe140936695f38c6bc6838c5a9a0100ca9e6f98b6e77984e587fdbea28b6bf4", + "upstream/runtime/vendor/runtime/code/agent/orchestration/runtime.py": "887e82115154f043801719b5e292895e3eeef0d25e277748805a0c1dabc39acc", + "upstream/runtime/vendor/runtime/code/agent/orchestration/skill_loader.py": "7528bada337e98b69ef61ccf558574f5940d26fcfc2106530f8c9d0d765964f6", + "upstream/runtime/vendor/runtime/code/agent/orchestration/table_field_lookup.py": "93319c4ee044439f9e25449cec220dfaededa1c6b799816486d19402b569dc22", + "upstream/runtime/vendor/runtime/code/agent/orchestration/topk_selection.py": "a58411d1acb421bd5e5c134f4ceef7ccce76851f684a397446904b7a17b1eb4c", + "upstream/runtime/vendor/runtime/code/agent/orchestration/validator.py": "15fd2b9faca5addf49f4fdf5d816eb2ecb3723c6b800da3fd591cda93b0bedcd", + "upstream/runtime/vendor/runtime/code/script/orchestration/__init__.py": "7c61249a27a9c7ad011f983e9857e3889ae54837d00faf2660fe2f72350b30d6", + "upstream/runtime/vendor/runtime/code/script/orchestration/run_agent_loop_batch.py": "6605815bf479e8fc50a3041af3c2430b3f121740bf38b8e6d497212d4425616d", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_ASSESS_CATEGORY_SUPPORT.md": "85ea64633013f69d582707acf4ab1b32201cf5ebbf7e0356041119d6c08c106a", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_CHECK_EVIDENCE_SUFFICIENCY.md": "f879c21470b9daeb773a80a256a07f7a6f43e38e8452347f128eca6391362f4a", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_COMPARE_CATEGORY_HYPOTHESES.md": "3fa31691df6bb353b4171a8c837da229549f8f4eff827bf502f81cb2318432a5", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_DECIDE_NEXT_EVIDENCE_SOURCE.md": "647a447ae584d92e67c17d2d7040ff48d14cb8d2b5039345d65eb072b5c14fd3", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FINALIZE_CLASSIFICATION.md": "cd66513c1ca4b6728bd217ae57a7ad7cd96257f61443a7af2ac1edd7e746f937", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_CONTEXT_EVIDENCE.md": "b0028a313fa3f81350f07093c5389c7ec0447539119e37930f1f05bceec539d1", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_DATABASE_EVIDENCE.md": "e27e9b2f57b5b1929d5286a10cc15233ab48ba40948fe520fde8853f66bc1e56", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_FIELD_EVIDENCE.md": "6dfef164297cb75a02f64ebbdafecb3f0519f1a5f1992c3bb9a98157b7a7ee54", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_METADATA_EVIDENCE.md": "e3a7e0de334bc39836c3a909edc79233075b0b73f7c30e879d4e087bc94aa6f2", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_STANDARD_EVIDENCE.md": "eddac25de1c3dda0baf876d29a7a32ae4fab980f2ae7ce892def82af3469087a", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_TABLE_EVIDENCE.md": "24c7eb0ea3381b640465e78e41eb2ff43495e7bb3eadcc652e5587c3bd167ab4", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_IDENTIFY_MISSING_EVIDENCE.md": "237695b8b5d6f9a7105902dc32a120297c364ee5f8d1c27f4ead212df110668f", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY.md": "032b758517d3a8535c29d20d8391bd8d27a2ad1d5c7e9f62246578076ddb5197", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY_FROM_TOPK.md": "7af18e11373d633fe4369338336c653b47238c214cb5f1937e0a5b30041f722f", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_GRADING_LEVEL.md": "276b5e1a8b080d9aecec762762939372f40e56c8fb4b1eb92964c53073b6d603", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_FIELD_CONTEXT_CONFLICT.md": "94e552edccadc03cb55a5ab8e54511a296971363bd4aafb30b3ab1385e5a18f3", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_METADATA_CONFLICT.md": "643472dc5b7e7cdb38f3483857acc94d539884efede9b9ed0f4cccc79a037d0b", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_STANDARD_CONFLICT.md": "be9b9c9c879b964acf4b5f7317dcf90c68833a1f98f6cb8ab9f8199460066b25", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CATEGORY_WITH_STANDARD.md": "deb07baab54b052a9323b475c912d7743115c96dd4b1dd0b7c4a80df3e3d0dfc", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CLASSIFICATION_GRADING_CONSISTENCY.md": "ff8ef17c1ce1408496c6634389ecba87f6d282d0264e4ac1ee9a8c2deb12bab1", + "upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_WEIGH_EVIDENCE.md": "c928a03a7a8f2d8b1a28ffb01c100543000a62e4a388a0e60051a4b6815dd43f", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_ACQUIRE_MISSING_METADATA.md": "cc788f17f7c63e1f7974f220eca3becfb186d3df287a35d814a5387964d20ce3", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_CHECK_METADATA_AVAILABILITY.md": "fe27b6f518a98be4e88414d6180180954572f222fec37dfc4591e2b72a05ad5a", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_EXPAND_SELECTED_CONTEXT.md": "323b136804a9db277d6642b71de277ae3fcd7021910645475872b1b018d79a79", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_DATABASE_CONTEXT.md": "0008e4dc6b220757208cb656912ebb609c8230aaf62ce56a33f247dc5879f7cd", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_TABLE_CONTEXT.md": "09b1f136ca45244c603e98aba0bec6fad8997ca2e618eeaeaf7ac1c1de4fb711", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_DESCRIPTIONS.md": "0b5f7d9ced8902a3a7157d6781a3b3d6bc9fb9f2d2c55da99f31977ceb5609b8", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_TYPES.md": "61cfe7efeb0980d22e72c4069e70cb91d1a007c7352b528b600b7efbcfd313e5", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_VALUES.md": "780305d46253a0ab132f35feee6ba420347f351326827c4be781bd4224349504", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_DATABASE_DESCRIPTION.md": "37b2146b9cd669e1dc0cb0a54c0f6ee36dc0157ce1e4e753d96be08a6fdada21", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_DESCRIPTION.md": "d545962070cb10fa90ee1693261d48ee08df616cad4d098bfcd98643100fce7b", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_METADATA.md": "372ce861fcca6138cda43b693a8fc32f85c0ec380b0c275c3cfe613dbe25f629", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_DESCRIPTION.md": "912334b76f95c6f468cf4bab840ae0ec3a262639c59e545db63bfce866710ea0", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_FIELDS.md": "99c01c5869b511754feccfd335017c7f4290683fbcc46ed6355b95f775d605b0", + "upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_SELECT_CONTEXT_FIELDS.md": "aacb11c41507d2c2743e56a7915c9dae9d6b4e2ff3ece6fec6b3d73babeb6683", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_DISAMBIGUATE_FIELD_SEMANTICS.md": "e497bddaf99da95503677a969926921b507b92ad542055dafaa88ee724a235d0", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_DATABASE_ABBREVIATION.md": "2338078f06cd5276b326747fa0b80790d3c638ba369f6579e4ddcff84882eb51", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_FIELD_ABBREVIATION.md": "fc8d4265d620c57c6d181e395aa08a0f19319f86437739037659b29016e15fad", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_TABLE_ABBREVIATION.md": "6ce033822d4d4f4ae7babba7bd0b44fa871476353b81fd86ac6742d266b3a426", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_CONTEXT_FIELDS.md": "297bf09515bbbc0a4b9e6f80fb77c7135b7484c5c428da0868d7a0d0732717cb", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_DESCRIPTION.md": "acc57509cff6330476522f8fc5a7b286b0ede3e477c9806000b54542f89070dc", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_NAME.md": "e427e888ac10d742784d390ad2bdce40d0aec7c9a7abd52b5f5497fc83b866fc", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DOMAIN.md": "6e1760ff387797cb4c171f404f744b6b6b87445969659bf8b37c8bca66de5375", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_DESCRIPTION.md": "fdbd8cf655fdc0ecddafd9ce7a73f4996d7b4698405e87d8a41e134c7d85adac", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_NAME.md": "861a87c3342531c281779ac572c118f92a4c4b9e13a43e5af3eb39620e5e1d39", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_TYPE.md": "93500ee995113eed8ee9d6d8887d6f65f2d4aa6ac52b12697fb8e82982c1b49d", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_VALUE.md": "c0dad806fde006487b38b9d56fc0b5fde6cc2e367b8a46d13adb5c4171ed19ad", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_DESCRIPTION.md": "901cef9276dd30ff1b2e35fb39bbe9138cf42bb4ded47b37fb3afe83c1945ac2", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_NAME.md": "bfd2506982a98615fa40d2a2760893f7634a270a663f65a84547e4a2ec7a4186", + "upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_SYNTHESIZE_METADATA_SEMANTICS.md": "4d64debe0b8657d6c0960eb5e3eebad7aa5125e34e35f66deb38c050f665ec26", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_CONTEXT_TO_STANDARD.md": "debf2fb145c10526e4952ba7cb322d8a88370ae136d1ae6f0b74b7b9d60734b0", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DATABASE_TO_STANDARD.md": "5bdb03e45720da5ff53297fae7ce476e6fd46f3af6277e8be147a87033917630", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DESCRIPTION_TO_STANDARD.md": "ec00677a4433b6110f529eafc28d5c0272746f186121ee563ab86eb8a3a9a6ec", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_FIELD_TO_STANDARD.md": "19c1e2c28c4df124d9ef56501635c31443552cc3d103d727b927db78891eb53f", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_TABLE_TO_STANDARD.md": "7fa123b49aca5ec7f2525d3697e0108aaf3d85f61c966eb9492a19b657e2a300", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_VALUE_TO_STANDARD.md": "d71db5f6adcf269897e8c9b662cc31edc18b7a7727db7a241624e319a390588e", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_COMPARE_STANDARD_CANDIDATES.md": "93213f34a0d765acb7a4925097fe6691c1e27aed539baa72893abf94872ec39e", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_REFINE_STANDARD_QUERY.md": "51c1ead62e9d83c194ac1fc24c8f43b9879497477b79054e03384b6ded980420", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_RESEARCH_STANDARD.md": "05ccd5f271a060ed497242c494c34d8ce3cac22f03e6349d0d6edfe0a4b303ac", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD.md": "dcbd3841440dc7bf0ee22c02b9ba1014fff1fa6a68900f270bb8cbb40aa9074c", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_CONTEXT.md": "6c1c9ad8fd7036ddf2504e30bc4f920d9754c8c3de68f2ff5263f760630d839e", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_DATABASE.md": "fc38fe4bccc6637de6abe3b8434f7d069d42b6eabf1a8dc29b98cfcdd443a362", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_FIELD.md": "98862fb615e97f189420455219b484ef990e0f4768f7ba4e49983e237d3998f2", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_TABLE.md": "ae067ae1570ad6e24e5c43a08aa4057223923b45e2593525255419144847ebd0", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SELECT_CATEGORY_TOPK.md": "c0dac66df87884c2ecd0e6dbfe2af1027b522a1653972ffa4d79e8a957a9bb74", + "upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SUMMARIZE_STANDARD_EVIDENCE.md": "ea3ee35fa9ddde244353f533a527bf25674ffb711218f74c8960862183bd664b", + "upstream/runtime/vendor/skill_reference.json": "752ed72d4ed6e987542197dfff40786ebd4487bcf6b1042cbf769d1fed2392b8", + "upstream/runtime/vendor/study.py": "d8c78d387e00b13762fb509d23e147210b0016931879c4af7089c593d6eddbb3", + "upstream.json": "ff08d44731fe8b3252879d9cfafa0ab60ba93d74330a4ddd6f5d8346afc57e90", + "vendor_snapshot.py": "fcb7ddf4ca32ba398e7d6bd88882500a353c2272979662d34e1dba866131fbbe" + }, + "publication_changes": [ + "api_backend.py", + "night_prepare.py", + "README.md" + ], + "change_scope": "README replaced; credential/tokenizer paths configurable without changing model/control/scoring logic", + "historical_identity_reusable_with_publication_copy": false +} diff --git a/research/vehicle_program_v1/results/diagnosis.json b/research/vehicle_program_v1/results/diagnosis.json new file mode 100644 index 0000000..1bb8512 --- /dev/null +++ b/research/vehicle_program_v1/results/diagnosis.json @@ -0,0 +1,71 @@ +{ + "scope": "397 final; descriptive diagnosis, no new model requests", + "raw_source_gold_and_metadata_mapping_verified": true, + "source_counts": { + "metadata_acquired": 379, + "terminated": 115, + "top5_any_hit": 214, + "no_match": 176, + "duplicate_blocked": 148, + "repeat_no_match_blocked": 142, + "budget_stop_including_backend": 282, + "no_match_despite_gold_candidate": 51, + "category_correct": 33, + "top5_compatible_entry_any_hit": 104, + "description_interpreted": 94, + "unavailable_type_attempted": 98, + "joint_correct": 26, + "output_length": 10, + "truncation_flag": 10 + }, + "events": { + "comparison_success": 100, + "finish_stop": 2553, + "comparison_error": 259, + "no_match": 232, + "comparison_blocked": 216, + "repeat_no_match_blocked": 210, + "finish_None": 17, + "no_match_despite_gold_candidate": 60, + "finish_length": 12 + }, + "no_match_stop_reasons": { + "planner_budget_exhausted": 9, + "validated_terminal": 18, + "program_budget_exhausted": 149 + }, + "intersections": { + "compatible_candidate_but_no_legal_terminal": 46, + "compatible_candidate_and_legal_terminal": 58, + "compatible_candidate_but_not_joint_correct": 78, + "joint_correct_without_compatible_top5": 0, + "no_match_then_legal_terminal": 18, + "no_match_then_joint_correct": 2, + "gold_compatible_no_match_without_legal_terminal": 44 + }, + "retrieval_groups": 412, + "distinct_leafs_in_five_entry_candidates": { + "5": 221, + "3": 84, + "4": 96, + "2": 11 + }, + "distinct_leaf_grade_pairs_in_five_entry_candidates": { + "5": 248, + "4": 103, + "2": 11, + "3": 50 + }, + "mean_distinct_leafs_per_five": 4.279126213592233, + "mean_model_entrypoints_per_source": 6.503778337531486, + "input_ambiguity": { + "exact_metadata_groups": 396, + "conflicting_joint_groups": 1, + "sources_in_conflicting_joint_groups": 2, + "max_consistent_joint_labels_by_exact_metadata": 396 + }, + "service_error_best_case_joint_bound": 0.10831234256926953, + "joint_excluding_17_backend_errors": 0.06842105263157895, + "legal_conditioned_joint": 0.22608695652173913, + "finance_published_legal_conditioned_joint": 0.41361256544502617 +} diff --git a/research/vehicle_program_v1/results/evidence-summary.json b/research/vehicle_program_v1/results/evidence-summary.json new file mode 100644 index 0000000..f3ade64 --- /dev/null +++ b/research/vehicle_program_v1/results/evidence-summary.json @@ -0,0 +1,96 @@ +{ + "experiment": "vehicle NONT-D / ProgSkill", + "data_repository": "DocAILab/DCG", + "data_revision": "84a9a462dcb828332f980ef121f40e1281ce1794", + "scope": "development Val397; all failures retained; not blind Test395", + "metrics": { + "denominator": 397, + "missing_or_failed_traces": 17, + "terminated": 115, + "category_correct": 33, + "grading_correct": 53, + "joint_correct": 26, + "top5_first_hit": 208, + "top5_final_hit": 208, + "top5_any_hit": 214, + "top5_compatible_entry_any_hit": 104, + "category_accuracy": 0.08312342569269521, + "grading_accuracy": 0.13350125944584382, + "joint_accuracy": 0.0654911838790932, + "termination_rate": 0.28967254408060455, + "macro_f1_supported": 0.21236454577082495, + "macro_f1_all_leaf_paths": 0.06757053729071703, + "supported_leaf_paths": 14, + "registry_leaf_paths": 44, + "complete": true, + "run_manifest_sha256": "a2d51f64c9403b67078e7d3e0e5bfc7120d616ff2a43ae1ef9680cce238d2cf3" + }, + "receipt_statuses": { + "completed": 380, + "backend_error": 17 + }, + "stop_counts": { + "completed:validated_terminal": 115, + "backend_error:planner_budget_exhausted": 16, + "completed:program_budget_exhausted": 219, + "completed:planner_budget_exhausted": 46, + "backend_error:program_budget_exhausted": 1 + }, + "resources": { + "model_entrypoints_recorded": 2582, + "known_usage_input_tokens": 29999427, + "known_usage_output_tokens": 271045, + "continuation_http_attempts": 2536, + "continuation_http_states": { + "response_received": 2524, + "transport_or_json_error": 10, + "http_503": 2 + }, + "continuation_known_usage_cny": "47.5121280", + "continuation_uncertain_reserve_cny": "0.654336", + "carried_pilot_usage_and_uncertainty_cny": "0.9031365", + "total_accounted_cny": "49.0696005", + "invoice": false, + "original_wall_start": "2026-10-19T00:50:48.906294+08:00", + "generation_completed": "2026-10-19T10:18:33.008919+08:00", + "wall_elapsed_seconds": 34064.10262489319 + }, + "provenance": { + "pilot_sources_preserved": 9, + "remaining_sources_generated": 388, + "resumed_after_preserved_sources": 332, + "resumed_sources": 65, + "old_332_receipts_unchanged": true, + "old_2095_requests_unchanged": true, + "infrastructure_recovery": "v7 to v8: cooldown reconstruction only; no reroll, budget or clock reset" + }, + "private_evidence_sha256": { + "plan.json": "384b40af8a6dc1831d336fc5176e73c85ca09783433357b2cdfbd1c2106351a4", + "run/run.json": "a2d51f64c9403b67078e7d3e0e5bfc7120d616ff2a43ae1ef9680cce238d2cf3", + "run/night-status.json": "006ee674b7d1247d64ac336729619e1b94f3025cbc1159968d992d834a199389", + "run/backend/api-budget.json": "51c697d2dab97083cc552e0339bb7e01ab1b60df657d824a1d12e4cc3dac6a1b", + "score.json": "b58489e9f9134f83a7201da0c9b05eb59ceb0fc9620f2669aef4f9ccc63cedcc", + "replay.json": "572e0d7731bc4a397006dbd6ae63249531d2dba955cd4206671f9f52ff0496e9", + "supervisor.json": "2ea1311f344cabf628f113a465f1b093350af4325dcd5623c8d09606ff4c1c6f" + }, + "validation": { + "receipt_file_integrity": "397 verified", + "independent_primary_scoring": "pass", + "controller_replay": { + "completed_sources_replayed": 380, + "receipt_count": 397, + "expected_count": 397, + "model_calls": 0, + "pass": true + }, + "ledger_reconciliation": "2536 attempts verified", + "worker_exit_code": 0, + "worker_and_supervisor_exited": true + }, + "limitations": [ + "Hosted weight precision/revision not exposed; no local FP16 equivalence claim", + "9 pilots used earlier service scheduling profiles; all results retained", + "59 standard entries / 44 leaf paths: Top5 entries are not necessarily five distinct categories", + "A/B/C cross-method alignment pending; no SFT/RL result in this release" + ] +} diff --git a/research/vehicle_program_v1/results/files.sha256.json b/research/vehicle_program_v1/results/files.sha256.json new file mode 100644 index 0000000..b637ddc --- /dev/null +++ b/research/vehicle_program_v1/results/files.sha256.json @@ -0,0 +1,164 @@ +{ + ".gitattributes": "e02c0835e8bc53f09cdc2a1e4417019a8ff3da0d8d48b8369e8cdcf340a3530d", + ".gitignore": "a503cbd30a8d024ca267f473dd9cc82016b378cd0d5f873354a476838fbfb1a7", + "BRIEF.zh-CN.md": "e930ec6709068846626f571b9023335a0e53ea068251e5957f14b0cd48648132", + "code/api_backend.py": "d43e8913ab3507ac675967ecf870a99c3aa9c1bc88cb7b432b3ca28c14fd01c2", + "code/api_check.py": "bca1d75b8d865f3501dc79e7e61d648e2b53899bc108cdf3f17eb729014992b0", + "code/api_discover.py": "3a47a1dfb2bd4ddb8bd0741a7ca7f3b96ca73e85ca62affd93de409f44f29e2b", + "code/api_offline_audit.py": "38b71f0a322008f85d72db2b3c9ea2057600889864c079db4a3e3ace2b30cb88", + "code/api_parallel.py": "997d9446a5079aa8617ba54dc28551ed70cf8ccbd00b902f5824eb7ebd5182fc", + "code/api_prepare.py": "d41bba6844a9a3d5fedee642dc842c2f8b2008fa151ae0ae02b4d09f73a97794", + "code/artifacts.py": "a275c2d4d855509aff039999e09d9015f75b4cc29583a53d659624232ba40f64", + "code/backends.py": "1de00b39e5f834a646ffb35c04bc80d233bd0ee281162d5dbfa5a10e7a9f39da", + "code/bundle.py": "35084ac081219e785db8aeb8c4a4fd54caa7d9dc1f8e7c823a209fe3408d3f6f", + "code/Night-Control.ps1": "c84e01dc7f6ad328d632078d73fc3ad842386901a0d16ac10f35405d444ca912", + "code/night_ops.py": "754e6fb1935a60893ed3ded35a2937d5856381e504464160fb55e40b09cf70b5", + "code/night_prepare.py": "c9f11d7ee172d41f5b9404083d7184ee38bc569870cd4d5b66fbe6b10993d513", + "code/night_resume.py": "17e9db5de8d9887483cb0215a555b76ed1cf15e76ed010011a9f8675cc1492b6", + "code/night_runner.py": "1680180706aa665dd58815100b9e173ae2f0f801549f9c1084b815778b1bdae2", + "code/night_supervisor.py": "108bc00c65020b9d63a60e28397bdd1ee93173b970eea818285e99d07301a08a", + "code/pinned-inputs.json": "b102eaa6b614e5246ddccf9eabe027117cbe27422da947065c14cb6fc4617e32", + "code/prepare.py": "6d7cf76783cf16a1a311ba1d702713ee023405be33706a000fb98c17fcdae426", + "code/README.md": "b5da0ab8eff6d0fafedff06d0b4c69ca8ce3c3a3835544fbf6610ce86c645e4d", + "code/replay.py": "3815818e3b9b093e6d20c53aedc21863931974ac7a71b43dbdbafda7e4a18e07", + "code/runner.py": "06eac150c6864d1a051311a4783961f342fdd907efc54d21616392f44db48912", + "code/score.py": "18378a86f09893aa9aa2ef18246318fcd82687f32fad965521b0c086ae4bd2fc", + "code/tests/test_api.py": "ccd557fc0e43cfa3e07ea327d812560f0ba1105f3118e7d2451980c2f2fae6ce", + "code/tests/test_night.py": "0df57ab7835b40d418b37a4bb2c54c2bf1a66d63fc30d6708972567388bb6750", + "code/tests/test_night_recovery.py": "33bcc68cb78fcd69d2c2169e8a08cfa4f0b482a7f15ad716c2c8aef6ac852781", + "code/tests/test_parallel.py": "e7accf1a049be28f1c9dc57cbad7489fd7894676bff34f0c42c0d3617a53b342", + "code/tests/test_vehicle.py": "5c6bf49df62d1562c760398d98fc166c323da493dd193ab86a4af0c2f7097b7f", + "code/upstream/backend/download_model.py": "5e435d6f87d920d58ff6d305afe61be3a98fec8d12b6f79986072a9a208a8edd", + "code/upstream/backend/local_smoke.py": "6517f5bc9bbb8c5fa4bc7e9100d1c4cf65a7829b4d42543e837d5798ad81f8ef", + "code/upstream/runtime/program_loop/__init__.py": "8ebc1b9a3937a1848c199126edf97e5862741ede120a491e87a8473d6c0f7c46", + "code/upstream/runtime/program_loop/__main__.py": "935a1c1166b0c1ea35a82256345000bf2c73ded718d77773bc27a71ecce28f7d", + "code/upstream/runtime/program_loop/_vendor.py": "cfb187e38a0b8acd86e83476ee24f3aa02928c86fd3deb532f752ae8784bbf80", + "code/upstream/runtime/program_loop/catalog_context.py": "7e08a466d701eb8b67733629c7af4723f9baf1eedf16ba768a78ba4d6d0e10cb", + "code/upstream/runtime/program_loop/cli.py": "82d3841baa287c2d5e0cf519ae2532a4fd5a4d79e89d907e29ad029c17d59335", + "code/upstream/runtime/program_loop/contracts.py": "04f584b9acfb033fb1cbd38c8f08595f46bd3d4c57a55495fc282920bd9169a4", + "code/upstream/runtime/program_loop/controller.py": "a3a8ad54e30a0fb623cf9524bc766c4ee6d334de84648aaec77addda3f0c93e1", + "code/upstream/runtime/program_loop/execution.py": "960800db0c9f49ca7db61c9f20fa3967f6373d2df000da1a7eb253d150efa72e", + "code/upstream/runtime/program_loop/failure_policy.py": "92282fb6456a3df91deab50c141964abbad68741e948116ef0e4a3ad8057762f", + "code/upstream/runtime/program_loop/planner.py": "6edeab5d5164663a6e9f1ac5c31848836469f970b7accbeff937e033dc80e22b", + "code/upstream/runtime/program_loop/planning_contracts.py": "b0cd81f6c0aac6c0ef09bf460176522683f9612b908de06f0841abd16958e312", + "code/upstream/runtime/program_loop/prompt_encoding.py": "20862c2334a809ac06629058495eaaf4e94a601345dbcdbd5b6d573df07ebebc", + "code/upstream/runtime/program_loop/recovery_context.py": "0d67d47e51f55edc4bbffe1164327c7f74d7aff1b629dae6df3e5c9c201af2f7", + "code/upstream/runtime/program_loop/rejection_context.py": "fd71ad424ff4a201b01f4db7276f2d958fd94eba7aaf26deb16cc54dbcb9de1f", + "code/upstream/runtime/program_loop/validation.py": "c96220f85cc14375ce6171dc59c2ff5d46389017ca9549cb41ff00cda82b51eb", + "code/upstream/runtime/runtime_identity.py": "a183d837ec5f666ea8bd0c1965ff0004d63be74cd1c369505cd7e6ddcb2dccd1", + "code/upstream/runtime/tests/test_grounding.py": "8bc06990233ef500ea3b7cd572c65b0dd7ca14565c5269ec25f2a0ced649e716", + "code/upstream/runtime/tests/test_integration.py": "c38e9d6b4c0cae512637c050c951770d5bca35df519927ab2ce01905941946f9", + "code/upstream/runtime/vendor/common.py": "c79a02e987a169436796f438a415d7c309c0e0cf50bb447e493cd68cb9301fa4", + "code/upstream/runtime/vendor/comparison_output.py": "ed7b3bb552aeaff329ab4531bfa4fbd93772bf560475dc4d3fb848c12bee68e8", + "code/upstream/runtime/vendor/evidence_roles.py": "6b76fbc8696a70d9ed5723a4ba71edc4de540f29e883d5ee5f99d9f806536064", + "code/upstream/runtime/vendor/grounded_semantics.py": "4f2f764e2256ad1dda8716f42277e688fe710fbb607e310cdbe57995e43c254c", + "code/upstream/runtime/vendor/pool_availability.py": "c6fa8e23a678b4451ef78d301efb347507fb0513a9d18bfda76eeefc824d70aa", + "code/upstream/runtime/vendor/pool_bindings.py": "3c0c65e78f7bcdc5bb81003b455fd957dba5c4abdbfdaa20b8fddb3b155e07db", + "code/upstream/runtime/vendor/pool_catalog.py": "bd19db2c908d41120498eae4afc26e30726b60ddf7fbf834e77ca6251e8c8a82", + "code/upstream/runtime/vendor/pool_metadata.py": "0705318048785fd34edd970274145dec359269e5e30756a996eb156e168e3a8a", + "code/upstream/runtime/vendor/pool_runtime.py": "ae5f3927e4b4faa5a7808c9ae9975af9ae9ef7d7f2f347fd034f34cadf2e9f45", + "code/upstream/runtime/vendor/reference_skills_63.json": "a63b6ee85f983caf9f55d537fe85d24a283a860ccb0985c8a7fce8b1466e3caf", + "code/upstream/runtime/vendor/runtime/code/agent/__init__.py": "c2ee5b379bec6f6381e0651bee6078e38ad6ca294f497d40b6d27e583fd1492e", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/__init__.py": "64f470fa2d85450ebbd8819a66c63064bcdbf993c829d248c64315285e7aca73", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/agent_loop.py": "176bc72eebd17a7d2e45276654c49cbd5b1828306e11d2579faffe143eebcade", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/artifact_projection.py": "d71939b3e4b04b83d33f7aedbc3173cd44043b8a55d44c149c46f35d5aa3f7d8", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/catalog.py": "66ceb63dd02030cecb3965247a671da64c4d29724ece9a5eb52961a0647c8e6a", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/category_catalog.py": "baf7b0b234822060b48dded26f00d526654749f1384c3e23f9eb07ddc8fa0a58", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/contracts.py": "19ed69938bdc517be14625e0ef886945dc4483faeb6a8260fdfcacaffa83f233", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/metadata_lookup.py": "e338e0d804caa20218637074422b08f26b16fdbdd9faff39077a1cb73beb89fc", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/metrics.py": "fffb69bd3e8efa149e46e3413b4056457840b9cd8f6798fd5e0871cdb7e3a0b3", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/model_skill_executor.py": "6ad63e1de6cb77bbf6d3cc537f907b59e94147b7e58de885d0d0c43bdb31671b", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/planner.py": "5be67a873f8fa54c183c9fa381def425b73cc92181fab6079506eb2296fa53d8", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/retrieval.py": "fbe140936695f38c6bc6838c5a9a0100ca9e6f98b6e77984e587fdbea28b6bf4", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/runtime.py": "887e82115154f043801719b5e292895e3eeef0d25e277748805a0c1dabc39acc", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/skill_loader.py": "7528bada337e98b69ef61ccf558574f5940d26fcfc2106530f8c9d0d765964f6", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/table_field_lookup.py": "93319c4ee044439f9e25449cec220dfaededa1c6b799816486d19402b569dc22", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/topk_selection.py": "a58411d1acb421bd5e5c134f4ceef7ccce76851f684a397446904b7a17b1eb4c", + "code/upstream/runtime/vendor/runtime/code/agent/orchestration/validator.py": "15fd2b9faca5addf49f4fdf5d816eb2ecb3723c6b800da3fd591cda93b0bedcd", + "code/upstream/runtime/vendor/runtime/code/script/orchestration/__init__.py": "7c61249a27a9c7ad011f983e9857e3889ae54837d00faf2660fe2f72350b30d6", + "code/upstream/runtime/vendor/runtime/code/script/orchestration/run_agent_loop_batch.py": "6605815bf479e8fc50a3041af3c2430b3f121740bf38b8e6d497212d4425616d", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_ASSESS_CATEGORY_SUPPORT.md": "85ea64633013f69d582707acf4ab1b32201cf5ebbf7e0356041119d6c08c106a", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_CHECK_EVIDENCE_SUFFICIENCY.md": "f879c21470b9daeb773a80a256a07f7a6f43e38e8452347f128eca6391362f4a", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_COMPARE_CATEGORY_HYPOTHESES.md": "3fa31691df6bb353b4171a8c837da229549f8f4eff827bf502f81cb2318432a5", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_DECIDE_NEXT_EVIDENCE_SOURCE.md": "647a447ae584d92e67c17d2d7040ff48d14cb8d2b5039345d65eb072b5c14fd3", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FINALIZE_CLASSIFICATION.md": "cd66513c1ca4b6728bd217ae57a7ad7cd96257f61443a7af2ac1edd7e746f937", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_CONTEXT_EVIDENCE.md": "b0028a313fa3f81350f07093c5389c7ec0447539119e37930f1f05bceec539d1", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_DATABASE_EVIDENCE.md": "e27e9b2f57b5b1929d5286a10cc15233ab48ba40948fe520fde8853f66bc1e56", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_FIELD_EVIDENCE.md": "6dfef164297cb75a02f64ebbdafecb3f0519f1a5f1992c3bb9a98157b7a7ee54", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_METADATA_EVIDENCE.md": "e3a7e0de334bc39836c3a909edc79233075b0b73f7c30e879d4e087bc94aa6f2", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_STANDARD_EVIDENCE.md": "eddac25de1c3dda0baf876d29a7a32ae4fab980f2ae7ce892def82af3469087a", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_TABLE_EVIDENCE.md": "24c7eb0ea3381b640465e78e41eb2ff43495e7bb3eadcc652e5587c3bd167ab4", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_IDENTIFY_MISSING_EVIDENCE.md": "237695b8b5d6f9a7105902dc32a120297c364ee5f8d1c27f4ead212df110668f", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY.md": "032b758517d3a8535c29d20d8391bd8d27a2ad1d5c7e9f62246578076ddb5197", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY_FROM_TOPK.md": "7af18e11373d633fe4369338336c653b47238c214cb5f1937e0a5b30041f722f", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_INFER_GRADING_LEVEL.md": "276b5e1a8b080d9aecec762762939372f40e56c8fb4b1eb92964c53073b6d603", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_FIELD_CONTEXT_CONFLICT.md": "94e552edccadc03cb55a5ab8e54511a296971363bd4aafb30b3ab1385e5a18f3", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_METADATA_CONFLICT.md": "643472dc5b7e7cdb38f3483857acc94d539884efede9b9ed0f4cccc79a037d0b", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_STANDARD_CONFLICT.md": "be9b9c9c879b964acf4b5f7317dcf90c68833a1f98f6cb8ab9f8199460066b25", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CATEGORY_WITH_STANDARD.md": "deb07baab54b052a9323b475c912d7743115c96dd4b1dd0b7c4a80df3e3d0dfc", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CLASSIFICATION_GRADING_CONSISTENCY.md": "ff8ef17c1ce1408496c6634389ecba87f6d282d0264e4ac1ee9a8c2deb12bab1", + "code/upstream/runtime/vendor/runtime/skills/full_pool/evidence_reasoning_WEIGH_EVIDENCE.md": "c928a03a7a8f2d8b1a28ffb01c100543000a62e4a388a0e60051a4b6815dd43f", + "code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_ACQUIRE_MISSING_METADATA.md": "cc788f17f7c63e1f7974f220eca3becfb186d3df287a35d814a5387964d20ce3", + "code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_CHECK_METADATA_AVAILABILITY.md": "fe27b6f518a98be4e88414d6180180954572f222fec37dfc4591e2b72a05ad5a", + "code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_EXPAND_SELECTED_CONTEXT.md": "323b136804a9db277d6642b71de277ae3fcd7021910645475872b1b018d79a79", + "code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_DATABASE_CONTEXT.md": "0008e4dc6b220757208cb656912ebb609c8230aaf62ce56a33f247dc5879f7cd", + "code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_TABLE_CONTEXT.md": "09b1f136ca45244c603e98aba0bec6fad8997ca2e618eeaeaf7ac1c1de4fb711", + "code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_DESCRIPTIONS.md": "0b5f7d9ced8902a3a7157d6781a3b3d6bc9fb9f2d2c55da99f31977ceb5609b8", + "code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_TYPES.md": "61cfe7efeb0980d22e72c4069e70cb91d1a007c7352b528b600b7efbcfd313e5", + "code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_VALUES.md": "780305d46253a0ab132f35feee6ba420347f351326827c4be781bd4224349504", + "code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_DATABASE_DESCRIPTION.md": "37b2146b9cd669e1dc0cb0a54c0f6ee36dc0157ce1e4e753d96be08a6fdada21", + "code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_DESCRIPTION.md": "d545962070cb10fa90ee1693261d48ee08df616cad4d098bfcd98643100fce7b", + "code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_METADATA.md": "372ce861fcca6138cda43b693a8fc32f85c0ec380b0c275c3cfe613dbe25f629", + "code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_DESCRIPTION.md": "912334b76f95c6f468cf4bab840ae0ec3a262639c59e545db63bfce866710ea0", + "code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_FIELDS.md": "99c01c5869b511754feccfd335017c7f4290683fbcc46ed6355b95f775d605b0", + "code/upstream/runtime/vendor/runtime/skills/full_pool/metadata_acquisition_SELECT_CONTEXT_FIELDS.md": "aacb11c41507d2c2743e56a7915c9dae9d6b4e2ff3ece6fec6b3d73babeb6683", + "code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_DISAMBIGUATE_FIELD_SEMANTICS.md": "e497bddaf99da95503677a969926921b507b92ad542055dafaa88ee724a235d0", + "code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_DATABASE_ABBREVIATION.md": "2338078f06cd5276b326747fa0b80790d3c638ba369f6579e4ddcff84882eb51", + "code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_FIELD_ABBREVIATION.md": "fc8d4265d620c57c6d181e395aa08a0f19319f86437739037659b29016e15fad", + "code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_TABLE_ABBREVIATION.md": "6ce033822d4d4f4ae7babba7bd0b44fa871476353b81fd86ac6742d266b3a426", + "code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_CONTEXT_FIELDS.md": "297bf09515bbbc0a4b9e6f80fb77c7135b7484c5c428da0868d7a0d0732717cb", + "code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_DESCRIPTION.md": "acc57509cff6330476522f8fc5a7b286b0ede3e477c9806000b54542f89070dc", + "code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_NAME.md": "e427e888ac10d742784d390ad2bdce40d0aec7c9a7abd52b5f5497fc83b866fc", + "code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DOMAIN.md": "6e1760ff387797cb4c171f404f744b6b6b87445969659bf8b37c8bca66de5375", + "code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_DESCRIPTION.md": "fdbd8cf655fdc0ecddafd9ce7a73f4996d7b4698405e87d8a41e134c7d85adac", + "code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_NAME.md": "861a87c3342531c281779ac572c118f92a4c4b9e13a43e5af3eb39620e5e1d39", + "code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_TYPE.md": "93500ee995113eed8ee9d6d8887d6f65f2d4aa6ac52b12697fb8e82982c1b49d", + "code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_VALUE.md": "c0dad806fde006487b38b9d56fc0b5fde6cc2e367b8a46d13adb5c4171ed19ad", + "code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_DESCRIPTION.md": "901cef9276dd30ff1b2e35fb39bbe9138cf42bb4ded47b37fb3afe83c1945ac2", + "code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_NAME.md": "bfd2506982a98615fa40d2a2760893f7634a270a663f65a84547e4a2ec7a4186", + "code/upstream/runtime/vendor/runtime/skills/full_pool/semantic_interpretation_SYNTHESIZE_METADATA_SEMANTICS.md": "4d64debe0b8657d6c0960eb5e3eebad7aa5125e34e35f66deb38c050f665ec26", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_CONTEXT_TO_STANDARD.md": "debf2fb145c10526e4952ba7cb322d8a88370ae136d1ae6f0b74b7b9d60734b0", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DATABASE_TO_STANDARD.md": "5bdb03e45720da5ff53297fae7ce476e6fd46f3af6277e8be147a87033917630", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DESCRIPTION_TO_STANDARD.md": "ec00677a4433b6110f529eafc28d5c0272746f186121ee563ab86eb8a3a9a6ec", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_FIELD_TO_STANDARD.md": "19c1e2c28c4df124d9ef56501635c31443552cc3d103d727b927db78891eb53f", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_TABLE_TO_STANDARD.md": "7fa123b49aca5ec7f2525d3697e0108aaf3d85f61c966eb9492a19b657e2a300", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_ALIGN_VALUE_TO_STANDARD.md": "d71db5f6adcf269897e8c9b662cc31edc18b7a7727db7a241624e319a390588e", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_COMPARE_STANDARD_CANDIDATES.md": "93213f34a0d765acb7a4925097fe6691c1e27aed539baa72893abf94872ec39e", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_REFINE_STANDARD_QUERY.md": "51c1ead62e9d83c194ac1fc24c8f43b9879497477b79054e03384b6ded980420", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_RESEARCH_STANDARD.md": "05ccd5f271a060ed497242c494c34d8ce3cac22f03e6349d0d6edfe0a4b303ac", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD.md": "dcbd3841440dc7bf0ee22c02b9ba1014fff1fa6a68900f270bb8cbb40aa9074c", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_CONTEXT.md": "6c1c9ad8fd7036ddf2504e30bc4f920d9754c8c3de68f2ff5263f760630d839e", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_DATABASE.md": "fc38fe4bccc6637de6abe3b8434f7d069d42b6eabf1a8dc29b98cfcdd443a362", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_FIELD.md": "98862fb615e97f189420455219b484ef990e0f4768f7ba4e49983e237d3998f2", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_TABLE.md": "ae067ae1570ad6e24e5c43a08aa4057223923b45e2593525255419144847ebd0", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SELECT_CATEGORY_TOPK.md": "c0dac66df87884c2ecd0e6dbfe2af1027b522a1653972ffa4d79e8a957a9bb74", + "code/upstream/runtime/vendor/runtime/skills/full_pool/standard_alignment_SUMMARIZE_STANDARD_EVIDENCE.md": "ea3ee35fa9ddde244353f533a527bf25674ffb711218f74c8960862183bd664b", + "code/upstream/runtime/vendor/skill_reference.json": "752ed72d4ed6e987542197dfff40786ebd4487bcf6b1042cbf769d1fed2392b8", + "code/upstream/runtime/vendor/study.py": "d8c78d387e00b13762fb509d23e147210b0016931879c4af7089c593d6eddbb3", + "code/upstream.json": "ff08d44731fe8b3252879d9cfafa0ab60ba93d74330a4ddd6f5d8346afc57e90", + "code/vendor_snapshot.py": "fcb7ddf4ca32ba398e7d6bd88882500a353c2272979662d34e1dba866131fbbe", + "CROSSCHECK.zh-CN.md": "0e98a297dd9b0bcca2c0953e9b0805f3851b3bc778d45db980e6f27e259c5385", + "publication_tests/test_publication.py": "7be6f0b282469a81f853d648d0ef25fb6d8610a3f58d2d2dc72904ef9b95625e", + "README.md": "1a551361b1d8a32cf6781e7142408357d0c84f18526e1c00dc281b198a5d7e97", + "REPORT.zh-CN.md": "f9a02132713e7957bd4b14155bd71a2a979fe690c6e8b2d9e47e523332b76446", + "REPRODUCE.md": "c2775719c10390851900ec4a11d16d9b392d310cf0ecff0cd6fc03d6aba0a6e4", + "results/code-publication.json": "9c841e0e94e4c683321e9e80baf2e8012ebe382cf8e1698882f3ab561c1e5171", + "results/diagnosis.json": "26686508e005d531b34a27d91af5db67b23bbe1a6aafa6634b1e07dacd8771d5", + "results/evidence-summary.json": "fd49df5afb07cf5888ec57989b01cda8d128cce31e220c4e8a08fbf5b7cb3ac5", + "results/model-roles.json": "37b16e346be89c9902cc72feaf87ca0713ff44745127acf740117eb0a61b991a", + "results/publication-verification.json": "c374616ca9be7034ab1e9e3ac06421144385cd573bba78758a78eef03c2db2b6", + "results/trajectory-export.json": "a7817d5372da40b1fb1b4fffa2f47ab8b8c770fa9a55d2ec9c89a60ff93ed4ca", + "tools/redact_trajectories.py": "171ae722c634b9a8bc3eabf65cb4dfe9bf66530e729f309fb4782ff360ed351e", + "tools/verify_release.py": "4e553eb7e0daa21538b402973765d8a53021f9a47d9b0b947f35f0b8bde85a3b", + "trajectories/val397.redacted.jsonl": "0cc031e9bbad74d43e2fd362d89327193a260aeae2b997402d02e57428a9ce1d" +} diff --git a/research/vehicle_program_v1/results/model-roles.json b/research/vehicle_program_v1/results/model-roles.json new file mode 100644 index 0000000..69795d6 --- /dev/null +++ b/research/vehicle_program_v1/results/model-roles.json @@ -0,0 +1,50 @@ +{ + "planner": { + "model": "Qwen/Qwen3.5-9B", + "provider": "SiliconFlow", + "adapter": null + }, + "model_skills": { + "model": "Qwen/Qwen3.5-9B", + "provider": "SiliconFlow", + "adapter": null + }, + "hosted_revision": "not_exposed", + "hosted_precision": "not_verified", + "provider_seed": "not_requested; unsupported", + "decoding": { + "temperature": 0, + "enable_thinking": false, + "max_output_tokens": 512, + "context_cap": 32768 + }, + "budget": { + "max_programs": 3, + "max_program_steps": 10, + "max_skill_steps": 12, + "max_planner_attempts": 4, + "max_identical_calls": 1 + }, + "scheduler": { + "workers": 2, + "tokens_per_minute": 72000, + "requests_per_minute": 900, + "token_margin": 128, + "window_seconds": 60.0, + "window_guard_seconds": 0.1 + }, + "initial_fields": [ + "domain", + "field_name" + ], + "tool_metadata_allowlist": [ + "field_name", + "field_description" + ], + "candidate_unit": "standard_entry", + "stop_limits": { + "estimated_total_cny": 70, + "original_wall_hours": 14 + }, + "reference_tokenizer_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a" +} diff --git a/research/vehicle_program_v1/results/publication-verification.json b/research/vehicle_program_v1/results/publication-verification.json new file mode 100644 index 0000000..ca98eed --- /dev/null +++ b/research/vehicle_program_v1/results/publication-verification.json @@ -0,0 +1,39 @@ +{ + "publication_date": "2026-10-19", + "code_tests": { + "passed": 55, + "failed": 0 + }, + "upstream_tests": { + "passed": 8, + "failed": 0 + }, + "publication_tests": { + "passed": 5, + "failed": 0 + }, + "total_tests_passed": 68, + "network_or_model_calls_for_publication_tests": 0, + "credentials_read_for_publication": false, + "privacy_scan": { + "pass": true, + "checked_private_values": 741, + "full_real_source_ids_checked": 397, + "raw_metadata_values_minimum_length": 8, + "all_free_text_omitted_by_schema": true, + "credentials_read": false, + "findings": [], + "substring_matches_in_byte_identical_preexisting_public_upstream": [ + "code/upstream/runtime/runtime_identity.py", + "code/upstream/runtime/vendor/grounded_semantics.py" + ] + }, + "historical_full_controller_replay": { + "completed_sources": 380, + "passed": true, + "backend_error_receipts_preserved": 17 + }, + "historical_independent_scoring_and_ledger_reconciliation": "pass", + "public_semantic_accuracy_independently_reproducible_without_private_gold": false, + "git_byte_preservation": ".gitattributes -text" +} diff --git a/research/vehicle_program_v1/results/trajectory-export.json b/research/vehicle_program_v1/results/trajectory-export.json new file mode 100644 index 0000000..be0576d --- /dev/null +++ b/research/vehicle_program_v1/results/trajectory-export.json @@ -0,0 +1,17 @@ +{ + "files": [ + { + "path": "trajectories/val397.redacted.jsonl", + "rows": 397, + "bytes": 1277255, + "sha256": "0cc031e9bbad74d43e2fd362d89327193a260aeae2b997402d02e57428a9ce1d" + } + ], + "full_replay_supported": false, + "training_data": false, + "gold_included": false, + "arguments_included": false, + "source_metadata_included": false, + "private_source_id_mapping_included": false, + "aliases_resolvable_to_standard_labels": false +} diff --git a/research/vehicle_program_v1/tools/redact_trajectories.py b/research/vehicle_program_v1/tools/redact_trajectories.py new file mode 100644 index 0000000..9108d8e --- /dev/null +++ b/research/vehicle_program_v1/tools/redact_trajectories.py @@ -0,0 +1,56 @@ +"""Allowlisted lossy projection; not a training set or full replay artifact.""" +import hashlib +import json +import re + + +def public_trace(trace, public_id, trace_sha256, receipt_status, allowed_tools, category_aliases, grade_aliases): + if not re.fullmatch(r'val-\d{4}', public_id) or not re.fullmatch(r'[0-9a-f]{64}', trace_sha256): + raise ValueError('invalid public identity') + statuses = {'accepted', 'rejected', 'error', 'blocked', 'success', 'completed', 'terminated', 'backend_error', 'interrupted'} + stops = {'validated_terminal', 'program_budget_exhausted', 'planner_budget_exhausted', 'skill_budget_exhausted'} + def status(value): + return value if isinstance(value, str) and value in statuses else 'other' + def number(value): + return value if type(value) is int and 0 <= value <= 10000 else None + def call(value): + value = value if isinstance(value, dict) else {} + tool = value.get('tool') + return {'tool': tool if isinstance(tool, str) and tool in allowed_tools else '[REDACTED_TOOL]', + 'arguments_omitted': True} + def program(value): + if not isinstance(value, dict) or not isinstance(value.get('program'), list): + return None + return [call(step) for step in value['program']] + attempts = [] + for attempt in trace.get('planner_attempts', []): + raw = attempt.get('raw_response') + try: + parsed = json.loads(raw) if isinstance(raw, str) else raw + except (ValueError, TypeError): + parsed = None + attempts.append({'attempt': number(attempt.get('attempt')), 'status': status(attempt.get('status')), + 'tools': program(parsed), 'unparseable': program(parsed) is None}) + programs = [] + for item in trace.get('programs', []): + programs.append({'program_index': number(item.get('program_index')), 'tools': program(item.get('plan')), + 'status': status(item.get('status')), 'terminated': item.get('terminated') is True, + 'failure_index': number(item.get('failure_index')), + 'steps': [{'call': call(step.get('call')), 'status': status(step.get('status'))} + for step in item.get('steps', [])]}) + final = trace.get('final_result') + projected_final = None + if isinstance(final, dict): + category, grade = final.get('category'), final.get('grading_level') + projected_final = {'category_alias': category_aliases.get(category) if isinstance(category,str) else None, + 'grade_alias': grade_aliases.get(grade) if isinstance(grade,str) else None} + stop = trace.get('stop_reason') + budget = trace.get('budget', {}) + return {'public_id': public_id, 'source_trace_sha256': trace_sha256, + 'privacy': {'lossy': True, 'arguments_and_state_references_omitted': True, + 'category_and_grade_are_opaque_aliases': True, 'not_training_data': True, + 'full_replay_supported': False, 'gold_and_private_source_ids_included': False}, + 'receipt_status': status(receipt_status), 'terminated': trace.get('terminated') is True, + 'stop_reason': stop if isinstance(stop,str) and stop in stops else 'other', + 'planner_attempts': attempts, 'programs': programs, 'final_result': projected_final, + 'budget': {key: number(budget.get(key)) for key in ('programs','planner_attempts','skill_steps') if key in budget}} diff --git a/research/vehicle_program_v1/tools/verify_release.py b/research/vehicle_program_v1/tools/verify_release.py new file mode 100644 index 0000000..165c65b --- /dev/null +++ b/research/vehicle_program_v1/tools/verify_release.py @@ -0,0 +1,24 @@ +"""Verify public artifact bytes and completion counts; no model or private data.""" +import hashlib +import json +from pathlib import Path + + +def verify(root): + root=Path(root).resolve() + manifest=json.loads((root/'results/files.sha256.json').read_text(encoding='utf-8')) + for relative,expected in manifest.items(): + path=(root/relative).resolve() + if not path.is_relative_to(root) or hashlib.sha256(path.read_bytes()).hexdigest()!=expected: + raise ValueError('publication file identity differs: '+relative) + rows=[json.loads(line) for line in (root/'trajectories/val397.redacted.jsonl').read_text(encoding='utf-8').splitlines()] + summary=json.loads((root/'results/evidence-summary.json').read_text(encoding='utf-8')) + assert [row['public_id'] for row in rows]==[f'val-{i:04d}' for i in range(1,398)] + assert sum(row['terminated'] for row in rows)==summary['metrics']['terminated']==115 + assert sum(row['receipt_status']=='backend_error' for row in rows)==17 + return {'pass':True,'verified_files':len(manifest),'public_rows':len(rows), + 'independent_semantic_accuracy_verified_from_public_rows':False} + + +if __name__=='__main__': + print(json.dumps(verify(Path(__file__).resolve().parents[1]))) diff --git a/research/vehicle_program_v1/trajectories/val397.redacted.jsonl b/research/vehicle_program_v1/trajectories/val397.redacted.jsonl new file mode 100644 index 0000000..4044e22 --- /dev/null +++ b/research/vehicle_program_v1/trajectories/val397.redacted.jsonl @@ -0,0 +1,397 @@ +{"public_id":"val-0001","source_trace_sha256":"52dd8352c5c6ffb7484db2a6cae2dbbe301476e353855ccab9a8a8d8bea4ddaf","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-057","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0002","source_trace_sha256":"b2685c3c1686f17b16ae296f868badecefd8694e3cbefef08714a5a4b1c8917b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0003","source_trace_sha256":"4aea1d3619bae29ebabd0299408a4ce824f394dd8f525e30f7f9b433ee500573","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"error","tools":null,"unparseable":true},{"attempt":3,"status":"error","tools":null,"unparseable":true},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":1,"planner_attempts":4,"skill_steps":4}} +{"public_id":"val-0004","source_trace_sha256":"dc0790df3366418d318e59857eeac962973d0ff66f0a32a2fd2af6faf3b29499","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-055","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0005","source_trace_sha256":"8b4eba401d977addcddb4bfb06775f6c897f0df0221912dadc936ded657fad9e","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"error","tools":null,"unparseable":true},{"attempt":2,"status":"error","tools":null,"unparseable":true},{"attempt":3,"status":"error","tools":null,"unparseable":true},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0006","source_trace_sha256":"e0a93772a491a7a4cf9f809e86146eb6e2ab8fa664f8d1e15654bdfc56c4d1bc","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0007","source_trace_sha256":"2da5e5f779c15567a4f7600069e1c51d8c5623eaa0175403bd047f6682ad66d2","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0008","source_trace_sha256":"4f9077f423a70d9865a0dd34e8e517e90239de00ea45a5152bb56d50ec2d1d43","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"error","tools":null,"unparseable":true},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0009","source_trace_sha256":"a087dfa1a33302e34e025db21b4fb4a07010d5867c50dcc740d790cf81722c5c","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"error","tools":null,"unparseable":true},{"attempt":3,"status":"error","tools":null,"unparseable":true},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":1,"planner_attempts":4,"skill_steps":4}} +{"public_id":"val-0010","source_trace_sha256":"4c545d83b4924594354046a45184af9618734cdfc29df715f12a8e8e1033fff0","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0011","source_trace_sha256":"ecbcd4eb6560e72f827233d7f3705fc6eb9be021cb06fc613483bc437910be87","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0012","source_trace_sha256":"d9c88ae762b65622c95ca040c3f94f072a3cc21a92f565e44288cbd33f898b44","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-034","grade_alias":"grade-002"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":8}} +{"public_id":"val-0013","source_trace_sha256":"7e7b7cef356e7fb8e5149203efd827a6995b303ddb45aec079941018dc20a670","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-055","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0014","source_trace_sha256":"a36359036959efee8f9a0c4a22eb8a95fada4c20ef3a806aa23ed21fed6ee4f5","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0015","source_trace_sha256":"d872f97438000413ba3434c4d7fa55f5d7a0aae383d7964fddbe8a43fd915993","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-023","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0016","source_trace_sha256":"25013cdc3ae0a3738ae8ecafc9afce76a71201c8db79de3afc2c001c27616aa3","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-048","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":8}} +{"public_id":"val-0017","source_trace_sha256":"e5e65680f1a32a1bc501ded31369905681b0901151b631dedcd73e18123bf878","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0018","source_trace_sha256":"fa6c4857abdc89a335b5483ff7dc007c32a86dbc08111d1adaac6c1795caf21f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":4}} +{"public_id":"val-0019","source_trace_sha256":"e0fe81c61f43185ec13cf7a4388aad44a59f3ebc06c98263a33619155e6c6816","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-018","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0020","source_trace_sha256":"936a883e16cc543b8d7f5a0ed8f38a415b2c83d4f71fe3bf6a279c6206d8c801","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0021","source_trace_sha256":"f00550f01c7fd130478e79951983cc6d4045a6a24b74fcd869bbe279edd39537","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0022","source_trace_sha256":"0ee8c2218c8e0cb6ed5ea9bc31fb82b82f22f1b6810f5d443d53f5089a55ee1e","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0023","source_trace_sha256":"253f2d0c25bd5a76d4362eb2934cbb82bb933b9e2daa972b8c7f2c4d1962e22d","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0024","source_trace_sha256":"b3024926ebfb2b2a04591908fbf7f6cff83b1d555b0baa9e4403ae0face7a63c","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"CHECK_EVIDENCE_SUFFICIENCY","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"CHECK_EVIDENCE_SUFFICIENCY","arguments_omitted":true}],"status":"completed","terminated":false,"failure_index":null,"steps":[{"call":{"tool":"CHECK_EVIDENCE_SUFFICIENCY","arguments_omitted":true},"status":"success"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0025","source_trace_sha256":"6e5b5346eb7f76c1c475a0aa6f60d6b1a8a535569faf34b28098e5502a9ec700","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0026","source_trace_sha256":"1d80a99f29d25da5e078706d20acb0867bdb1faf2045592f65163326bf6ad533","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0027","source_trace_sha256":"521859a17d24418c868e1823777d919e5b24b06ed9e07a4eea9f3aae8c500add","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-034","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0028","source_trace_sha256":"1528d71ae87a50b7f537e090014a66045d0032c625323e09959f7bccdf5a82f6","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0029","source_trace_sha256":"897210d51c532ad73b8bc73ce09b95645b9343242bc6b99b9efd12aa7baa65b8","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0030","source_trace_sha256":"243650c06aa9070a63a656206c2da122d1398fcafa8cc2643bc3a37bf97a334f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0031","source_trace_sha256":"e0306fa239bfef88a53d702240623c38f8ed9a5dceb364787f4873907fa4489c","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0032","source_trace_sha256":"513a9a432253192b97e9e702f68b8d2d8e5122f4aebb9592f55e435b62a8d1e2","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0033","source_trace_sha256":"b59a54a79dc91c992e8a0dce52dafb4f5b80e30b0baca03d93e99e454e1d1057","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0034","source_trace_sha256":"716d9360f7646eecb7737e46fec0ec25696c0d7f2f8d21560760830a50576475","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0035","source_trace_sha256":"7f2a6ebb1c4670abb2f8401290a037fd5e01f28a2512a1323bbb6c46199cbd4d","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"error","tools":null,"unparseable":true},{"attempt":2,"status":"error","tools":null,"unparseable":true},{"attempt":3,"status":"error","tools":null,"unparseable":true},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0036","source_trace_sha256":"a197a5ae81ce32718a3476eea6f5bca2d6e7b3f795b99134d7585c45e950db3e","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0037","source_trace_sha256":"0e2d4cc9446024255b22fb2bfd5495179ded85d1bf18aed09554c25b5f8a04c4","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-018","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0038","source_trace_sha256":"8b95ffb3c3ca11e9714671dbf8fb53e28863fb0a04bd43e7c022d38ab61c9d08","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-034","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0039","source_trace_sha256":"486a084be67f87c5fa609c2953307eba88a9ff1f0e14188c4bec172d3fbe570f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0040","source_trace_sha256":"0aee7a5b45a29fd687a5ce10cf5b78e458e478d089b7a4e208cd148943982900","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0041","source_trace_sha256":"91c9e8f6e861f81b2e08c260a318929a42e4433667945e8049ec112034bae716","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0042","source_trace_sha256":"f7f1f0e11a1aa2a0d23a0c426084c1cd65e04aa90b4482e818b7de5b64e79b53","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0043","source_trace_sha256":"02c971a267b98682e4e1cfbf8d8b8c2ae67a71635d63a18e4ad8b6d905a9594c","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":4}} +{"public_id":"val-0044","source_trace_sha256":"eec9a64b36f769669eb4d4f95afb6099eadf05915fd2f1e0e4883fcf27903424","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":8}} +{"public_id":"val-0045","source_trace_sha256":"8133eb093c557c723cb94bec306a883b4e33ecbbc7e6e61625045f43f01dc34d","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-017","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0046","source_trace_sha256":"80f07c3b5b7f76c02809397a48190975810c85add836dba1bf12b80deae7d28f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0047","source_trace_sha256":"069dd852ddeab07bc1b4a02f467f97db92f5a774046295aafe78f9989ec06b77","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-049","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0048","source_trace_sha256":"18db073acb98901c091867a93d264904e9d44db7442e8329e23aaf4c8148df41","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0049","source_trace_sha256":"a2fb2ce4948faf31a1cdb3986d8f6efe6032fa167e84222d2cfea14ef6d7eaa6","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0050","source_trace_sha256":"fbe6831dff5aa778e6b5893c5c0bd8352d9751a5c53380d6defb5520ce294ab4","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-034","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0051","source_trace_sha256":"fa9116eadd8468dd573c64432dae380b3df681a247b84e5c7fa8956f7be4619a","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0052","source_trace_sha256":"c17dd9f6fac099bb99d8415e807b832f92f790876c1d0aedfeaa6f316949687b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"CHECK_EVIDENCE_SUFFICIENCY","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"CHECK_EVIDENCE_SUFFICIENCY","arguments_omitted":true}],"status":"completed","terminated":false,"failure_index":null,"steps":[{"call":{"tool":"CHECK_EVIDENCE_SUFFICIENCY","arguments_omitted":true},"status":"success"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0053","source_trace_sha256":"1238fcfcb76533cfe59cae8a006b91fab3a798fdea8064c9ccbb1683328d17ae","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-017","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0054","source_trace_sha256":"d3e0e5284b7b10a09859b2c3a14f44de6c0aeec69b500230cbe4158264428a69","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-031","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0055","source_trace_sha256":"bd6b6974ce85e3fe6654cffd37076e9931faf668ad4e9ba356a5ffc4b38dc6ed","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-055","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0056","source_trace_sha256":"09d4e5dabb3541b7f50582dcdd84435e9ce22cfffe0954d64a4344cf108be45b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0057","source_trace_sha256":"e21bc0c96a727411468a3d471dec5ff15b3ee478c223e02ee1d928dc0300aa4a","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0058","source_trace_sha256":"e94779102bfade6163040f3c25c2d72d10b17036044a493d49cee6d8faae1af7","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0059","source_trace_sha256":"6ffca7d492f965ebb169dec02fb21dc27eeb5f2f4a1984e21c3fdbe72bf47f79","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":8}} +{"public_id":"val-0060","source_trace_sha256":"b735636b9016738936256298e616f09f9c6e05bfbd15b7ad30a7628973660f35","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":8}} +{"public_id":"val-0061","source_trace_sha256":"29132bd7aec5bf205292e52212cc57d78c512f884061331eb7a73ec7e508b60e","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0062","source_trace_sha256":"79adec65d5e4363594b12e2764ecf62eaf6afea7bf34302e0bb009d341973ce7","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-011","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":3,"skill_steps":8}} +{"public_id":"val-0063","source_trace_sha256":"12a44914d802d6f3f205e6c9dac6dc46b7d1624a3f96b917a97d989ddfb7eaef","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-018","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0064","source_trace_sha256":"41e2d47ac000f6e035344a030e1574bb9eeba4a88f150eb3e7ff81caf7e06f80","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0065","source_trace_sha256":"8153d2a758aa71e53b3742135df3774b00f58efac481bd010a02c336dd45b8b3","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"completed","terminated":false,"failure_index":null,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0066","source_trace_sha256":"bb70e50d6e6ba7b1934031b5e96bb284c3d874094f15bc8126f5322977897be2","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"CHECK_EVIDENCE_SUFFICIENCY","arguments_omitted":true},{"tool":"IDENTIFY_MISSING_EVIDENCE","arguments_omitted":true},{"tool":"DECIDE_NEXT_EVIDENCE_SOURCE","arguments_omitted":true},{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":7}} +{"public_id":"val-0067","source_trace_sha256":"cc6e9c05730e3d11bdf52c234cfa7325256897a939216d35364b613afab09aa0","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":null,"unparseable":true},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-055","grade_alias":"grade-003"},"budget":{"programs":3,"planner_attempts":4,"skill_steps":9}} +{"public_id":"val-0068","source_trace_sha256":"21b684ede71da4a0490401ee58bddb7d1efb6849beaf17e73e758273b8afe07c","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0069","source_trace_sha256":"8f8425f74cfb4a2e46f76eb618c039f0c91d16d9e2ce996d84f2f8c127ca205d","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-018","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0070","source_trace_sha256":"1b7307d5edf15592db6c6667f33957042cfef5b6d00938db4ce1bf94266006b3","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-017","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0071","source_trace_sha256":"c6749cad8e76d5c741d83fce7c4a2a2d5375374adbe0957eeb91f87762748ab3","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":null,"unparseable":true},{"attempt":4,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":8}} +{"public_id":"val-0072","source_trace_sha256":"1c919916c9ecc7c2d13512204cc44c0cb748232bd21355b207ab8ca6c8cf7927","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0073","source_trace_sha256":"55fb3db9b34e19f08532c5c3d5e22a3b8d0c956a16e7affbc798a09c4f122bc7","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0074","source_trace_sha256":"a08a26ce0fdfb123cec8e4d32b76aeeffe2c115bdb3e1d2853cac7df6672df88","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0075","source_trace_sha256":"41a4f7ec72d3fe565b238e317c129fa5c36648e645f6094f41aa3a018f391848","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-055","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0076","source_trace_sha256":"dabf9d386dcee140441e8104b6b3cc08f7ddb05b88251fd73b69b70b6b0faf0b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"SELECT_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"EXPAND_SELECTED_CONTEXT","arguments_omitted":true},{"tool":"INTERPRET_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_CONTEXT","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"SELECT_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"EXPAND_SELECTED_CONTEXT","arguments_omitted":true},{"tool":"INTERPRET_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_CONTEXT","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0077","source_trace_sha256":"b9422d57f03198d025dff5608b678631abce055c5987d4912b40fb354888f7a6","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0078","source_trace_sha256":"a15a599ac65c6f1bc4c48c3382e8a1e7e6b5d99df1923e131c9b59bd378dbe9b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0079","source_trace_sha256":"c7f42dbec3e6453f70464719d6913a58b9b3ba66a049c7f5ec0bbe068ddfcc14","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0080","source_trace_sha256":"d99e8083f3470625eecaa06e67779a635b44e0dbef9cf567a103ef004e823d87","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-002","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0081","source_trace_sha256":"ccd74e932314ba3f289d1331fcea9aac91825fbe92db80b31f9d3eae23224575","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0082","source_trace_sha256":"feaf1dce9d27fc9fbb116c7828101df11a0d34bf515dfb080a1e583815f18f2f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0083","source_trace_sha256":"0da4b51781a5f50fa14926f4a8384424a1e960247748cfc5fe33249ad06ab564","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0084","source_trace_sha256":"52eda0785e11692868be9d83d23b3d6822f016fe19fa48e44ad074bb20b9eddd","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-034","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0085","source_trace_sha256":"48ad64f3e2c963200a6d21710118e2e2f56b524344e89d83c46bf077be0c3c03","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0086","source_trace_sha256":"fb9ce5e39d83efbc095f796a09ad348c6bd41e169eb3395651fabf41bde216ab","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0087","source_trace_sha256":"7b95b383b02e8f29bf48175941ce724bcf15a1b0874a924d5a8be7fe9f89425b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"SELECT_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"EXPAND_SELECTED_CONTEXT","arguments_omitted":true},{"tool":"GET_CONTEXT_FIELD_DESCRIPTIONS","arguments_omitted":true},{"tool":"GET_CONTEXT_FIELD_TYPES","arguments_omitted":true},{"tool":"GET_CONTEXT_FIELD_VALUES","arguments_omitted":true},{"tool":"INTERPRET_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"SELECT_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"EXPAND_SELECTED_CONTEXT","arguments_omitted":true},{"tool":"GET_CONTEXT_FIELD_DESCRIPTIONS","arguments_omitted":true},{"tool":"GET_CONTEXT_FIELD_TYPES","arguments_omitted":true},{"tool":"GET_CONTEXT_FIELD_VALUES","arguments_omitted":true},{"tool":"INTERPRET_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":1,"planner_attempts":4,"skill_steps":2}} +{"public_id":"val-0088","source_trace_sha256":"cceefdf97f0f38d1644be27cfa520dbb7ca4eaef898d45b94a36343bcb55e66d","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-002","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0089","source_trace_sha256":"c97c280057f7818e702a06cbedeb5b860bd8e8d67dfb2083f2a2d3c2357e052f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"completed","terminated":false,"failure_index":null,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0090","source_trace_sha256":"0de0980385dc5bcb3e7baaf6e260d9175a9ea9cbfe24102545d5d12fa67201d3","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-017","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0091","source_trace_sha256":"fd2eabe2075daf6574a4eee784afaceaca33a93ec423e91b84769b1b6dc369a9","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0092","source_trace_sha256":"082d3fc47c559faf8b1c5e45b4d9d1a3467de06c9311f690f3c7a79e08d26768","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0093","source_trace_sha256":"a2b77a46cea66cbac57aee619e06d741423ae09fa806efb140066a18d5e8a667","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-057","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0094","source_trace_sha256":"0a93ffe7a9cc32135ccef8910ada67361c704916af8cae8cdef40ef7d4bd616b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":null,"unparseable":true},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"GET_TABLE_DESCRIPTION","arguments_omitted":true},{"tool":"GET_DATABASE_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_DOMAIN","arguments_omitted":true},{"tool":"INTERPRET_TABLE_NAME","arguments_omitted":true},{"tool":"INTERPRET_TABLE_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"GET_TABLE_DESCRIPTION","arguments_omitted":true},{"tool":"GET_DATABASE_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_DOMAIN","arguments_omitted":true},{"tool":"INTERPRET_TABLE_NAME","arguments_omitted":true},{"tool":"INTERPRET_TABLE_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":1,"planner_attempts":4,"skill_steps":2}} +{"public_id":"val-0095","source_trace_sha256":"d8ce387b06a4958823cce0d156ba26fa96d2d039df49a62171177aa9cd330609","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":7}} +{"public_id":"val-0096","source_trace_sha256":"e5994f5d29f84274787df83c896b6707725cf8433319e56b4a1b0ead328a0f07","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0097","source_trace_sha256":"b50690aa9d511150b88656a797757d164bb302cb626d1da47fa0855bcac5e2f3","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-011","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0098","source_trace_sha256":"fd9612820752bca631c287246ef89e7abd91ed18d6d187ed7748f30220907797","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":null,"unparseable":true},{"attempt":3,"status":"rejected","tools":null,"unparseable":true},{"attempt":4,"status":"rejected","tools":null,"unparseable":true}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":1,"planner_attempts":4,"skill_steps":2}} +{"public_id":"val-0099","source_trace_sha256":"2b438f69becac9443c05430dd1aaa1b35ebe39c856204c59b30b6737d1311f54","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0100","source_trace_sha256":"55221f91308ba05351849f8cd31e629342df04520e0bf43908ba4daa3ccced8d","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0101","source_trace_sha256":"7c4eb16930945e2fb41ed17f8de53a423e0c2c19e77a331b798e973fd1d89323","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-034","grade_alias":"grade-002"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":8}} +{"public_id":"val-0102","source_trace_sha256":"e51ea6df712d3177e8f4a6d85fb9e70ecf71b06e6eb701529a0fda974105f0c2","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0103","source_trace_sha256":"ec0b555ccd7ce0039c7f70877736b639672b56a6d5ab860b8f5fc53bf0dbd238","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0104","source_trace_sha256":"c0b5d4e44b48d88842ad2796f686304498353f7a16e8b60634f13276d85fa452","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-057","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0105","source_trace_sha256":"cab2b8067118330b8bc59548f71d2a637a7db94f6db9df9a01e3e6616eb4169e","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0106","source_trace_sha256":"03202423f75ca84e3fb5d22f0fa96e78d667e62341c5fed986a0bca9c959649a","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-018","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":8}} +{"public_id":"val-0107","source_trace_sha256":"66c80b86f1df918adb1f9eddce106d1887c218e5edd93a505871c6135b51d24e","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0108","source_trace_sha256":"bec269188d339184075936781037f84647d710fcc21df0e71dbd55636061cfb5","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0109","source_trace_sha256":"819a9a319a18890623e20cc6f2f12935360ee66e3d1e16a1f78dded95dd33a9d","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0110","source_trace_sha256":"956c7002624abd399e137b2b14d84450eaafe5c80c1dce7b37d0c2acd668c824","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0111","source_trace_sha256":"76e6a8a97e2a9ccf3576e2435471c68a8c03c45135f261808e13abe57d3261fc","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0112","source_trace_sha256":"201fa8e501e51ccede1559703279612be041313639386c1ed299c4908e1b886a","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0113","source_trace_sha256":"f3bf42c9c81f53bb348c9708b049b5bdf6be2e134716c1fa424f78707bada5fe","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-002","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0114","source_trace_sha256":"a6aa8a7933de0135eb7ec96e627e450714cb5ab73d3d80f7fc6605a43d1714b0","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":8}} +{"public_id":"val-0115","source_trace_sha256":"89919c3fe479ddbfbee9932906739e193df020c22db1aa755b93fef5eec49c43","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0116","source_trace_sha256":"df9d2cdb04cb18d522ad7d5f706b54f1285279bfb9827af7045435df313269c7","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":8}} +{"public_id":"val-0117","source_trace_sha256":"fc613758b4f90cb5c35d59cdd0fe1ddb67524692d7bf3b91b00bbbe7adcdd18f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0118","source_trace_sha256":"539a42b0dabfcb7a41c6e3f67f230cca404608227e2ab99d86e821aa5a0ff885","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-055","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0119","source_trace_sha256":"d6efad5938c18eb126f505d0f6d2619649c40fcfae0068fa69cbea3b7159b8cd","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0120","source_trace_sha256":"c0eebbb7dfde95657e68aef6e3e128892478c1fd2fda9feb84a72d3990b33599","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-036","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0121","source_trace_sha256":"034eff7046b0eb5be1788327f832a46f9d5bbe1dce00fd590ec00fef093bcde8","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},"status":"other"}]},{"program_index":3,"tools":[{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0122","source_trace_sha256":"7f7b3d61530a7d83647825d6706533933c68ad01c8bb05d9b7cd37eca341e280","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0123","source_trace_sha256":"2e35d159a67d6f8fc75f7012b32c8e911528042235101dc16b712fdfc3997fe0","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0124","source_trace_sha256":"169034dcfcdb4a69d49e6ee45ff7089244366244edb4612b13d4c0cafcc02234","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0125","source_trace_sha256":"aaf997351667087c294b7cfb4737503e24767cf7ff1c8eedaa41ad1ef8be6faa","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-018","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0126","source_trace_sha256":"d28449d74c7ea1ac6299eda46e3cfefcd9c2694680cf90f02acb1a0f2a2f4c87","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-023","grade_alias":"grade-003"},"budget":{"programs":3,"planner_attempts":3,"skill_steps":11}} +{"public_id":"val-0127","source_trace_sha256":"6423a93ed331506bdfbee6c66d33b136d9d9fa9938cca58dcf6388d065b13777","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0128","source_trace_sha256":"6376aa82735b9645b6df22542d299b9cac7ba8128ce27914c01be6e9777cb5e4","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0129","source_trace_sha256":"608a4de744457093ec436194cd769765885f3aab9870f39b62c4ff35d67339da","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0130","source_trace_sha256":"fe80c54ff38a86a8c14a16654de689d86e6a7b34952accfbc609af7322c7bdec","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0131","source_trace_sha256":"e9378428567a390c492204cf93eb7f4ddd7ac43a416cb4c8e700fa98ae0d5c72","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0132","source_trace_sha256":"1b88a57470bd9da9b9da1378fb9a17a261471bd0f1983c795fd444d2ceed2f11","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-043","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0133","source_trace_sha256":"9f11acaa8cb7f914c04bb690bb76cfae3efb79cfd64a16f7fa436d44e4a386c3","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0134","source_trace_sha256":"30587683df3f87299a6678f1a5918f58c31f365c58161587e94a732c6ad46a85","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0135","source_trace_sha256":"420bbfb8a944c4ac63985002dc83d7c1b5962bd9bff68c2e5faed9a708b90503","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0136","source_trace_sha256":"8a5b9baa2f339b48c263ad40af774e1aa2e9d0a33dc12a3b31ee8796cb35e590","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-017","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0137","source_trace_sha256":"6b868695a12f42b69ec17f0a189ba82cee6466adf77b1d9838188623ad60e513","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0138","source_trace_sha256":"016195510e86efaacce41ab806b1c52e285e8752885349fb426fdd33732c7687","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0139","source_trace_sha256":"b8432b5145c74dfaf27a45acc73424e06be195c4c4e21fefec621cfe92c0437b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0140","source_trace_sha256":"568a10a292533d231bbe5147f1388c38bd5149679fbdd4e9fd740cf8af837e8e","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0141","source_trace_sha256":"d077c5591bb6bf0f9fb04c0dfaf97f0c1fa1502963181a63973f0b2517d85021","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0142","source_trace_sha256":"313ad1eb8a1a4a8a89e6212320206938999aeb5989ba071f87a4ee8fd53c5ac0","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-007","grade_alias":"grade-003"},"budget":{"programs":3,"planner_attempts":3,"skill_steps":12}} +{"public_id":"val-0143","source_trace_sha256":"667a34803673c5430cc789defebf61e306d40b24daccf01d2d6640feb9489611","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":8}} +{"public_id":"val-0144","source_trace_sha256":"0925aa266098843c8dc315434bcad16e78f02f117b2e6c28b366a772a70d527c","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0145","source_trace_sha256":"9b2575aec514310242d481f9dc118ae1ed52085acaf46b7c6923668f5f260bbb","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":7}} +{"public_id":"val-0146","source_trace_sha256":"a68aeface0bd63b9ab3446a8122412cde4135b350a98c85967cb824ac371da6d","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0147","source_trace_sha256":"8c8dce8f6445797e42496ff9e44d81b3420a070cca5efef9d8cf450a0dd92c28","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":null,"unparseable":true},{"attempt":4,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":8}} +{"public_id":"val-0148","source_trace_sha256":"8832ab2c52408fe3c46c88fa591f63ef69224605246095a4ec51230e9d4ce92f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0149","source_trace_sha256":"bfeae6c601bd17796abc5467aa3296e685eb0dc95cd0466933379152c77d20ad","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0150","source_trace_sha256":"a8048071502b8cab6882163a89809023c61a0d187a6d124a15dc8bfd29d2d7b1","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-017","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0151","source_trace_sha256":"cb5eed712373d2551c90f7903345ca19f1173625d8a28c28dacb4bea175505b5","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":4,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":10}} +{"public_id":"val-0152","source_trace_sha256":"aa8e92c661c2ae09f728c877c439b33d7fd48d6b6deac337b62388434c4772f7","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0153","source_trace_sha256":"e78b5f44f1a979358b25f9c81e78b20d4784b8f387d3b35f941e541f02fcc645","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-049","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0154","source_trace_sha256":"52f644aeca6b7e5c5a81e94ab0ead8abce67427c01ad73cd8586740b63735178","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0155","source_trace_sha256":"476eeb7bfa91bbd6eb79edba0aa7cbf1e11eb55cc45d63df70d47fd5960fda80","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0156","source_trace_sha256":"2a2390fdf1b606efedfc0bb8d688e3b3a662d1f3ae804966497e53b0e6796b6e","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0157","source_trace_sha256":"0e51a2243d869cc53db38c3844a643a855fa17dab763a0f9107cbf7ae63fa52f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0158","source_trace_sha256":"317ddb7efd2b03f019b1e6395250c7437919f77fdb2a6458d6e22f2d4fe729a3","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"ACQUIRE_MISSING_METADATA","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":8}} +{"public_id":"val-0159","source_trace_sha256":"69adcb909c88cecde2dc682b303f8d5dd948d8efcef55ec4314fcfd37d6a7d88","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-048","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0160","source_trace_sha256":"5054d24915e6b51f11098f6393a4b604fed004ebdd946cf810d61aaa31218b27","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0161","source_trace_sha256":"dee5390a6be1f439f0f34c70e761f7a1b88a10ac3c68e7bafee7e13eb86f35bd","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0162","source_trace_sha256":"feccb96627074a49544763d1af9e63df0a95afb01a48fe23831c55a700a86709","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0163","source_trace_sha256":"bd1a9e6763a11fe933b7e764a5c4dce031a60c1e162fb2ea1df49771afffd6a1","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-036","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0164","source_trace_sha256":"e77e44809a2a558b8a181a86278fced7b60b1487161fd6608fe3a957dda7c203","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-008","grade_alias":"grade-003"},"budget":{"programs":3,"planner_attempts":3,"skill_steps":11}} +{"public_id":"val-0165","source_trace_sha256":"5b64a28a5b54898c6898ddba473be1f09a1761f823f1b97feada08a13774d5aa","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0166","source_trace_sha256":"36fe024591d0b54edc00c96872e40704339b03a89c74d8601e2bf6b4943cc23a","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":7}} +{"public_id":"val-0167","source_trace_sha256":"ca3288bd432650b46e13020550bb3c846244517ca7589a537a94a59c7e08b18e","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0168","source_trace_sha256":"605f3bce48ddd82e62724e45f40b25143f2a4cd7d10bca35e17acd49e3ad46f0","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0169","source_trace_sha256":"f7515d9ac5ae885588ced29ebcc188256029c27f8214f2b31b674e645008649b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-011","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0170","source_trace_sha256":"28cb7412c7e5bcb40c61c13ac94eb2426a76de9729b91287fd9edc1441073cec","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0171","source_trace_sha256":"84d9a3c9c8955c8622f44df530ddec0e856589e49eea2fbfbde1d6977927251f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-001","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0172","source_trace_sha256":"209f3909580d8e590bd48bcbb22e489840f09afb1a5555ed339a95b454bb5361","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0173","source_trace_sha256":"4b7a910ab5ebeec0118dafa048a852ec4f4cde798d7ee7628a207d8249ba388d","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0174","source_trace_sha256":"9ce5a59d035bb2c041e0a51f5c0c770cec4db1811346d945da90899243f26ed9","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-002","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0175","source_trace_sha256":"a26531be9b0ee378735ae988dc9398e632faa82b4f6c1b410ad9931463670ce4","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0176","source_trace_sha256":"134ad15cecd99b74be272163fae040d83dbf287826a0811881a19b2a9bd5841b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":null,"unparseable":true},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"GET_TABLE_DESCRIPTION","arguments_omitted":true},{"tool":"GET_DATABASE_DESCRIPTION","arguments_omitted":true},{"tool":"SELECT_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"EXPAND_SELECTED_CONTEXT","arguments_omitted":true},{"tool":"INTERPRET_DOMAIN","arguments_omitted":true},{"tool":"INTERPRET_TABLE_NAME","arguments_omitted":true},{"tool":"INTERPRET_TABLE_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_CONTEXT","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_TABLE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_DATABASE","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"GET_TABLE_DESCRIPTION","arguments_omitted":true},{"tool":"GET_DATABASE_DESCRIPTION","arguments_omitted":true},{"tool":"SELECT_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"EXPAND_SELECTED_CONTEXT","arguments_omitted":true},{"tool":"INTERPRET_DOMAIN","arguments_omitted":true},{"tool":"INTERPRET_TABLE_NAME","arguments_omitted":true},{"tool":"INTERPRET_TABLE_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_CONTEXT","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_TABLE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_DATABASE","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":1,"planner_attempts":4,"skill_steps":2}} +{"public_id":"val-0177","source_trace_sha256":"d9b184306d3369cf617cbd820dce9a3fe09dd478732f43d56eeef81451a5e0e3","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-057","grade_alias":"grade-002"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0178","source_trace_sha256":"a4ab1706072b7d77285b86e2c008ce6ae617939eb3474dfe4a1056f86b0f9136","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-011","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0179","source_trace_sha256":"d56018e2423cc76ebc55aad6ccfc0a5b2874100d18e8c82e2dd172a970c9e31b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0180","source_trace_sha256":"1db000e478aa59bd709ab419e2f91bbea4e81ebc95c8704ad5700f563a93b387","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0181","source_trace_sha256":"3b10c3ab024b16e047b0fc3411f8c04a1ecd762b2d5913e2cd67e24fe0162f7b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0182","source_trace_sha256":"5eb999dd7bee180255de33015bbc447acd716bf0c98c57863defd37dfd3489b3","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-017","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0183","source_trace_sha256":"18fc8b7bc61d19f5b2dacdc4007c5598d74294aac14a4d10abe8710af4350332","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-049","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0184","source_trace_sha256":"c6d68a32c63dd6486e9f9f8b38c7c1e678121ee4704fa9efa6faa6b464f451c2","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"error","tools":null,"unparseable":true},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0185","source_trace_sha256":"9ea9f4da63855e6a2c22cbd63a0e8e0f09415619ed57d667699102b2eb726c61","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"error","tools":null,"unparseable":true},{"attempt":3,"status":"error","tools":null,"unparseable":true},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":1,"planner_attempts":4,"skill_steps":4}} +{"public_id":"val-0186","source_trace_sha256":"b0efa3487826a916596a60f7044aa5030180206c79acf438f5c997151e9f010f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0187","source_trace_sha256":"6d9ec774eabc9766a22b8a6655203a3f42b94f128a67b9c95d9cba0b919ef491","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-049","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0188","source_trace_sha256":"d3056613fbb7a700521751e6bd4902ab006725ae4514207756d02b38467ffd4f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-002","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0189","source_trace_sha256":"6878d9366e1cb8770897fb9ad5b8ac2f95ca6826e86f3dbb29ee517a4c784bd9","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"error","tools":null,"unparseable":true},{"attempt":3,"status":"error","tools":null,"unparseable":true},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":1,"planner_attempts":4,"skill_steps":3}} +{"public_id":"val-0190","source_trace_sha256":"d03f17f2d26f890b9fe23a38d93ad5ea6b933b5c71a897451da48467252a38af","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0191","source_trace_sha256":"380d48ca8eb90f138fbb7d36be85a4bac8cbfcfa2785ee4073ead9cfccea934a","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"error","tools":null,"unparseable":true},{"attempt":2,"status":"error","tools":null,"unparseable":true},{"attempt":3,"status":"error","tools":null,"unparseable":true},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0192","source_trace_sha256":"22251d638848272440cd71398607ef76fd0811cad5db239bc752fc39a9249774","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":9}} +{"public_id":"val-0193","source_trace_sha256":"0d3ed9e35f6a6c1d32ce42d411f6341a8e40fa3fe158b3bcfbd84b360f1ebcbe","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0194","source_trace_sha256":"e14da7e48e9c8b8eec355a057cd4c1ce094c26c00a1a5acbc046b63d2a00e2d0","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0195","source_trace_sha256":"61a96a4df921a19b767e8422b9df1468127af4cac6964d52e6ecae1e82983a24","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0196","source_trace_sha256":"4c19ca33b658c7ed96343014b0c3eb9d922949b7af8ce68451d009e087f84052","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0197","source_trace_sha256":"86687078c0fac5a00a9164bc35e0402098280bdca85aa8be6c61ec477f1d3913","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0198","source_trace_sha256":"52a2b2b22dacff584202a2ff5853afd7b7e5058545eae2a3454cf70b0ca75998","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0199","source_trace_sha256":"9146e88b749de8c5d6993b004d3b84769b21fb1c0f844a61d68c1b886a1446b6","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-034","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0200","source_trace_sha256":"3fb8f8ba8e9a11d456583f0e08e8a60b1dd86a49e1774067f1c346285ba346c7","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0201","source_trace_sha256":"a776457d062862829f308c6b48d1e1a643a4f015b488c569a003bfb4df29d6bf","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0202","source_trace_sha256":"9d69e074be848e78088a925df2a3e6ab0379b3632db701c5a0af6ece7ec0e1b5","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":null,"unparseable":true},{"attempt":4,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":8}} +{"public_id":"val-0203","source_trace_sha256":"9c81e435712ffacff0253cb37ea0e8fd55189e5a53053491c24680dda484d361","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0204","source_trace_sha256":"574f6d244ad1ab5dbdc03affced8c921e5cbef51dd18a0feefa322bb551fa8b8","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-049","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0205","source_trace_sha256":"26ebaadd6bb9859f2231055ddf38aabbb8ccdabb46f2d2f7ae5c041c1cdef5cc","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0206","source_trace_sha256":"07c97e8a0cec064ee03f3066393bb5199891f087b41c8798c5d7e00e85b9345c","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0207","source_trace_sha256":"b32c23a116312bb325062cbcf72dc43d988078780cef3a91eb8fe5e6c4c058c9","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-018","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0208","source_trace_sha256":"c51eaff3c0f9a92bcfce274ce59f03b6cdd3e2684ee27c2b9b469b254445e77d","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0209","source_trace_sha256":"fb5a5dc85bbff480018b2b7d5d162d4491d1e68414fe23fbccdf48aba97d7a02","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-018","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0210","source_trace_sha256":"85a345a45e0bacf6ac369bccdd4cae4de9c31ae2b29940a3f6fa167b895f612a","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0211","source_trace_sha256":"fa44f4e87547070cf6fc183b1cd6b0731f78e00e7148cc62673255b763f65349","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0212","source_trace_sha256":"f72c366649ec42c94811c2a058132b13d31dc7776d21bb02fed6b6a69ed1eafc","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0213","source_trace_sha256":"8bf9e24cf2de98b22ae75405a131554d5a34051dfcd260993f14c7300ccc997c","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0214","source_trace_sha256":"95f1d9c1399d8ed1843f320d36d45ebddad0e77c16e7ed43a2e168ff23c518da","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-017","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0215","source_trace_sha256":"0364b7c77c3dd73055ca956a6dad4a045d28cb53cc3d18ad0d3ab2cb9f6a3a51","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":9}} +{"public_id":"val-0216","source_trace_sha256":"66f9118a8cc8c4558d556256a5267dd816e843ba00284dbe64e0da15fca66637","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-028","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0217","source_trace_sha256":"bdab3c6c9d9705380684dfdf1c5c119713b228dcdb98977e5d1c9a669a1e903c","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0218","source_trace_sha256":"e99274fe05f888ed29ed38ab687dd2ba599edbb208993063bec8aa41327c55b5","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0219","source_trace_sha256":"8200f69e2fc4790034229a3f5839408ddc641cba9b1483e11e0ce50c13f83ec5","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-036","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0220","source_trace_sha256":"aa6e3c9f934168cfa589938ff4dc93206944032d26fa7266dbeac9c5f98c886c","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0221","source_trace_sha256":"8bab24468b3854f16b04e6fd9c9335177ae25ef7b3167f24b91b90d40c1592c8","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-055","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0222","source_trace_sha256":"608882a259b32bde3769f069d4205c10ca621f7dce430831cd47786d8e6d464b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0223","source_trace_sha256":"ada2e3108ea507eadf13a01a7df6ff72383e5cb7d323fb6b340198c66119da6f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0224","source_trace_sha256":"605af6caf2e75f3b87edcd7546df25525ebaba94b3a3e670d5d0e36e2c44e3ec","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-023","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0225","source_trace_sha256":"a5ef656fb6848c2832ab7bebf35a8321f530e4e3c4167158a3b78c1931774480","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-034","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0226","source_trace_sha256":"87e2c66c1f917c4700f428bac1663ffe812af05a2f6cea4f47278e80d4999570","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0227","source_trace_sha256":"59b381e2732724489bc802c29b28f2e5515d5191803eb5ad37b4788414c6131c","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0228","source_trace_sha256":"33abe9cf480fd7bb9976b4742f5b1c03c043cb849d4e4026e51e415a7624a951","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0229","source_trace_sha256":"d1eac7054e3b6004200da5ed6337076f4d45178157ea844fd2f62f8d31ec0fb0","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-023","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0230","source_trace_sha256":"55dd29e0197da5e745fbb1fe9c032421fd32309883dd73132a59af5e900db9ee","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-055","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0231","source_trace_sha256":"dd768114b7793217015141683ce1295d888074017eb8caaf0e6fda33a817dce2","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-057","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0232","source_trace_sha256":"29c3c87b81dcef4c563390bac33e604f818abb10f9c54175be48d5100360a4d5","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-049","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":8}} +{"public_id":"val-0233","source_trace_sha256":"099b47af9f5165fe82535b29759cf453ee0845e66ccc2ae4e373e2c349c74742","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0234","source_trace_sha256":"bb097aea8af6e13771646a91e98010eeb9bdcf2e876c19f3c0c93c874f085d7f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0235","source_trace_sha256":"a04c17f8f001fb616cfd092e07690625b4bfa3457a7109c5518d071586c370f1","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0236","source_trace_sha256":"598adc2ef26f88cad3916268a7b3fe9224acb1ae53dbf2d299b266a1711caa5f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0237","source_trace_sha256":"f97ba352ed025627d5ee064f22e71687a7bfab76d127fbb872a6508a8c93a149","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-036","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0238","source_trace_sha256":"2b2d361c900808ad1dd15890492f3455bee13b5e295de18f02e004cc13d62792","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0239","source_trace_sha256":"8323d448da3be3250c74a6981e6c5c6c039033171e1bac1f5f0429f1d8bfe1ae","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0240","source_trace_sha256":"28aaa52f9b2152128c2e73cf8cbddbc150318871df5ade94fb15e00e1ed67cf7","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-049","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0241","source_trace_sha256":"05023ff15fd275fdd2a1827172a0be29cd532b313b45d916e4179a31f97d601b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0242","source_trace_sha256":"7523b11b1ffddd77c2323cec013580d7582c9f77c4ea57a390b4ce266cf226a4","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-002","grade_alias":"grade-002"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":8}} +{"public_id":"val-0243","source_trace_sha256":"427e65d96d14cc89ce386a4299d631b717d5a3cf6304f8e3de9f313598ea00d3","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0244","source_trace_sha256":"f22afc93d6b5b6443fb4c40721eae8c964d4433350eec4cf6d854a9485f42d75","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0245","source_trace_sha256":"7211b64359667eba5d6688c59b7487b9691f9eaf4d33a361970c2ae73246f7ce","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-034","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0246","source_trace_sha256":"1bcc0c93fb585bb1770708212c549517ab0ae8ea50bcb519746da4224b428e4d","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0247","source_trace_sha256":"15f30fd59d67368be343ff6144df6bbf4930f133c920a3153332c8b1e1fa2f1b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0248","source_trace_sha256":"0f41f7cbbb5a539cc7f9a0c9b2f105974cf20d6f238db0793f97a576e3211876","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0249","source_trace_sha256":"e86b42083e596a00ea2a25a9223b17e73d972250144643da537a4c405db0cc94","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-018","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0250","source_trace_sha256":"9de262e5770ead4dd5ae68fdd18e0e66aa34a946b1e05554f3819c8a7b447485","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-002","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0251","source_trace_sha256":"8678d81acf16ec721896ba3b68e8cb272739c66a11b999ebca393a0ae69c2f07","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0252","source_trace_sha256":"acedfdfffebc629acae2a20d2cd300fc388ff23c92b7013383ec4d374e715ac0","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0253","source_trace_sha256":"0dbbbee87d0d56d341fda923f6db22b188161b49ecb122d0244d0720a2975676","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0254","source_trace_sha256":"43eb28f9312d25f93a0283a498e3f31fa8a22398d7b99c187281b6e21ad8c050","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0255","source_trace_sha256":"e13827b843c6b6211d7bdc0570690265bf0d1496f5e9471d954b1a903b20023e","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":9}} +{"public_id":"val-0256","source_trace_sha256":"884e2f21c63359e8c281268358ff5e1836f45bb51fca07e6e38c0fc76a466126","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-055","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0257","source_trace_sha256":"fc4cbbf8ae1f7009de88ace3b0a74827ed6170457ec4eded8475dc00327f2bbe","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-018","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":8}} +{"public_id":"val-0258","source_trace_sha256":"2864da6a1722f4a744778502698c979d0cb515c51befd1fbed91bdef0c52fb69","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0259","source_trace_sha256":"e041552bce5e66afa37a2ea1b27701a4ca6cf2133caf857c875efe1da2b1c129","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-049","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0260","source_trace_sha256":"b8a59027f670d38ac6f276ce62aa15a6aa72845a9973833db129becc98916944","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0261","source_trace_sha256":"548dfa9c4809d4f292faae544bdab37bcdafd350a4c5457caceb34af35843784","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-036","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0262","source_trace_sha256":"d8dfa42ae7a69e64800bf59287548703dad69092d8a2344f165932776755b54a","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-049","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":8}} +{"public_id":"val-0263","source_trace_sha256":"e7d0ad91bae496476b56858cb38ac0475f705dd2f62f35ea83f5a75a036a4c2b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-054","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":8}} +{"public_id":"val-0264","source_trace_sha256":"65d456c2490e960627161f3feb16ae5d764a33ed89d4ea092d6c0ad61d863ef9","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0265","source_trace_sha256":"d0589bfd7f0d4e08b745df9be5d9d618250b912ae6a69fd187e2e06c9b3430c9","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":null,"unparseable":true},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0266","source_trace_sha256":"de08c006478b194659b9c51aab4b5de2e778d16a4fef6d0249885110e19adca2","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0267","source_trace_sha256":"2caab4e0e79eee8b3c4e90e002517c1a7b46f3a6b2dafc7c079727a8592e669b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0268","source_trace_sha256":"243eb9f3937eb3e3a7fb37f5a498e3df32e2ef1d6a0b59c97432a2387655fac0","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":null,"unparseable":true},{"attempt":4,"status":"accepted","tools":[{"tool":"INTERPRET_DOMAIN","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_DOMAIN","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":4,"steps":[{"call":{"tool":"INTERPRET_DOMAIN","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":10}} +{"public_id":"val-0269","source_trace_sha256":"3ab6246306fb59eb5933409a3b573806b3482200bad13cb9a72b7709d6bb65e2","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-043","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0270","source_trace_sha256":"5af2b478b4e51acb9d4aba79265e7b2c8600bb09b2796990ca7cb0840cf99d09","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-017","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0271","source_trace_sha256":"d1e07f9e815ce89181d06021f29b01fe9e598570d4823d862e97469cd9e37c93","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0272","source_trace_sha256":"39be4e9c7910e4727bb99742a9d3a62bc58f42c996e8db1b79ab6adbe40f8571","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0273","source_trace_sha256":"b909c7d786403ebb591e2fc30a5b3b2c761b570142c2eee6e3c5562d0ad4c926","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0274","source_trace_sha256":"4f5ee952f632bbd1d79eb742b4dfdde61a154747526c44bb5c26ac846df9bf8b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-031","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0275","source_trace_sha256":"4b77b04f673564ce60a51399cc3a519a988576a91581ea12aec82b86e030375a","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0276","source_trace_sha256":"6afa862edf2115101e797aae4276c7391f46089fc1e2ab27eee296b79fe97741","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":8}} +{"public_id":"val-0277","source_trace_sha256":"ce3ebff89de52dcb0416aaf8802a6a81c5bbf37f49d25825270eefd1a353eabd","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-017","grade_alias":"grade-002"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0278","source_trace_sha256":"01cf7912e1eec98f308abaf252369236ffb6ffe759d098c3aa6f4c402b1c61d7","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-026","grade_alias":"grade-002"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0279","source_trace_sha256":"09e4fcc5ec1177a195f57e253fcd4a87626c5b0d9cf9ed30ae039e79af17bb3b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0280","source_trace_sha256":"75f32fc31e8c0f65cd1e8726938c5de0a10363e73a0b3d62d795afc9ac2fdb4b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-018","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0281","source_trace_sha256":"1a5ad1702998fd206fb17a3101df55d252a2881aa26155b50e239a23c4d55174","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0282","source_trace_sha256":"7f4b3211215aacc37dbcf68b920005295206a4fa68d2d662ef24b49c4fc035d6","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0283","source_trace_sha256":"408bcc6d200f35eb8691ceba14f5aca0ae4535e60a464c3a7eafae5fb7950097","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0284","source_trace_sha256":"044096c805b5da369557e28845aaed76ebca7e46341198da67dbdf8cfe8869a5","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"GET_TABLE_DESCRIPTION","arguments_omitted":true},{"tool":"GET_DATABASE_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_TABLE_NAME","arguments_omitted":true},{"tool":"INTERPRET_DATABASE_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_TABLE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_DATABASE","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":3}} +{"public_id":"val-0285","source_trace_sha256":"c629959cd9092d81b70781b87477e30c60cb95f839eb69298b250dd974571be6","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":7}} +{"public_id":"val-0286","source_trace_sha256":"ce58f72f26442fa0e2a532ce7e64258b02a7ae68469c75ea79508c74bbbb98a4","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"GET_TABLE_DESCRIPTION","arguments_omitted":true},{"tool":"GET_DATABASE_DESCRIPTION","arguments_omitted":true},{"tool":"SELECT_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"EXPAND_SELECTED_CONTEXT","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"GET_TABLE_DESCRIPTION","arguments_omitted":true},{"tool":"GET_DATABASE_DESCRIPTION","arguments_omitted":true},{"tool":"SELECT_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"EXPAND_SELECTED_CONTEXT","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":1,"planner_attempts":4,"skill_steps":4}} +{"public_id":"val-0287","source_trace_sha256":"0ea1f37b9a714bd4545aba4288c57147ae3bf77bb865802931cf0675fa92bda5","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0288","source_trace_sha256":"c0f73c4a2d3e2dddb0dd61103cd276d77b427c33bb9e84c512626ee27e22bcf9","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0289","source_trace_sha256":"87c46399bb6dcea96529a7b5f25dd87f414488f8f508546be290393a81759e1c","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"error","tools":null,"unparseable":true},{"attempt":3,"status":"error","tools":null,"unparseable":true},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":1,"planner_attempts":4,"skill_steps":4}} +{"public_id":"val-0290","source_trace_sha256":"e7b98d7206838580c4b67c355eef1df9befab4d2094e65b7c6906f9b9100b3a7","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0291","source_trace_sha256":"bb2f8f2ed9c04d274785d60ef974f15cf856d307f189f5cf257569415e5306a9","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0292","source_trace_sha256":"de69fbfa30c6b24068714cf3280309fc4dec1ede8c53f7ec1fb6bdf225237133","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0293","source_trace_sha256":"5dc6ba3d03b135cb1e7dbe1dc1e4dc12daa432670f1ae7c47c633d1e1ea12032","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-017","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0294","source_trace_sha256":"e9cfdea3682a02d7757f477b6e0e8a91a9e4c0b4bbfd98a008687eec2c42f7e8","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0295","source_trace_sha256":"d30b66ae61e7ea94e48951614b8270301bf5b19d4aff67ae94c8c254a4a54538","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0296","source_trace_sha256":"765eb52626c616e96e95140c2f83aea31fc2216dd0753d1b02212a3db5b9debc","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0297","source_trace_sha256":"a3afcac75917bcfbed92b99c91146a2adf0c503f2ac80498143297049424ed45","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0298","source_trace_sha256":"ba89801047a5ab4cee042af4c059592a454be858ddedbac6d2793d579a899534","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-031","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0299","source_trace_sha256":"a007198613963cb786a77088f8931a2e70b0565cf7f5709aa692db5fc248c2c5","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"GET_TABLE_DESCRIPTION","arguments_omitted":true},{"tool":"GET_DATABASE_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_TABLE_NAME","arguments_omitted":true},{"tool":"INTERPRET_DATABASE_NAME","arguments_omitted":true},{"tool":"INTERPRET_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0300","source_trace_sha256":"ba62bb2409402234ae893e2d01010fa6a5291013e630bf4cea9fd92dd8066577","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0301","source_trace_sha256":"585eadae9b1532c52c73246e4b7fbfa83dcab55a432cb44f65a483e0fbb08303","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0302","source_trace_sha256":"8e89db6790dfd1eca753a6d6b5322aa4545695bffc1d9748efdebe746021961a","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-031","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0303","source_trace_sha256":"a7cb029cd59f3cf28dfc6eb21006ae7f1c393bbba54e36df843fb7c3eb678689","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0304","source_trace_sha256":"48fd8edf8c89fddce71b256a8b3d2f3f4edac0ff67246e5ed1d9e656ad7c5acc","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":8}} +{"public_id":"val-0305","source_trace_sha256":"b977ff815fcf9bfdab2b192b18877043af5b949870f9773172583e7acb121795","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0306","source_trace_sha256":"c99a3d63e6d9910b06ee9912688d9c52ee7b3d35ba003e2fb63af54808e36ff7","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0307","source_trace_sha256":"5a7bd1ee4d76993a422e6883d6db71cf67c2cd21227a8e38e3defba18e22e982","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0308","source_trace_sha256":"5bed488ed6e96b311a49be8f5416339d8810ea5eafabb54a262e2d4ab9dc265f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-027","grade_alias":"grade-002"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0309","source_trace_sha256":"bf9ef1ff3751344638b1e438e0ab83c3bbbe519f253cf0dbb4f382733298ce94","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0310","source_trace_sha256":"c03530ed11652a476bb3cc306620dec4004ae89ebf49602c27536c57dfb47318","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0311","source_trace_sha256":"b416e876d3b665e7446689c99b2d9d39d508279600a9a2291110906c005ed458","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0312","source_trace_sha256":"dd954981a1413ede1e8a9c78ede55e9fcb8d6e5c41b21f99cf687e29d72ffc4f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0313","source_trace_sha256":"d413188c629fd24d715f5e2173dfb289a7ce7a675c1ac5ab4257b8547b36bbc2","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-002","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0314","source_trace_sha256":"9fc1c4f0f8d6bfcfa29714021bed455234eedb98fa5ea17a84aa71a5fbe187e9","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0315","source_trace_sha256":"62177c9140d82e539bd9a8a0a62f41be671c4ff7f919826802d2618311ad4826","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-015","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0316","source_trace_sha256":"b938d6e2632c8607b38caffb53ae4fdb044712c0ef57cf5998cd01ec58c5a342","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-057","grade_alias":"grade-002"},"budget":{"programs":3,"planner_attempts":4,"skill_steps":9}} +{"public_id":"val-0317","source_trace_sha256":"37e12308d269ce5fdb9a3e1ed6697d003b95405caca60324b1cc53aebdd59b2c","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-002","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0318","source_trace_sha256":"6d8fbc6d4813f41a2eed10ee6be74fdd91466d528086d0303562ee0c18a480e2","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0319","source_trace_sha256":"7a9756c28912e96a3605534709d21229008cea5ce965ae73e350c054ffdae084","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0320","source_trace_sha256":"95598b85de87b918ebce53be1fda1bc626d2b8d332ccb8b186c8974f2f5b5300","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"error","tools":null,"unparseable":true},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0321","source_trace_sha256":"6f02143cb62cea8955e4eed596d87035892db435bf40b42205046c9574fc5221","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-057","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0322","source_trace_sha256":"46c97afccfcf8a812c9e87028804e845d7ab6dbe649d3f7e155ed660796f9e82","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0323","source_trace_sha256":"dc4adafde62e184e37746ed3501c7a5d27423b080cc9629f192f0943a6e67e4d","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[],"final_result":null,"budget":{"programs":0,"planner_attempts":4,"skill_steps":0}} +{"public_id":"val-0324","source_trace_sha256":"8b57b14044da88f681ecebd4942bf39e614584b615f6e41a20f4a7423407e092","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":8}} +{"public_id":"val-0325","source_trace_sha256":"2fb29c5c2e3f95ea8192d0fb60d8b243b81edc2887feec8e393e632adb2a5005","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0326","source_trace_sha256":"6f40b598dbeb19f9095d584e1d98925fc52c4faf6da6061395d9ea42124128e1","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":null,"unparseable":true},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0327","source_trace_sha256":"ad10e310537916583875ed810ed02922b673f83a75b86f659e82ed68a0bd4124","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0328","source_trace_sha256":"e2ac22cb00deec6de33dda13bfe0b0a57acc194ecc75518fcfcfd493bb7d8400","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0329","source_trace_sha256":"062e3ccdeaa1fd188fc6dd01c4f455f6f1e238d2b5bd57ab3fae702edac7ee81","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"error","tools":null,"unparseable":true},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0330","source_trace_sha256":"960ffeb963364f45db545156512c9a291f86b0b03bcd6f68abc81f325d635bf7","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"error","tools":null,"unparseable":true},{"attempt":3,"status":"error","tools":null,"unparseable":true},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":1,"planner_attempts":4,"skill_steps":2}} +{"public_id":"val-0331","source_trace_sha256":"bb6e7b21d8101340aae25b1bdf3b4373ad4a3e3decb4cb3a3202b5e7c7a2d5a7","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"error","tools":null,"unparseable":true},{"attempt":3,"status":"error","tools":null,"unparseable":true},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":1,"planner_attempts":4,"skill_steps":4}} +{"public_id":"val-0332","source_trace_sha256":"307ca4b649a743e089dbf98f9c87d39bd8d1ad33e9ce24c69f4228ad754a1db6","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"backend_error","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"error","tools":null,"unparseable":true},{"attempt":3,"status":"error","tools":null,"unparseable":true},{"attempt":4,"status":"error","tools":null,"unparseable":true}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":1,"planner_attempts":4,"skill_steps":3}} +{"public_id":"val-0333","source_trace_sha256":"09cd7808165bcb06ef44746e9b890b3a5c3c890b3ad3b4be02f5442c0818cabb","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0334","source_trace_sha256":"d68214a89393d74baf3252554d294546071cd96fe1d44ee2a1b614f7e4c087e8","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-020","grade_alias":"grade-002"},"budget":{"programs":3,"planner_attempts":3,"skill_steps":10}} +{"public_id":"val-0335","source_trace_sha256":"bd6df07cef90ca0989db34f9062c6f00f1bc508b1c916e33f87a102f32b4af30","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0336","source_trace_sha256":"9078f48659ef2ca3da957b77fc5fd0763610edd9e5fa4c04a576efc17d501ac0","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0337","source_trace_sha256":"5999bef47445533b422d82a1ccf6273d7d8c2d3cb4e1e0e25ed4e9e54019228a","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0338","source_trace_sha256":"8d2bb1eccc7111c8af426afbb5449ebe2bce7ab9ed889c52c4df7565910f0eac","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0339","source_trace_sha256":"8ca59814032cc8ee78ff2250f45a82b06fef0feb6e8359880f4002d26ca2f899","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0340","source_trace_sha256":"ad3511ee96b07334621665675d02d22c7da72f8b37abf6f5fd3b2b08704c2b34","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0341","source_trace_sha256":"406b2e0b11e47ab47c2feeb3e1e1929150f023d0f32087004646279c8af1bf2b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0342","source_trace_sha256":"b580ecf153bbb8da736f70de934d22b39e36c4879c6a8125860f448f29bbc11e","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0343","source_trace_sha256":"6ac1069a8cf0cd36b72838f83d0370d1c3078c43ba0ec4f8548298fa1af149f3","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-055","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0344","source_trace_sha256":"fa665c42bc3faad02e87ba73d79a9e1d8853895e32a89a8442e66adb0c6a2cf1","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0345","source_trace_sha256":"b423433b251103063d92f78a9ff0dde3b16242fdf5815c8ca7497be7d042982f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-002","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0346","source_trace_sha256":"c198768e61e6b0bee121eb8f2d29ad275a3e28c9e43a738480edc19ae97e4a05","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0347","source_trace_sha256":"f919a4473f71d45e11e072f8065435c4282cfcd73f99e373819b59185f53d5ad","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-034","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0348","source_trace_sha256":"defda118aa1b38fa6c19e430d2be446461b42964e3be196defe1c5e5bdd71827","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-034","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0349","source_trace_sha256":"f6548e33170bd4e54d1280a53935c9930ec971eac4675f693231a40e0db6cea4","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0350","source_trace_sha256":"32f2325317445af2f3ab6b8fc9580eb6e753a2c1b7953e21d81b9b5926f0b2ae","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0351","source_trace_sha256":"09e5f1cf7e0b79a7356c856ff055e5ea5806acaf547a1e2210100ba1ca9d3251","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INFER_CATEGORY_FROM_TOPK","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-055","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":7}} +{"public_id":"val-0352","source_trace_sha256":"17ec4590114d060ffbfa40109034ed0db454e16d9b0394979615e7a4e37eda03","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0353","source_trace_sha256":"804763a62f0d0af06418766aa8d7bf1fc728317f3ddb0a749330006e4e4fdf66","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0354","source_trace_sha256":"628e7fd30581eddedf014bfe6c7022057853fb813b242bb7be42d9c71cb0c54f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"ALIGN_FIELD_TO_STANDARD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0355","source_trace_sha256":"36d92225a1278a1096fa27c60b820ed1e7e2a89d3442de113b6cbce54ea4dab6","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-057","grade_alias":"grade-002"},"budget":{"programs":2,"planner_attempts":2,"skill_steps":8}} +{"public_id":"val-0356","source_trace_sha256":"1b91b8a31cc5b046b0ae1babd8f36028c268873862fe7e6e756d171d028d6ede","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0357","source_trace_sha256":"ad79fe6ca03edd9099a8c7c61a3238591fc57efdcf4a2f3d6c2291a3e712d5a2","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0358","source_trace_sha256":"23352b1e42ba9f741f5db298e55ac1f2b1d4820510745367b6dc18083fdec79e","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"GET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"GET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0359","source_trace_sha256":"3121a6ece6157bd55e99392ab9575fca4a0c2194cd25235dc365727c8a4dbb20","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"GET_TABLE_DESCRIPTION","arguments_omitted":true},{"tool":"[REDACTED_TOOL]","arguments_omitted":true},{"tool":"[REDACTED_TOOL]","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"SELECT_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"EXPAND_SELECTED_CONTEXT","arguments_omitted":true},{"tool":"INTERPRET_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_CONTEXT","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0360","source_trace_sha256":"52604d2fd54e6df529779b61f8cd61f892850ac0719def667b7c8bb47aefc7fe","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-002","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0361","source_trace_sha256":"a68398a48be326c9620ff591bdd14e8f0a3a44d6a022c0742a4f3ebaf944ae71","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0362","source_trace_sha256":"29d40333ecf07e3b8939f147339662a4c220cbe672270cbda42c6d310dd59d47","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0363","source_trace_sha256":"536cbb9ad6759a4027753c31608e8b80c7e6e15bc0f316fbc437fe2330ac049e","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-001","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0364","source_trace_sha256":"64d3493b26412e70f938c8010c128da8d1e1025b448c81d7c8d58808a2f2d7d9","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-048","grade_alias":"grade-003"},"budget":{"programs":3,"planner_attempts":3,"skill_steps":10}} +{"public_id":"val-0365","source_trace_sha256":"654ec0c4ee6ccc47ab8429df296833c12d58edf08058f7e4622bb0e2a4832216","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"completed","terminated":false,"failure_index":null,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0366","source_trace_sha256":"1e15b154eefb81e08fef7bf0ca5d9cae0a2dcd9503aa8349772c2002558bccf5","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":3}} +{"public_id":"val-0367","source_trace_sha256":"eaa02b124cac3a147ac33b9520cabcb41f683cb037be916aa30e0356568f3f51","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":6}} +{"public_id":"val-0368","source_trace_sha256":"e719d9477f2b27efdcd955eabe95af68635f95e176f2d39dc289fef39e431ecb","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":5}} +{"public_id":"val-0369","source_trace_sha256":"7b8a540a26a4c3f6d02ec379b24bb64a415a1bfa74bc269a202c20290abfd761","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0370","source_trace_sha256":"83cf9bf91bd129a46e44e662296ad43c66cfa68ff8de57a73dd87a2392ec63b6","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-031","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0371","source_trace_sha256":"7d51692ca834a1239bbce570015466c141ac3115dea6c88386c6c9b43a662e9b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0372","source_trace_sha256":"0c2c36d234a899a2f24c6e0eaecf0aef7043322c38b733ceb047da832c9ddd26","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-018","grade_alias":"grade-003"},"budget":{"programs":2,"planner_attempts":3,"skill_steps":8}} +{"public_id":"val-0373","source_trace_sha256":"6bb97fd7a7aa34beb503b499bbba3cb7a54f3f7cae6af642ee23e461e495b442","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0374","source_trace_sha256":"d2f6903a02f5403584e783252885c38b1487f37eaf9c35328b6ef7f35e36c1de","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0375","source_trace_sha256":"990ecc3efd3bf8d3f0ef9361863a0575448633a8cfe5c87486d1ddd39f42f29e","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-002","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0376","source_trace_sha256":"8f7ee1e8e1cbfd565c14f441beabde175e07cd4b6c9898e9ebeac0ef3dcbb536","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-002","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0377","source_trace_sha256":"a0da9a49fd8f8fa3846cb07eb938ab5d104e03a2ec25d24ce1c45fdcf46bc2d6","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0378","source_trace_sha256":"89f052398032ca907b1cd7850488aae4c9000787d665edc88a9cfe6c574d3792","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-018","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0379","source_trace_sha256":"8a5f7219ca727c59bac2899c616d3eb7000ff9f2f6d7aa97b839ee613ff32647","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-043","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0380","source_trace_sha256":"8036022583e67afe49dcbda9956c33064b23f48aa1a77c19fd8c36b8392e86c3","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-018","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0381","source_trace_sha256":"3836209cc551961248c3fd83e76d9918cb352728518bbfed0be00853e57df8c1","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0382","source_trace_sha256":"d2608ce56cb9823d9b184daa1a6e6243a6ef4d3b8a55409b6855856bc3cbe1d0","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0383","source_trace_sha256":"8981d28a87bc7f70ae10483a7752490e4c9e4e43c50faac5e9f11888027f5f7f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-034","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0384","source_trace_sha256":"b9653af9849e28713a8df4121718b7053bc27c444b69e651f263030fab996595","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0385","source_trace_sha256":"36430f6f5396b3f7bcb724287979aa1d25f338abd7002cf8b6c4e3978dfbb8c5","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0386","source_trace_sha256":"37c11051c6f5f32ea473fa127d9ffc973a9735b11a079aebfeb0622c7bb22838","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"planner_budget_exhausted","planner_attempts":[{"attempt":1,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"rejected","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":2,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0387","source_trace_sha256":"1eeac19cc4c8854fc526f608967a787165f9530474b1e30e6283d6557b08fe2b","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":8}} +{"public_id":"val-0388","source_trace_sha256":"36f3b6dbc73b21deaff44654e7c9f1ffb82a782618b04166e1d33c6da1cec8c6","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-049","grade_alias":"grade-003"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0389","source_trace_sha256":"e657acbfab6202fa3be7e0dc6bdd69bab4949a4b5814b6d271b635f883a410aa","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"INTERPRET_DOMAIN","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_DOMAIN","arguments_omitted":true},{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"INTERPRET_DOMAIN","arguments_omitted":true},"status":"success"},{"call":{"tool":"SYNTHESIZE_METADATA_SEMANTICS","arguments_omitted":true},"status":"error"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":7}} +{"public_id":"val-0390","source_trace_sha256":"08281a696344106941d78ca941a99e45f4ed3b94b883aad363ad8352b390199c","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"GET_TABLE_FIELDS","arguments_omitted":true},{"tool":"SELECT_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"EXPAND_SELECTED_CONTEXT","arguments_omitted":true},{"tool":"INTERPRET_CONTEXT_FIELDS","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_CONTEXT","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0391","source_trace_sha256":"07cab0dd9d658685af219a3e3bf110e0e6eed9a967a8affa80deede2314c0830","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0392","source_trace_sha256":"26fba99c41e172ffead0796118a28134ef8864d7284144d73bb0a951ff979010","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0393","source_trace_sha256":"6b828070b5974955bb936bd05b0d07260bf01406da7d050616ee5c20c8795f19","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"accepted","tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":3,"tools":[{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":1,"steps":[{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":3,"skill_steps":8}} +{"public_id":"val-0394","source_trace_sha256":"b6a2a5364500bf4b716a9b1bad54d23b4fec68d585cd384321361f5a45fd6cf1","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_DESCRIPTION","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},"status":"blocked"}]},{"program_index":2,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_TYPE","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":6}} +{"public_id":"val-0395","source_trace_sha256":"1207b10b81f5295a29adada474f369b5d94fff0c0517b64cdb0782cde4ea895f","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":false,"stop_reason":"program_budget_exhausted","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":2,"status":"accepted","tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"unparseable":false},{"attempt":3,"status":"rejected","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false},{"attempt":4,"status":"accepted","tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":3,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"error"}]},{"program_index":2,"tools":[{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":0,"steps":[{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]},{"program_index":3,"tools":[{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"other","terminated":false,"failure_index":2,"steps":[{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"blocked"}]}],"final_result":null,"budget":{"programs":3,"planner_attempts":4,"skill_steps":8}} +{"public_id":"val-0396","source_trace_sha256":"4850893af82eaf58f7784fb95e8b41751e2767127b3b5849d4077ad3e14e52ec","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-036","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}} +{"public_id":"val-0397","source_trace_sha256":"b612824722ef923a297d36ebf06f00ec6923bb29da2c9c4573a9efeddd672e8a","privacy":{"lossy":true,"arguments_and_state_references_omitted":true,"category_and_grade_are_opaque_aliases":true,"not_training_data":true,"full_replay_supported":false,"gold_and_private_source_ids_included":false},"receipt_status":"completed","terminated":true,"stop_reason":"validated_terminal","planner_attempts":[{"attempt":1,"status":"accepted","tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"unparseable":false}],"programs":[{"program_index":1,"tools":[{"tool":"GET_FIELD_METADATA","arguments_omitted":true},{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},{"tool":"INFER_CATEGORY","arguments_omitted":true},{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true}],"status":"terminated","terminated":true,"failure_index":null,"steps":[{"call":{"tool":"GET_FIELD_METADATA","arguments_omitted":true},"status":"success"},{"call":{"tool":"INTERPRET_FIELD_NAME","arguments_omitted":true},"status":"success"},{"call":{"tool":"SEARCH_STANDARD_BY_FIELD","arguments_omitted":true},"status":"success"},{"call":{"tool":"COMPARE_STANDARD_CANDIDATES","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_CATEGORY","arguments_omitted":true},"status":"success"},{"call":{"tool":"INFER_GRADING_LEVEL","arguments_omitted":true},"status":"success"},{"call":{"tool":"FINALIZE_CLASSIFICATION","arguments_omitted":true},"status":"success"}]}],"final_result":{"category_alias":"standard-002","grade_alias":"grade-002"},"budget":{"programs":1,"planner_attempts":1,"skill_steps":7}}