From 6400ab5fa5445feb3e448ea40c280bb81679a0fb Mon Sep 17 00:00:00 2001 From: Ra Date: Sat, 26 Sep 2026 00:06:23 -0400 Subject: [PATCH 1/2] Add WS2 Office smoke workflow --- .github/workflows/system-tests.yml | 25 ++- .github/workflows/ws2-smoke.yml | 284 ++++++++++++++++++++++++++++ tools/ws2_bench/test_ci_workflow.py | 43 +++++ 3 files changed, 351 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ws2-smoke.yml create mode 100644 tools/ws2_bench/test_ci_workflow.py diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 3ddebda58..359e814fe 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -34,8 +34,31 @@ on: description: "Run ID to use as baseline for metric comparison (blank = latest successful run on main)" default: "" required: false + ws2_smoke: + description: "Run only the pinned WS2 Office/Kim smoke workflow" + type: boolean + default: false + required: false + ws2_office_usd: + description: "Container-visible Office 4.5 USD path or Nucleus URL" + default: "omniverse://airlab-nucleus.andrew.cmu.edu/NVIDIA/Assets/Isaac/4.5/Isaac/Environments/Office/office.usd" + required: false + ws2_rebuild_airstack_images: + description: "Build AirStack images when registry pulls are incomplete" + type: boolean + default: false + required: false jobs: + ws2-smoke: + name: WS2 Office smoke + if: github.event_name == 'workflow_dispatch' && inputs.ws2_smoke == true + uses: ./.github/workflows/ws2-smoke.yml + with: + office_usd: ${{ inputs.ws2_office_usd }} + rebuild_airstack_images: ${{ inputs.ws2_rebuild_airstack_images }} + secrets: inherit + run-tests: name: Run Tests runs-on: [self-hosted, airstack-ephemeral] @@ -50,7 +73,7 @@ jobs: # gate is what keeps random commenters from running code on the # self-hosted runner. if: | - github.event_name == 'workflow_dispatch' || + (github.event_name == 'workflow_dispatch' && inputs.ws2_smoke != true) || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name == 'issue_comment' && diff --git a/.github/workflows/ws2-smoke.yml b/.github/workflows/ws2-smoke.yml new file mode 100644 index 000000000..dcfa0664f --- /dev/null +++ b/.github/workflows/ws2-smoke.yml @@ -0,0 +1,284 @@ +name: WS2 Office Smoke + +on: + workflow_call: + inputs: + office_usd: + description: Container-visible Office 4.5 USD path or Nucleus URL + type: string + default: omniverse://airlab-nucleus.andrew.cmu.edu/NVIDIA/Assets/Isaac/4.5/Isaac/Environments/Office/office.usd + required: false + rebuild_airstack_images: + description: Build AirStack images when registry pulls are incomplete + type: boolean + default: false + required: false + workflow_dispatch: + inputs: + office_usd: + description: Container-visible Office 4.5 USD path or Nucleus URL + type: string + default: omniverse://airlab-nucleus.andrew.cmu.edu/NVIDIA/Assets/Isaac/4.5/Isaac/Environments/Office/office.usd + required: true + rebuild_airstack_images: + description: Build AirStack images when registry pulls are incomplete + type: boolean + default: false + required: true + +permissions: + contents: read + +jobs: + preflight: + name: Validate bench source + runs-on: ubuntu-latest + outputs: + tested_sha: ${{ steps.identity.outputs.tested_sha }} + steps: + - name: Checkout requested revision + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Record immutable revision + id: identity + run: echo "tested_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install CPU test dependencies + run: pip install pytest pyyaml + + - name: Resolve the smoke scenario + run: | + python3 tools/ws2_bench/episode.py \ + tools/ws2_bench/scenarios/validate_timeout.yaml \ + --resolve-only + + - name: Run WS2 CPU contracts + run: >- + python3 -m pytest -q + tools/ws2_bench/test_conditions.py + tools/ws2_bench/test_ci_workflow.py + + smoke: + name: Kim timeout smoke on OSMO GPU + needs: preflight + runs-on: [self-hosted, airstack-ephemeral] + timeout-minutes: 180 + concurrency: + group: ws2-office-smoke-${{ github.ref }} + cancel-in-progress: false + env: + WS2_OFFICE_USD: ${{ inputs.office_usd }} + WS2_OUTPUT: robot/ros_ws/ws2_runtime/ci-smoke-${{ github.run_id }}-${{ github.run_attempt }} + DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} + steps: + - name: Checkout immutable AirStack revision + uses: actions/checkout@v4 + with: + ref: ${{ needs.preflight.outputs.tested_sha }} + submodules: recursive + + - name: Clone pinned planner companions as siblings + run: | + workspace_root="$(dirname "$GITHUB_WORKSPACE")" + git clone --recursive https://github.com/engcang/MonoNav.git "$workspace_root/MonoNav" + git -C "$workspace_root/MonoNav" checkout cc5ec88 + git clone --recursive https://github.com/engcang/Collision-avoidance.git "$workspace_root/Collision-avoidance" + git -C "$workspace_root/Collision-avoidance" checkout 395485b + + - name: Install host dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends python3-venv unzip + python3 -m venv .venv + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" + .venv/bin/pip install --upgrade pip + .venv/bin/pip install pytest pyyaml + + - name: Create Isaac Sim credential file + env: + CI_OMNI_USER: ${{ secrets.WS2_OMNI_USER }} + CI_OMNI_PASS: ${{ secrets.WS2_OMNI_PASS }} + run: | + mkdir -p simulation/isaac-sim/docker + printf 'OMNI_USER=%s\n' "${CI_OMNI_USER:-guest}" > simulation/isaac-sim/docker/omni_pass.env + printf 'OMNI_PASS=%s\n' "${CI_OMNI_PASS:-guest}" >> simulation/isaac-sim/docker/omni_pass.env + printf 'OMNI_SERVER=%s\n' 'omniverse://airlab-nucleus.andrew.cmu.edu/NVIDIA/Assets/Isaac/5.1' >> simulation/isaac-sim/docker/omni_pass.env + printf 'ACCEPT_EULA=Y\nOMNI_ENV_PRIVACY_CONSENT=Y\n' >> simulation/isaac-sim/docker/omni_pass.env + + - name: Log in to internal Docker registry + id: docker_login + if: ${{ vars.DOCKER_REGISTRY_URL != '' && env.DOCKER_REGISTRY_PASSWORD != '' }} + uses: docker/login-action@v3 + with: + registry: ${{ vars.DOCKER_REGISTRY_URL }} + username: ${{ vars.DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} + + - name: Pull or build AirStack images + env: + AIRSTACK_ROOT: ${{ github.workspace }} + REBUILD: ${{ inputs.rebuild_airstack_images }} + run: | + chmod +x airstack.sh + export COMPOSE_PROFILES=desktop,isaac-sim + ./airstack.sh --progress=quiet image-pull --ignore-pull-failures || true + missing=() + while IFS= read -r image; do + [[ -z "$image" ]] && continue + docker image inspect "$image" >/dev/null 2>&1 || missing+=("$image") + done < <(docker compose -f docker-compose.yaml config --images) + if (( ${#missing[@]} > 0 )); then + printf 'Missing AirStack image: %s\n' "${missing[@]}" + if [[ "$REBUILD" != "true" ]]; then + echo "::error::Images are missing. Re-dispatch with rebuild_airstack_images=true." + exit 2 + fi + ./airstack.sh --progress=quiet image-build + fi + + - name: Build pinned Kim planner image and install weights + run: | + workspace_root="$(dirname "$GITHUB_WORKSPACE")" + cd "$workspace_root/Collision-avoidance" + bash docker/build_image.sh + bash docker/download_models.sh + + - name: Create and prepare stopped AirStack containers + env: + AIRSTACK_ROOT: ${{ github.workspace }} + COMPOSE_PROFILES: desktop,isaac-sim + AUTOLAUNCH: "false" + DISPLAY: "" + run: | + ./airstack.sh up isaac-sim robot-desktop + docker exec airstack-robot-desktop-1 bash -lc \ + 'bws --packages-select mononav_bridge' + docker stop --timeout 3 isaac-sim airstack-robot-desktop-1 + + - name: Record smoke provenance + env: + TESTED_SHA: ${{ needs.preflight.outputs.tested_sha }} + run: | + python3 - <<'PYEOF' + import json, os, subprocess + from pathlib import Path + payload = { + "schema_version": 1, + "tested_sha": os.environ["TESTED_SHA"], + "github_run_id": os.environ["GITHUB_RUN_ID"], + "github_run_attempt": os.environ["GITHUB_RUN_ATTEMPT"], + "office_usd": os.environ["WS2_OFFICE_USD"], + "mononav_commit": "cc5ec88", + "collision_avoidance_commit": "395485b", + "gpu": subprocess.run( + ["nvidia-smi", "--query-gpu=name,uuid", "--format=csv,noheader"], + check=False, capture_output=True, text=True, + ).stdout.splitlines(), + } + Path(os.environ["WS2_OUTPUT"] + "-provenance.json").write_text( + json.dumps(payload, indent=2) + "\n", encoding="utf-8" + ) + PYEOF + + - name: Run short simulated-time episode + id: episode + env: + WS2_HEADLESS: "1" + run: | + set +e + python3 tools/ws2_bench/episode.py \ + tools/ws2_bench/scenarios/validate_timeout.yaml \ + --output "$WS2_OUTPUT" > "$WS2_OUTPUT-stdout.json" + status=$? + set -e + echo "status=$status" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Stop smoke-owned containers + if: always() + run: | + docker stop --timeout 3 \ + ws2-episode-worker collision-avoidance-airstack mononav-airstack \ + isaac-sim airstack-robot-desktop-1 2>/dev/null || true + + - name: Verify expected smoke result + if: always() + id: verify + run: | + set +e + python3 - <<'PYEOF' + import json, os, shutil + from pathlib import Path + output = Path(os.environ["WS2_OUTPUT"]) + output.mkdir(parents=True, exist_ok=True) + for source, target in ( + (Path(str(output) + "-provenance.json"), output / "ci-provenance.json"), + (Path(str(output) + "-stdout.json"), output / "episode-stdout.json"), + ): + if source.is_file(): + shutil.copy2(source, target) + path = output / "result.json" + errors = [] + if not path.is_file(): + errors.append("result.json is missing") + result = {} + else: + result = json.loads(path.read_text(encoding="utf-8")) + if result.get("outcome") == "infrastructure_error": + errors.append("episode reported infrastructure_error") + if result.get("termination", {}).get("reason") != "simulation_time_budget": + errors.append("episode did not reach the deliberate simulation-time budget") + if result.get("planner_command_count", 0) < 1: + errors.append("planner published no commands") + if result.get("cleanup_errors"): + errors.append("episode cleanup reported errors") + summary = [ + "# WS2 Office smoke", + "", + f"- Outcome: `{result.get('outcome', 'missing')}`", + f"- Termination: `{result.get('termination', {}).get('reason', 'missing')}`", + f"- Planner commands: `{result.get('planner_command_count', 0)}`", + f"- Cleanup errors: `{len(result.get('cleanup_errors', []))}`", + ] + if errors: + summary += ["", "## Errors", *[f"- {error}" for error in errors]] + (output / "ci-summary.md").write_text("\n".join(summary) + "\n", encoding="utf-8") + raise SystemExit(1 if errors else 0) + PYEOF + status=$? + set -e + echo "status=$status" >> "$GITHUB_OUTPUT" + cat "$WS2_OUTPUT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" + exit 0 + + - name: Upload smoke evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: ws2-office-smoke-${{ needs.preflight.outputs.tested_sha }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.WS2_OUTPUT }}/ + if-no-files-found: error + retention-days: 30 + compression-level: 0 + + - name: Enforce smoke conclusion + if: always() + env: + EPISODE_STATUS: ${{ steps.episode.outputs.status }} + VERIFY_STATUS: ${{ steps.verify.outputs.status }} + run: | + if [[ -z "$EPISODE_STATUS" || "$EPISODE_STATUS" != "0" ]]; then + echo "::error::Episode process failed with status ${EPISODE_STATUS:-missing}." + exit "${EPISODE_STATUS:-4}" + fi + if [[ -z "$VERIFY_STATUS" || "$VERIFY_STATUS" != "0" ]]; then + echo "::error::Smoke verification failed with status ${VERIFY_STATUS:-missing}." + exit "${VERIFY_STATUS:-4}" + fi diff --git a/tools/ws2_bench/test_ci_workflow.py b/tools/ws2_bench/test_ci_workflow.py new file mode 100644 index 000000000..7a95ef089 --- /dev/null +++ b/tools/ws2_bench/test_ci_workflow.py @@ -0,0 +1,43 @@ +"""Static safety contract for the manually dispatched WS2 smoke workflow.""" + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_ws2_smoke_workflow_contract(): + path = ROOT / ".github/workflows/ws2-smoke.yml" + workflow = yaml.load(path.read_text(encoding="utf-8"), Loader=yaml.BaseLoader) + assert set(workflow["on"]) == {"workflow_call", "workflow_dispatch"} + assert workflow["permissions"] == {"contents": "read"} + + smoke = workflow["jobs"]["smoke"] + assert smoke["runs-on"] == ["self-hosted", "airstack-ephemeral"] + assert smoke["concurrency"]["cancel-in-progress"] == "false" + steps = {step["name"]: step for step in smoke["steps"]} + assert steps["Stop smoke-owned containers"]["if"] == "always()" + assert steps["Verify expected smoke result"]["if"] == "always()" + assert steps["Upload smoke evidence"]["if"] == "always()" + assert steps["Enforce smoke conclusion"]["if"] == "always()" + + +def test_existing_dispatch_entry_point_can_call_ws2_smoke(): + path = ROOT / ".github/workflows/system-tests.yml" + workflow = yaml.load(path.read_text(encoding="utf-8"), Loader=yaml.BaseLoader) + inputs = workflow["on"]["workflow_dispatch"]["inputs"] + assert inputs["ws2_smoke"]["type"] == "boolean" + job = workflow["jobs"]["ws2-smoke"] + assert job["uses"] == "./.github/workflows/ws2-smoke.yml" + assert job["secrets"] == "inherit" + + +def test_ws2_smoke_pins_companion_revisions(): + text = (ROOT / ".github/workflows/ws2-smoke.yml").read_text(encoding="utf-8") + assert 'workspace_root="$(dirname "$GITHUB_WORKSPACE")"' in text + assert 'dirname "$(dirname "$GITHUB_WORKSPACE")"' not in text + assert "checkout cc5ec88" in text + assert "checkout 395485b" in text + assert "validate_timeout.yaml" in text From b6331b606da1103c93052ff627e03aa1b36054af Mon Sep 17 00:00:00 2001 From: Ra Date: Sat, 26 Sep 2026 15:55:55 -0400 Subject: [PATCH 2/2] Reuse system test harness for WS2 smoke --- .github/workflows/system-tests.yml | 66 ++++--- .github/workflows/ws2-smoke.yml | 284 ---------------------------- tests/pytest.ini | 1 + tests/requirements.txt | 1 + tests/test_ws2_smoke.py | 76 ++++++++ tools/ws2_bench/test_ci_workflow.py | 43 ----- 6 files changed, 115 insertions(+), 356 deletions(-) delete mode 100644 .github/workflows/ws2-smoke.yml create mode 100644 tests/test_ws2_smoke.py delete mode 100644 tools/ws2_bench/test_ci_workflow.py diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 359e814fe..21c0899cb 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -10,8 +10,8 @@ on: marks: description: "pytest marks expression (e.g. 'build_docker', 'liveliness', 'takeoff_hover_land'). \ Use 'or' to combine marks: 'liveliness or takeoff_hover_land'. Leave blank to run all marks. \ - Note: 'build_packages' is automatically prepended whenever any marks are specified, \ - to ensure code is built before launch tests run." + Note: 'build_packages' is automatically prepended for launch tests. \ + The self-contained 'ws2_smoke' mark is the exception." default: "liveliness or takeoff_hover_land" required: false sim: @@ -34,31 +34,8 @@ on: description: "Run ID to use as baseline for metric comparison (blank = latest successful run on main)" default: "" required: false - ws2_smoke: - description: "Run only the pinned WS2 Office/Kim smoke workflow" - type: boolean - default: false - required: false - ws2_office_usd: - description: "Container-visible Office 4.5 USD path or Nucleus URL" - default: "omniverse://airlab-nucleus.andrew.cmu.edu/NVIDIA/Assets/Isaac/4.5/Isaac/Environments/Office/office.usd" - required: false - ws2_rebuild_airstack_images: - description: "Build AirStack images when registry pulls are incomplete" - type: boolean - default: false - required: false jobs: - ws2-smoke: - name: WS2 Office smoke - if: github.event_name == 'workflow_dispatch' && inputs.ws2_smoke == true - uses: ./.github/workflows/ws2-smoke.yml - with: - office_usd: ${{ inputs.ws2_office_usd }} - rebuild_airstack_images: ${{ inputs.ws2_rebuild_airstack_images }} - secrets: inherit - run-tests: name: Run Tests runs-on: [self-hosted, airstack-ephemeral] @@ -73,7 +50,7 @@ jobs: # gate is what keeps random commenters from running code on the # self-hosted runner. if: | - (github.event_name == 'workflow_dispatch' && inputs.ws2_smoke != true) || + github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name == 'issue_comment' && @@ -189,11 +166,17 @@ jobs: marks = args[i + 1] marks_idx = i + 1 + # WS2 owns its Isaac launch and is not parametrized over simulators. + if 'ws2_smoke' in marks: + sim = 'isaacsim' + # When the user specified any marks, prepend build_packages so code # is built before launch tests try to use it. Skipped when no marks # are given (pytest runs everything including build_packages) and - # when build_packages is already in the expression. - if marks and 'build_packages' not in marks: + # when build_packages is already in the expression. WS2 prepares the + # one package it needs below and must leave its stopped containers in + # place for episode.py. + if marks and 'build_packages' not in marks and 'ws2_smoke' not in marks: marks = f'build_packages or {marks}' args[marks_idx] = marks @@ -202,6 +185,7 @@ jobs: with open(os.environ['GITHUB_OUTPUT'], 'a') as f: f.write(f'pytest_args={quoted}\n') + f.write(f'marks={marks}\n') f.write(f'sim={sim}\n') f.write(f'skip_image_prep={"true" if skip_prep else "false"}\n') @@ -223,7 +207,7 @@ jobs: const args = ${{ toJSON(steps.parse.outputs.pytest_args) }}; const cmd = `pytest tests/ ${args}`.trim(); const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const note = `Note: \`build_packages\` is automatically prepended whenever any marks are specified, to ensure code is built before launch tests run.`; + const note = `Note: \`build_packages\` is automatically prepended for launch tests; the self-contained \`ws2_smoke\` mark is the exception.`; await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, @@ -350,10 +334,34 @@ jobs: echo "All required images present after pull — skipping build." fi + - name: Prepare WS2 Kim smoke + if: contains(steps.parse.outputs.marks, 'ws2_smoke') + env: + AIRSTACK_ROOT: ${{ github.workspace }} + AUTOLAUNCH: "false" + COMPOSE_PROFILES: desktop,isaac-sim + DISPLAY: "" + run: | + workspace_root="$(dirname "$GITHUB_WORKSPACE")" + git clone --recursive https://github.com/engcang/Collision-avoidance.git "$workspace_root/Collision-avoidance" + git -C "$workspace_root/Collision-avoidance" checkout 395485b + git -C "$workspace_root/Collision-avoidance" submodule update --init --recursive + ( + cd "$workspace_root/Collision-avoidance" + bash docker/build_image.sh + bash docker/download_models.sh + ) + ./airstack.sh up isaac-sim robot-desktop + docker exec airstack-robot-desktop-1 bash -ic 'bws --packages-select mononav_bridge' + docker stop --timeout 3 isaac-sim airstack-robot-desktop-1 + echo "WS2_SMOKE_READY=1" >> "$GITHUB_ENV" + - name: Run tests env: AIRSTACK_ROOT: ${{ github.workspace }} DISPLAY: "" + WS2_HEADLESS: "1" + WS2_OFFICE_USD: ${{ vars.WS2_OFFICE_USD || 'omniverse://airlab-nucleus.andrew.cmu.edu/NVIDIA/Assets/Isaac/4.5/Isaac/Environments/Office/office.usd' }} PYTEST_ARGS: ${{ steps.parse.outputs.pytest_args }} run: | # Re-split the shell-quoted args from the parse step so we forward diff --git a/.github/workflows/ws2-smoke.yml b/.github/workflows/ws2-smoke.yml deleted file mode 100644 index dcfa0664f..000000000 --- a/.github/workflows/ws2-smoke.yml +++ /dev/null @@ -1,284 +0,0 @@ -name: WS2 Office Smoke - -on: - workflow_call: - inputs: - office_usd: - description: Container-visible Office 4.5 USD path or Nucleus URL - type: string - default: omniverse://airlab-nucleus.andrew.cmu.edu/NVIDIA/Assets/Isaac/4.5/Isaac/Environments/Office/office.usd - required: false - rebuild_airstack_images: - description: Build AirStack images when registry pulls are incomplete - type: boolean - default: false - required: false - workflow_dispatch: - inputs: - office_usd: - description: Container-visible Office 4.5 USD path or Nucleus URL - type: string - default: omniverse://airlab-nucleus.andrew.cmu.edu/NVIDIA/Assets/Isaac/4.5/Isaac/Environments/Office/office.usd - required: true - rebuild_airstack_images: - description: Build AirStack images when registry pulls are incomplete - type: boolean - default: false - required: true - -permissions: - contents: read - -jobs: - preflight: - name: Validate bench source - runs-on: ubuntu-latest - outputs: - tested_sha: ${{ steps.identity.outputs.tested_sha }} - steps: - - name: Checkout requested revision - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Record immutable revision - id: identity - run: echo "tested_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install CPU test dependencies - run: pip install pytest pyyaml - - - name: Resolve the smoke scenario - run: | - python3 tools/ws2_bench/episode.py \ - tools/ws2_bench/scenarios/validate_timeout.yaml \ - --resolve-only - - - name: Run WS2 CPU contracts - run: >- - python3 -m pytest -q - tools/ws2_bench/test_conditions.py - tools/ws2_bench/test_ci_workflow.py - - smoke: - name: Kim timeout smoke on OSMO GPU - needs: preflight - runs-on: [self-hosted, airstack-ephemeral] - timeout-minutes: 180 - concurrency: - group: ws2-office-smoke-${{ github.ref }} - cancel-in-progress: false - env: - WS2_OFFICE_USD: ${{ inputs.office_usd }} - WS2_OUTPUT: robot/ros_ws/ws2_runtime/ci-smoke-${{ github.run_id }}-${{ github.run_attempt }} - DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} - steps: - - name: Checkout immutable AirStack revision - uses: actions/checkout@v4 - with: - ref: ${{ needs.preflight.outputs.tested_sha }} - submodules: recursive - - - name: Clone pinned planner companions as siblings - run: | - workspace_root="$(dirname "$GITHUB_WORKSPACE")" - git clone --recursive https://github.com/engcang/MonoNav.git "$workspace_root/MonoNav" - git -C "$workspace_root/MonoNav" checkout cc5ec88 - git clone --recursive https://github.com/engcang/Collision-avoidance.git "$workspace_root/Collision-avoidance" - git -C "$workspace_root/Collision-avoidance" checkout 395485b - - - name: Install host dependencies - run: | - sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends python3-venv unzip - python3 -m venv .venv - echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" - .venv/bin/pip install --upgrade pip - .venv/bin/pip install pytest pyyaml - - - name: Create Isaac Sim credential file - env: - CI_OMNI_USER: ${{ secrets.WS2_OMNI_USER }} - CI_OMNI_PASS: ${{ secrets.WS2_OMNI_PASS }} - run: | - mkdir -p simulation/isaac-sim/docker - printf 'OMNI_USER=%s\n' "${CI_OMNI_USER:-guest}" > simulation/isaac-sim/docker/omni_pass.env - printf 'OMNI_PASS=%s\n' "${CI_OMNI_PASS:-guest}" >> simulation/isaac-sim/docker/omni_pass.env - printf 'OMNI_SERVER=%s\n' 'omniverse://airlab-nucleus.andrew.cmu.edu/NVIDIA/Assets/Isaac/5.1' >> simulation/isaac-sim/docker/omni_pass.env - printf 'ACCEPT_EULA=Y\nOMNI_ENV_PRIVACY_CONSENT=Y\n' >> simulation/isaac-sim/docker/omni_pass.env - - - name: Log in to internal Docker registry - id: docker_login - if: ${{ vars.DOCKER_REGISTRY_URL != '' && env.DOCKER_REGISTRY_PASSWORD != '' }} - uses: docker/login-action@v3 - with: - registry: ${{ vars.DOCKER_REGISTRY_URL }} - username: ${{ vars.DOCKER_REGISTRY_USERNAME }} - password: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} - - - name: Pull or build AirStack images - env: - AIRSTACK_ROOT: ${{ github.workspace }} - REBUILD: ${{ inputs.rebuild_airstack_images }} - run: | - chmod +x airstack.sh - export COMPOSE_PROFILES=desktop,isaac-sim - ./airstack.sh --progress=quiet image-pull --ignore-pull-failures || true - missing=() - while IFS= read -r image; do - [[ -z "$image" ]] && continue - docker image inspect "$image" >/dev/null 2>&1 || missing+=("$image") - done < <(docker compose -f docker-compose.yaml config --images) - if (( ${#missing[@]} > 0 )); then - printf 'Missing AirStack image: %s\n' "${missing[@]}" - if [[ "$REBUILD" != "true" ]]; then - echo "::error::Images are missing. Re-dispatch with rebuild_airstack_images=true." - exit 2 - fi - ./airstack.sh --progress=quiet image-build - fi - - - name: Build pinned Kim planner image and install weights - run: | - workspace_root="$(dirname "$GITHUB_WORKSPACE")" - cd "$workspace_root/Collision-avoidance" - bash docker/build_image.sh - bash docker/download_models.sh - - - name: Create and prepare stopped AirStack containers - env: - AIRSTACK_ROOT: ${{ github.workspace }} - COMPOSE_PROFILES: desktop,isaac-sim - AUTOLAUNCH: "false" - DISPLAY: "" - run: | - ./airstack.sh up isaac-sim robot-desktop - docker exec airstack-robot-desktop-1 bash -lc \ - 'bws --packages-select mononav_bridge' - docker stop --timeout 3 isaac-sim airstack-robot-desktop-1 - - - name: Record smoke provenance - env: - TESTED_SHA: ${{ needs.preflight.outputs.tested_sha }} - run: | - python3 - <<'PYEOF' - import json, os, subprocess - from pathlib import Path - payload = { - "schema_version": 1, - "tested_sha": os.environ["TESTED_SHA"], - "github_run_id": os.environ["GITHUB_RUN_ID"], - "github_run_attempt": os.environ["GITHUB_RUN_ATTEMPT"], - "office_usd": os.environ["WS2_OFFICE_USD"], - "mononav_commit": "cc5ec88", - "collision_avoidance_commit": "395485b", - "gpu": subprocess.run( - ["nvidia-smi", "--query-gpu=name,uuid", "--format=csv,noheader"], - check=False, capture_output=True, text=True, - ).stdout.splitlines(), - } - Path(os.environ["WS2_OUTPUT"] + "-provenance.json").write_text( - json.dumps(payload, indent=2) + "\n", encoding="utf-8" - ) - PYEOF - - - name: Run short simulated-time episode - id: episode - env: - WS2_HEADLESS: "1" - run: | - set +e - python3 tools/ws2_bench/episode.py \ - tools/ws2_bench/scenarios/validate_timeout.yaml \ - --output "$WS2_OUTPUT" > "$WS2_OUTPUT-stdout.json" - status=$? - set -e - echo "status=$status" >> "$GITHUB_OUTPUT" - exit 0 - - - name: Stop smoke-owned containers - if: always() - run: | - docker stop --timeout 3 \ - ws2-episode-worker collision-avoidance-airstack mononav-airstack \ - isaac-sim airstack-robot-desktop-1 2>/dev/null || true - - - name: Verify expected smoke result - if: always() - id: verify - run: | - set +e - python3 - <<'PYEOF' - import json, os, shutil - from pathlib import Path - output = Path(os.environ["WS2_OUTPUT"]) - output.mkdir(parents=True, exist_ok=True) - for source, target in ( - (Path(str(output) + "-provenance.json"), output / "ci-provenance.json"), - (Path(str(output) + "-stdout.json"), output / "episode-stdout.json"), - ): - if source.is_file(): - shutil.copy2(source, target) - path = output / "result.json" - errors = [] - if not path.is_file(): - errors.append("result.json is missing") - result = {} - else: - result = json.loads(path.read_text(encoding="utf-8")) - if result.get("outcome") == "infrastructure_error": - errors.append("episode reported infrastructure_error") - if result.get("termination", {}).get("reason") != "simulation_time_budget": - errors.append("episode did not reach the deliberate simulation-time budget") - if result.get("planner_command_count", 0) < 1: - errors.append("planner published no commands") - if result.get("cleanup_errors"): - errors.append("episode cleanup reported errors") - summary = [ - "# WS2 Office smoke", - "", - f"- Outcome: `{result.get('outcome', 'missing')}`", - f"- Termination: `{result.get('termination', {}).get('reason', 'missing')}`", - f"- Planner commands: `{result.get('planner_command_count', 0)}`", - f"- Cleanup errors: `{len(result.get('cleanup_errors', []))}`", - ] - if errors: - summary += ["", "## Errors", *[f"- {error}" for error in errors]] - (output / "ci-summary.md").write_text("\n".join(summary) + "\n", encoding="utf-8") - raise SystemExit(1 if errors else 0) - PYEOF - status=$? - set -e - echo "status=$status" >> "$GITHUB_OUTPUT" - cat "$WS2_OUTPUT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" - exit 0 - - - name: Upload smoke evidence - if: always() - uses: actions/upload-artifact@v4 - with: - name: ws2-office-smoke-${{ needs.preflight.outputs.tested_sha }}-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ env.WS2_OUTPUT }}/ - if-no-files-found: error - retention-days: 30 - compression-level: 0 - - - name: Enforce smoke conclusion - if: always() - env: - EPISODE_STATUS: ${{ steps.episode.outputs.status }} - VERIFY_STATUS: ${{ steps.verify.outputs.status }} - run: | - if [[ -z "$EPISODE_STATUS" || "$EPISODE_STATUS" != "0" ]]; then - echo "::error::Episode process failed with status ${EPISODE_STATUS:-missing}." - exit "${EPISODE_STATUS:-4}" - fi - if [[ -z "$VERIFY_STATUS" || "$VERIFY_STATUS" != "0" ]]; then - echo "::error::Smoke verification failed with status ${VERIFY_STATUS:-missing}." - exit "${VERIFY_STATUS:-4}" - fi diff --git a/tests/pytest.ini b/tests/pytest.ini index 78a916a0c..26555970c 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -5,6 +5,7 @@ markers = liveliness: Container and process health (Docker, tmux, sentinel ROS 2 nodes) sensors: Sim and robot sensor topic rates, LiDAR validation, sim RTF takeoff_hover_land: End-to-end takeoff / hover / land action tests + ws2_smoke: Short Office/Kim planner smoke test testpaths = . addopts = -v --durations=0 cache_dir = /tmp/.pytest_cache diff --git a/tests/requirements.txt b/tests/requirements.txt index bc2a16d4d..d46d04ce5 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -4,3 +4,4 @@ pytest-dependency tabulate psutil pandas +PyYAML diff --git a/tests/test_ws2_smoke.py b/tests/test_ws2_smoke.py new file mode 100644 index 000000000..54cd33c18 --- /dev/null +++ b/tests/test_ws2_smoke.py @@ -0,0 +1,76 @@ +"""WS2 adapter for the existing AirStack system-test harness.""" + +import json +import os +import shutil +import subprocess +import uuid +from pathlib import Path + +import pytest + +import conftest + + +ROOT = Path(__file__).resolve().parents[1] +RUNTIME = ROOT / "robot/ros_ws/ws2_runtime" +SCENARIO = ROOT / "tools/ws2_bench/scenarios/validate_timeout.yaml" + + +@pytest.mark.ws2_smoke +@pytest.mark.timeout(1800) +@pytest.mark.skipif( + os.environ.get("WS2_SMOKE_READY") != "1", + reason="run through system-tests.yml with -m ws2_smoke", +) +def test_ws2_office_kim_smoke(): + output = RUNTIME / f"ci-smoke-{uuid.uuid4().hex[:12]}" + stdout = "" + returncode = -1 + try: + completed = subprocess.run( + [ + "python3", + str(ROOT / "tools/ws2_bench/episode.py"), + str(SCENARIO), + "--output", + str(output), + ], + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=1700, + check=False, + ) + stdout = completed.stdout + returncode = completed.returncode + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout or "WS2 episode timed out" + if isinstance(stdout, bytes): + stdout = stdout.decode(errors="replace") + finally: + subprocess.run( + [ + "docker", "stop", "--timeout", "3", "ws2-episode-worker", + "collision-avoidance-airstack", "isaac-sim", + "airstack-robot-desktop-1", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + artifact = conftest.RUN_DIR / "ws2-smoke" + artifact.mkdir(parents=True, exist_ok=True) + (artifact / "episode-stdout.log").write_text(stdout, encoding="utf-8") + if output.is_dir(): + shutil.copytree(output, artifact / "episode", dirs_exist_ok=True) + + assert returncode == 0, stdout[-4000:] + result_path = output / "result.json" + assert result_path.is_file(), "WS2 did not produce result.json" + result = json.loads(result_path.read_text(encoding="utf-8")) + assert result.get("outcome") != "infrastructure_error", result + assert result.get("termination", {}).get("reason") == "simulation_time_budget", result + assert result.get("planner_command_count", 0) >= 1, result + assert not result.get("cleanup_errors"), result diff --git a/tools/ws2_bench/test_ci_workflow.py b/tools/ws2_bench/test_ci_workflow.py deleted file mode 100644 index 7a95ef089..000000000 --- a/tools/ws2_bench/test_ci_workflow.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Static safety contract for the manually dispatched WS2 smoke workflow.""" - -from pathlib import Path - -import yaml - - -ROOT = Path(__file__).resolve().parents[2] - - -def test_ws2_smoke_workflow_contract(): - path = ROOT / ".github/workflows/ws2-smoke.yml" - workflow = yaml.load(path.read_text(encoding="utf-8"), Loader=yaml.BaseLoader) - assert set(workflow["on"]) == {"workflow_call", "workflow_dispatch"} - assert workflow["permissions"] == {"contents": "read"} - - smoke = workflow["jobs"]["smoke"] - assert smoke["runs-on"] == ["self-hosted", "airstack-ephemeral"] - assert smoke["concurrency"]["cancel-in-progress"] == "false" - steps = {step["name"]: step for step in smoke["steps"]} - assert steps["Stop smoke-owned containers"]["if"] == "always()" - assert steps["Verify expected smoke result"]["if"] == "always()" - assert steps["Upload smoke evidence"]["if"] == "always()" - assert steps["Enforce smoke conclusion"]["if"] == "always()" - - -def test_existing_dispatch_entry_point_can_call_ws2_smoke(): - path = ROOT / ".github/workflows/system-tests.yml" - workflow = yaml.load(path.read_text(encoding="utf-8"), Loader=yaml.BaseLoader) - inputs = workflow["on"]["workflow_dispatch"]["inputs"] - assert inputs["ws2_smoke"]["type"] == "boolean" - job = workflow["jobs"]["ws2-smoke"] - assert job["uses"] == "./.github/workflows/ws2-smoke.yml" - assert job["secrets"] == "inherit" - - -def test_ws2_smoke_pins_companion_revisions(): - text = (ROOT / ".github/workflows/ws2-smoke.yml").read_text(encoding="utf-8") - assert 'workspace_root="$(dirname "$GITHUB_WORKSPACE")"' in text - assert 'dirname "$(dirname "$GITHUB_WORKSPACE")"' not in text - assert "checkout cc5ec88" in text - assert "checkout 395485b" in text - assert "validate_timeout.yaml" in text