chore: harden the release pipeline — floor guard, release-PR auto-sync, publish gate, stderr fix, exit-code docs

.github/workflows/release.yml

@@ -17,6 +17,8 @@ jobs:
     outputs:
       release_created: ${{ steps.release.outputs.release_created }}
       tag_name: ${{ steps.release.outputs.tag_name }}
+      prs_created: ${{ steps.release.outputs.prs_created }}
+      pr: ${{ steps.release.outputs.pr }}
     steps:
       - uses: googleapis/release-please-action@16a9c90856f42705d54a6fda1823352bdc62cf38  # v4
         id: release
@@ -25,6 +27,64 @@ jobs:
           config-file: .github/release-please-config.json
           manifest-file: .github/.release-please-manifest.json
 
+  sync-release-pr:
+    name: Sync lockfile and floors on the release PR
+    needs: release-please
+    if: ${{ needs.release-please.outputs.prs_created == 'true' && needs.release-please.outputs.pr }}
+    runs-on: ubuntu-latest
+    # This job pushes to the release branch. Two commits landing on main in
+    # quick succession would otherwise run two of these against the same
+    # branch and the loser fails on a non-fast-forward push, reddening the
+    # release workflow for no real reason. Serialize instead of cancelling:
+    # the second run still has work to do once the first one's commit lands.
+    concurrency:
+      group: sync-release-pr
+      cancel-in-progress: false
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
+        with:
+          ref: ${{ fromJSON(needs.release-please.outputs.pr).headBranchName }}
+
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+        with:
+          python-version: "3.12"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b  # v8.1.0
+
+      # release-please bumps every pyproject version but updates neither
+      # uv.lock (CI's `uv sync --locked` then fails all jobs at the install
+      # step -- the 0.2.28/0.3.0 wall, twice) nor the root dependency floors
+      # (without which `dg update` on pip delivers the wrapper and skips the
+      # sub-packages the changelog advertises). Both are mechanical
+      # consequences of the version bumps, so regenerate them here.
+      #
+      # Caveat: this push uses GITHUB_TOKEN, which does not retrigger the
+      # PR's checks. The routine hand-edit of the release notes retriggers
+      # them; for an untouched release PR, re-run checks from the UI. If
+      # releases ever need to go out unattended, switch this push to a
+      # dedicated PAT or GitHub App token.
+      - name: Regenerate uv.lock and pin root floors
+        run: |
+          set -euo pipefail
+          uv run python scripts/check_dependency_floors.py --fix
+          uv lock
+          uv run python scripts/check_dependency_floors.py
+
+      - name: Commit and push if changed
+        run: |
+          set -euo pipefail
+          if git diff --quiet; then
+            echo "uv.lock and floors already in sync"
+            exit 0
+          fi
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add uv.lock pyproject.toml
+          git commit -m "chore: sync uv.lock and root dependency floors with release-please bumps"
+          git push
+
   build:
     name: Build packages
     needs: release-please
@@ -77,9 +137,85 @@ jobs:
           pip install --find-links "$DIST_DIR" dist/deepctl-*.whl
           deepctl --version
 
+  verify-published:
+    name: Verify release is installable from PyPI
+    needs: [release-please, publish]
+    # `pip install deepctl==X` must resolve the full dependency closure from
+    # PyPI before anything advertises the release. Two real failure modes:
+    # 0.2.27 published partially (root uninstallable, and pip *silently
+    # backtracks* to the previous version, exit 0), and the 0.3.0 rollout
+    # showed a fresh install backtracking to 0.2.26 during the CDN
+    # propagation window. Poll the actual resolver, not just the JSON
+    # endpoint, so mark-latest / deploy-web / brew never point at a version
+    # a user cannot install.
+    if: |
+      needs.release-please.outputs.release_created == 'true' &&
+      startsWith(needs.release-please.outputs.tag_name, 'v')
+    runs-on: ubuntu-latest
+    # Reads nothing and writes nothing: it installs from public PyPI. Drop the
+    # workflow-level contents/pull-requests write it would otherwise inherit.
+    permissions: {}
+    # Belt-and-braces ceiling for a hung runner or a wedged pip, so three
+    # release-advertising jobs can never sit queued behind this one
+    # indefinitely. It has to clear the poll's own worst case, which is 20
+    # minutes of sleeps *plus* 60 resolution attempts -- a ceiling that cuts
+    # the loop short would kill the job before it prints the diagnosis below.
+    timeout-minutes: 45
+    steps:
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+        with:
+          python-version: "3.12"
+
+      # Poll with --dry-run: it stops after resolution, which is cheap and is
+      # exactly what distinguishes "not propagated yet" from "broken".
+      - name: Wait until pip can resolve the full release
+        run: |
+          set -euo pipefail
+          VERSION="${TAG_NAME#v}"
+          python -m venv /tmp/verify
+          /tmp/verify/bin/pip install --quiet --upgrade pip
+          # The loop exits the moment resolution succeeds, so the ceiling only
+          # costs anything when it is actually hit. Propagation is normally
+          # seconds; 20 minutes is insurance, not an expected wait.
+          for i in $(seq 1 60); do
+            if /tmp/verify/bin/pip install --dry-run --no-cache-dir \
+                "deepctl==${VERSION}" >/dev/null 2>&1; then
+              echo "deepctl==${VERSION} resolves with its full closure"
+              exit 0
+            fi
+            echo "Waiting for deepctl==${VERSION} to resolve (${i}/60)..."
+            sleep 20
+          done
+          # Every attempt above discarded its output, so the operator would
+          # otherwise get a bare error line and have to reproduce by hand.
+          # Run once more with the output visible: pip's resolver says which
+          # requirement it could not satisfy.
+          echo "Resolution still failing. Final attempt, with output:"
+          /tmp/verify/bin/pip install --dry-run --no-cache-dir "deepctl==${VERSION}" || true
+          echo "::error::deepctl==${VERSION} did not resolve from PyPI after 20 minutes -- a dependency is missing or the index has not propagated. Once PyPI has propagated, use 'Re-run all jobs' on this workflow run -- re-running only this job leaves mark-latest, deploy-web and bump-brew-formula skipped. Re-running all jobs is safe: the PyPI publish step uses skip-existing."
+          exit 1
+        env:
+          TAG_NAME: ${{ needs.release-please.outputs.tag_name }}
+
+      # Resolving is not installing. --dry-run is satisfied by PyPI metadata
+      # (often via PEP 658, without fetching a single wheel), so a corrupt
+      # artifact or an entry point that cannot import still passes it. The
+      # build job smoke-tests `deepctl --version`, but against the local
+      # dist/ artifacts -- this is the only check against what PyPI serves,
+      # and it is the last gate before three jobs advertise the release.
+      - name: Install for real and run the CLI
+        run: |
+          set -euo pipefail
+          VERSION="${TAG_NAME#v}"
+          /tmp/verify/bin/pip install --quiet --no-cache-dir "deepctl==${VERSION}"
+          /tmp/verify/bin/deepctl --version
+        env:
+          TAG_NAME: ${{ needs.release-please.outputs.tag_name }}
+
   mark-latest:
     name: Mark root release as latest
-    needs: [release-please, publish]
+    needs: [release-please, publish, verify-published]
     # Re-assert latest on the root package tag (vX.Y.Z) after PyPI publish so
     # users clicking "latest" land on a tag whose artifact is actually
     # installable. Also re-asserts after all sub-package releases since
@@ -97,7 +233,7 @@ jobs:
 
   deploy-web:
     name: Deploy web to production
-    needs: [release-please, publish]
+    needs: [release-please, publish, verify-published]
     # Only fire on root-package releases (v0.2.4, v1.0.0, …) and only after
     # PyPI publish so cli.deepgram.com never advertises a version that isn't
     # installable yet. Sub-package tags look like deepctl-cmd-listen-v0.0.3 —
@@ -128,7 +264,7 @@ jobs:
 
   bump-brew-formula:
     name: Bump Homebrew formula
-    needs: [release-please, publish]
+    needs: [release-please, publish, verify-published]
     # Only fire on root-package releases (v0.2.4, v1.0.0, …).
     # Sub-package tags look like deepctl-cmd-listen-v0.0.3 — skip those.
     if: |
@@ -143,27 +279,10 @@ jobs:
         with:
           python-version: "3.13"
 
-      - name: Wait for new deepctl version on PyPI
-        run: |
-          set -euo pipefail
-          VERSION="${TAG_NAME#v}"
-          # Poll the PyPI JSON API rather than `pip index versions`. The
-          # JSON endpoint flips the moment a release is published, while
-          # the simple index pip queries can lag 5-15 minutes behind a
-          # successful publish (CDN caching of project metadata).
-          URL="https://pypi.org/pypi/deepctl/${VERSION}/json"
-          for i in $(seq 1 30); do
-            if [ "$(curl -fsS -o /dev/null -w '%{http_code}' "${URL}")" = "200" ]; then
-              echo "deepctl==${VERSION} is live on PyPI"
-              exit 0
-            fi
-            echo "Waiting for deepctl==${VERSION} on PyPI (${i}/30)..."
-            sleep 20
-          done
-          echo "::error::deepctl==${VERSION} did not appear on PyPI after 10 minutes"
-          exit 1
-        env:
-          TAG_NAME: ${{ needs.release-please.outputs.tag_name }}
+      # No PyPI poll here: verify-published is in `needs`, and it does not
+      # finish until `pip install deepctl==X` resolves its full closure and
+      # the installed CLI runs. A second, weaker poll behind that gate can
+      # only cost time.
 
       - name: Checkout homebrew-tap
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4

.github/workflows/test.yml

@@ -4,6 +4,11 @@ on:
   push:
   pull_request:
 
+# Every job here only reads the repository: no job comments, labels, or
+# pushes. Least privilege rather than whatever the repo default happens to be.
+permissions:
+  contents: read
+
 jobs:
   test:
     name: Test Python ${{ matrix.python-version }} / ${{ matrix.os }}
@@ -56,13 +61,33 @@ jobs:
         run: uv sync --group testing --locked
 
       - name: Check formatting
-        run: uv run ruff format --check src/ packages/*/src
+        run: uv run ruff format --check src/ packages/*/src scripts/
 
       - name: Lint
-        run: uv run ruff check src/ packages/*/src
+        run: uv run ruff check src/ packages/*/src scripts/
 
       - name: Type check
-        run: uv run mypy src/ packages/*/src
+        run: uv run mypy src/ packages/*/src scripts/
+
+  floors:
+    name: Dependency floors
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
+
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+        with:
+          python-version: "3.12"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b  # v8.1.0
+
+      # Root floors must equal workspace versions or `pip install --upgrade
+      # deepctl` (what `dg update` runs) silently skips the packages whose
+      # fixes the release advertises. See scripts/check_dependency_floors.py.
+      - name: Check dependency floors
+        run: make floors-check
 
   build-test:
     name: Test Build Process
@@ -86,3 +111,31 @@ jobs:
 
       - name: Verify packages
         run: make verify-packages
+
+  # Single rollup so branch protection needs exactly one required check.
+  # Requiring the matrix contexts directly means editing repo settings every
+  # time a Python version or OS is added or dropped, and a required context
+  # that stops reporting blocks every merge until someone notices.
+  #
+  # `if: always()` is load-bearing: without it this job is *skipped* when a
+  # prerequisite fails, and a skipped required check never reports failure --
+  # it just stalls. Run always, then fail on anything that is not success, so
+  # cancelled and skipped prerequisites are failures here too.
+  all-checks:
+    name: All checks
+    needs: [test, lint, floors, build-test]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Report prerequisite results
+        run: |
+          echo "test:       ${{ needs.test.result }}"
+          echo "lint:       ${{ needs.lint.result }}"
+          echo "floors:     ${{ needs.floors.result }}"
+          echo "build-test: ${{ needs.build-test.result }}"
+
+      - name: Fail unless every prerequisite succeeded
+        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }}
+        run: |
+          echo "::error::One or more required jobs did not succeed"
+          exit 1

CONTRIBUTING.md

@@ -64,6 +64,36 @@ Or manually create a package under `packages/` following the existing pattern. E
 
 Then run `make readmes` to update all READMEs.
 
+The root `pyproject.toml` dependency is not optional bookkeeping — it is the
+delivery manifest. `pip install --upgrade deepctl` (what `dg update` runs)
+only installs what root depends on, so a published package missing from that
+list never reaches anyone. `make floors-check` fails on both that omission and
+a floor left below the workspace version; run `make floors-fix` to pin floors.
+
+### Releasing
+
+Releases are driven by release-please. Two things are worth knowing before you
+run one:
+
+**Land your release-notes edits as a commit, not a PR description edit.**
+`sync-release-pr` regenerates `uv.lock` and the root dependency floors on the
+release branch automatically, but it pushes with `GITHUB_TOKEN`, which by
+design retriggers nothing — so the PR's checks still reflect the bot's first
+commit and stay red on content that is now correct. Editing `CHANGELOG.md` on
+the branch is a commit and retriggers them. Editing the PR *description* does
+not: that fires `pull_request: edited`, which is outside the default trigger
+types. If a release ever needs to go out unattended, switch that push to a
+dedicated PAT or GitHub App token.
+
+**Nothing advertises a release until it is installable.** `verify-published`
+polls PyPI until `pip install deepctl==X` resolves its full closure, then
+installs it for real and runs the CLI. `mark-latest`, `deploy-web`, and the
+Homebrew bump all wait on it. If it times out, PyPI has not propagated or a
+sub-package failed to publish — re-run all jobs on that workflow run once
+PyPI has caught up; re-running only the failed job leaves the three
+downstream jobs skipped. Re-running all jobs is safe: the publish step uses
+`skip-existing`.
+
 ### Testing
 
 - Unit tests: `packages/*/tests/unit/`