From 3cf84ad8b7cdafda4031f9be846f4783c62b5691 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 15 Nov 2026 14:51:18 +0900 Subject: [PATCH 1/3] [v3-4-test] Sync fork CI changes from main Several fork CI changes merged to main since the last sync never reached v3-4-test, and the fork CI monitor reads the gate configuration of a pull request's base branch, so pull requests to v3-4-test were still closed after 7 days and missed ten pilot participants. The fork CI modules, their tests, the design document and its diagrams, the monitor workflow, the gate configuration checks and the contributor documentation now match main, taking in #75814, #75696, #76272, #76316, #76327, #76510, #76538, #76677, #76681, #76835, #76836, #76837, #76839, #76841, #76866, #76267 and #77100. The pilot participants new to this branch join at 2026-11-16 00:00 UTC rather than at their main dates, as the gate configuration check requires a future instant for an entry that is new on the base branch. Generated-by: Claude Opus 5 --- .github/fork-ci-gate.toml | 12 +- .github/workflows/fork-ci-monitor.yml | 5 +- .pre-commit-config.yaml | 16 +- contributing-docs/05_pull_requests.rst | 2 +- .../18_contribution_workflow.rst | 30 +- .../32_open_pull_request_limit.rst | 3 + dev/AGENTS.md | 4 +- .../08_fork_ci_for_external_contributors.md | 298 +++++-- .../diagram_fork_ci_architecture.md5sum | 1 + .../ci/images/diagram_fork_ci_architecture.py | 80 ++ .../images/diagram_fork_ci_architecture.svg | 157 ++++ .../images/diagram_fork_ci_lifecycle.md5sum | 1 + .../ci/images/diagram_fork_ci_lifecycle.py | 94 +++ .../ci/images/diagram_fork_ci_lifecycle.svg | 283 +++++++ .../ci/images/diagram_fork_ci_states.md5sum | 1 + .../doc/ci/images/diagram_fork_ci_states.py | 104 +++ .../doc/ci/images/diagram_fork_ci_states.svg | 284 +++++++ .../src/airflow_breeze/utils/fork_ci_gate.py | 7 +- .../airflow_breeze/utils/fork_ci_github.py | 59 +- .../airflow_breeze/utils/fork_ci_messages.py | 255 +++++- .../airflow_breeze/utils/fork_ci_reconcile.py | 441 ++++++++--- dev/breeze/tests/test_fork_ci_gate.py | 3 +- dev/breeze/tests/test_fork_ci_messages.py | 193 ++++- dev/breeze/tests/test_fork_ci_reconcile.py | 727 +++++++++++++++--- scripts/ci/prek/check_fork_ci_gate_config.py | 91 ++- .../check_scheduled_workflows_skip_forks.py | 104 +++ .../ci/prek/test_check_fork_ci_gate_config.py | 215 +++++- ...st_check_scheduled_workflows_skip_forks.py | 138 ++++ 28 files changed, 3237 insertions(+), 371 deletions(-) create mode 100644 dev/breeze/doc/ci/images/diagram_fork_ci_architecture.md5sum create mode 100644 dev/breeze/doc/ci/images/diagram_fork_ci_architecture.py create mode 100644 dev/breeze/doc/ci/images/diagram_fork_ci_architecture.svg create mode 100644 dev/breeze/doc/ci/images/diagram_fork_ci_lifecycle.md5sum create mode 100644 dev/breeze/doc/ci/images/diagram_fork_ci_lifecycle.py create mode 100644 dev/breeze/doc/ci/images/diagram_fork_ci_lifecycle.svg create mode 100644 dev/breeze/doc/ci/images/diagram_fork_ci_states.md5sum create mode 100644 dev/breeze/doc/ci/images/diagram_fork_ci_states.py create mode 100644 dev/breeze/doc/ci/images/diagram_fork_ci_states.svg create mode 100755 scripts/ci/prek/check_scheduled_workflows_skip_forks.py create mode 100644 scripts/tests/ci/prek/test_check_scheduled_workflows_skip_forks.py diff --git a/.github/fork-ci-gate.toml b/.github/fork-ci-gate.toml index 1266287825a04..6a0e279febd98 100644 --- a/.github/fork-ci-gate.toml +++ b/.github/fork-ci-gate.toml @@ -32,7 +32,7 @@ enabled = false enabled_since = 2026-01-01T00:00:00Z # Days without any fork CI activity after which a gated pull request is closed. -close_after_days = 7 +close_after_days = 30 # Report each job of a gated pull request's fork run as its own commit status. Set to false when # the fork CI monitor exceeds GitHub's rate limits (see dev/AGENTS.md); the `Fork CI / Tests (AMD)` @@ -50,4 +50,14 @@ job_statuses = true # check-fork-ci-gate-config hook rejects a new or changed instant that is in the past. [pilot] # "login-three" = 2026-10-20T00:00:00Z +"aaron-y-chen" = 2026-11-16T00:00:00Z +"Andrushika" = 2026-11-16T00:00:00Z +"Dev-iL" = 2026-11-16T00:00:00Z +"justinpakzad" = 2026-11-16T00:00:00Z +"Prab-27" = 2026-11-16T00:00:00Z +"SameerMesiah97" = 2026-11-16T00:00:00Z +"sreecharan-desu" = 2026-11-16T00:00:00Z +"stephen-bracken" = 2026-11-16T00:00:00Z +"raphaelauv" = 2026-11-16T00:00:00Z +"yuseok89" = 2026-11-16T00:00:00Z "meel-ayush" = 2026-11-11T00:00:00Z diff --git a/.github/workflows/fork-ci-monitor.yml b/.github/workflows/fork-ci-monitor.yml index e95becc9e273f..a2d09f59fb03a 100644 --- a/.github/workflows/fork-ci-monitor.yml +++ b/.github/workflows/fork-ci-monitor.yml @@ -27,7 +27,8 @@ name: "Fork CI monitor" on: # yamllint disable-line rule:truthy schedule: - - cron: '*/15 * * * *' + # MONITOR_INTERVAL in fork_ci_reconcile.py, whose per-run write budgets assume this cadence. + - cron: '*/5 * * * *' workflow_dispatch: inputs: dry-run: @@ -59,7 +60,7 @@ jobs: timeout-minutes: 20 steps: # What earlier runs reported, so that a problem that persists is posted to Slack once - # rather than every 15 minutes. Losing it, for example to cache eviction, re-posts once. + # rather than on every run. Losing it, for example to cache eviction, re-posts once. # Restored before anything that can fail, so a failing setup is reported once too, and # outside the checkout, where nothing but the steps below reads it. - name: "Restore what earlier runs reported" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c3ea15b7c4c5e..ec4bef1e678a9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -331,8 +331,9 @@ repos: PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_fork_ci_gate_config.py language: python - files: ^\.github/fork-ci-gate\.toml$|^dev/breeze/src/airflow_breeze/utils/fork_ci_gate\.py$ + files: ^\.github/fork-ci-gate\.toml$|^dev/breeze/src/airflow_breeze/utils/fork_ci_gate\.py$|^\.asf\.yaml$ pass_filenames: false + additional_dependencies: ['pyyaml'] - id: check-java-sdk-version-in-sync name: Check Java SDK JDK version is consistent across build files env: @@ -1036,7 +1037,8 @@ repos: ^airflow-core/docs/.*/diagram_[^/]*\.py$| ^contributing-docs/images/diagram_[^/]*\.py$| ^docs/images/.*\.py$| - ^airflow-ctl/docs/images/diagrams/.*\.py$ + ^airflow-ctl/docs/images/diagrams/.*\.py$| + ^dev/breeze/doc/ci/images/diagram_[^/]*\.py$ pass_filenames: true - id: prevent-deprecated-sqlalchemy-usage name: Prevent deprecated sqlalchemy usage @@ -1725,6 +1727,16 @@ repos: language: python pass_filenames: true files: ^\.github/workflows/.*\.yml$ + - id: check-scheduled-workflows-skip-forks + name: Check scheduled workflows skip their cron runs in forks + env: + PYTHONPYCACHEPREFIX: .build/pycache + description: Forks that enable Actions get every cron, so each scheduled job must skip outside apache/airflow + entry: ./scripts/ci/prek/check_scheduled_workflows_skip_forks.py + language: python + pass_filenames: true + require_serial: true + files: ^\.github/workflows/.*\.yml$ - id: check-docstring-param-types name: Check that docstrings do not specify param types env: diff --git a/contributing-docs/05_pull_requests.rst b/contributing-docs/05_pull_requests.rst index 7320241b18c46..d927b218f7caa 100644 --- a/contributing-docs/05_pull_requests.rst +++ b/contributing-docs/05_pull_requests.rst @@ -272,7 +272,7 @@ and approves them automatically when all of these hold: If your PR's CI runs in your fork (it carries the ``fork ci required`` label), the CI run in ``apache/airflow`` only checks that, so none of the rules above apply. The ``CI run started`` check -waits until the fork CI monitor sees your fork's run, usually within 15 minutes, and then links it. Its runs are approved +waits until the fork CI monitor sees your fork's run, usually within 5 to 10 minutes, and then links it. Its runs are approved automatically unless the PR changes files that decide which CI jobs run and on which runners: anything under ``.github/``, ``dev/breeze/src/`` or ``scripts/ci/prek/``, or ``dev/breeze/pyproject.toml``, ``dev/breeze/uv.lock`` or ``scripts/ci/install_breeze.sh``. Then diff --git a/contributing-docs/18_contribution_workflow.rst b/contributing-docs/18_contribution_workflow.rst index 7f886d2f04f42..e6978cb32aa10 100644 --- a/contributing-docs/18_contribution_workflow.rst +++ b/contributing-docs/18_contribution_workflow.rst @@ -132,12 +132,14 @@ CI in your fork Pull requests from contributors who are not committers or collaborators are moving to running their CI in the contributor's own fork, on GitHub's free runners, instead of on the shared CI -capacity of the Apache Software Foundation (`AIP-120 -`__). +capacity of the Apache Software Foundation (`ADIP-12, formerly AIP-120 +`__). CI on a public fork costs you nothing and has no minute limit; it runs on every push to your fork, so you get feedback before a maintainer looks at your pull request. The change is being piloted: until it is switched on for everyone, it applies to pilot participants (see below) and -to pull requests a maintainer labels ``use fork ci``. +to pull requests a maintainer labels ``use fork ci``. The `pull request lifecycle +<../dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md#pull-request-lifecycle>`__ diagram +shows what your pull request shows at each step and who acts next. Set up your checkout and fork once: @@ -156,7 +158,11 @@ Set up your checkout and fork once: gh workflow enable ci-amd.yml -R /airflow GitHub Actions must also be enabled in your fork and allow all actions: ``Tests (AMD)`` uses - actions from other repositories. + actions from other repositories. When your fork's Actions settings block the workflow, its run + ends in "Startup failure" before any job runs, and the fork CI monitor reports + ``Fork CI / Tests (AMD)`` as failed with "could not start". Open the run in your fork for GitHub's + error, which can also be an invalid workflow file, fix the settings under + **Settings > Actions > General** in your fork, and push again. Keep the workflow enabled for as long as you have open pull requests. If it gets disabled, from your fork's Actions tab or by GitHub after 60 days without activity in your fork, the fork CI @@ -178,15 +184,25 @@ Set up your checkout and fork once: breeze ci audit When fork CI applies to your pull request, it is labelled ``fork ci required`` and kept in draft -until the ``Tests (AMD)`` run in your fork for its latest commit is green. A +until the ``Tests (AMD)`` workflow runs in your fork for its latest commit. A failed run does not +draft it. A ``Fork CI / Tests (AMD)`` status on the pull request links to that run, and one comment on the pull request explains the current state. Once the run is green, a pull request the monitor drafted is marked ready for review and labelled ``ready for maintainer review``; a pull request you drafted -yourself stays a draft, and gets the label, once you mark it ready. A pull request whose fork CI -does not run for 7 days is closed with an explanation; you can reopen it after setting up fork CI. If your pull request changes +yourself stays a draft, and gets the label, once you mark it ready. Marking a draft ready needs a +free slot under the `limit on open pull requests <32_open_pull_request_limit.rst>`__: at the limit, +GitHub refuses it, also when a maintainer tries, so close or finish one of your other open pull +requests first. A pull request whose fork CI +does not run for 30 days is closed with an explanation; you can reopen it after setting up fork CI. If your pull request changes files that decide which CI jobs run and where, such as ``.github/``, the breeze sources or ``scripts/ci/prek/``, a green fork run is not enough: a maintainer reviews the change and runs project CI for it. +Fork CI tests your branch as it is, while project CI tests it merged with the target branch. The +``CI passed`` check that merging requires is therefore reported only when your branch is fewer than +200 commits behind the target branch and the target branch has not changed any file your pull request +changes since your branch's base. Otherwise the pull request comment lists those files and asks you +to rebase on ``upstream/main`` (or the target branch) and push, which runs fork CI again. + **Joining the pilot.** Open a pull request that adds your GitHub login to the ``[pilot]`` table in ``.github/fork-ci-gate.toml``, with a join instant after the expected merge, for example the next day at 00:00 UTC: diff --git a/contributing-docs/32_open_pull_request_limit.rst b/contributing-docs/32_open_pull_request_limit.rst index 4dd370d0beaf7..281509f560499 100644 --- a/contributing-docs/32_open_pull_request_limit.rst +++ b/contributing-docs/32_open_pull_request_limit.rst @@ -59,6 +59,9 @@ How the limit works * Draft Pull Requests do not count towards the limit yet, because GitHub does not support counting them. Once GitHub adds that, we will start counting drafts as well - so do not rely on drafts to get around the limit. Keep unfinished work on a branch in your fork instead. +* Marking a draft ready for review makes it count, so GitHub refuses it while you are at the limit - + also when a maintainer tries to do it for you. Close or finish one of your other open Pull + Requests first, then mark the draft ready. * When you have 5 open Pull Requests, GitHub does not let you open another one. A slot frees up when one of your Pull Requests is merged or closed. * The limit counts Pull Requests you have open at the same time - there is no limit on how many diff --git a/dev/AGENTS.md b/dev/AGENTS.md index f9e999acfc431..c2ac0cbbb2fb2 100644 --- a/dev/AGENTS.md +++ b/dev/AGENTS.md @@ -54,9 +54,9 @@ workflow of `apache/airflow` shares (see "Write budget" in If the budget is exceeded, turn the job statuses off: set `job_statuses = false` in `.github/fork-ci-gate.toml`. Signs that it is exceeded are the monitor's Slack note that job -statuses "have not fit in the budget" (posted only after three runs in a row deferred them) +statuses "have not fit in the budget" (posted only after job statuses were deferred for about 45 minutes in a row) staying for hours, pull requests listed because their fork run finished more than an hour ago with -job statuses still not posted, its Slack note that GitHub rate-limited it in two runs in a row, the +job statuses still not posted, its Slack note that GitHub rate-limited it for about 30 minutes in a row, the "Rate limits left" line of its run log approaching zero, or other workflows of the repository failing on secondary rate limits. The aggregate `Fork CI / Tests (AMD)` status and `CI passed` are posted either way, so turning job statuses off loses detail, not gating. diff --git a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md index 0e1ed2cb926fc..d74f2e6a751e6 100644 --- a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md +++ b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md @@ -78,26 +78,7 @@ Non-goals: ## Overview -``` - contributor fork apache/airflow - ┌──────────────────────────────────┐ ┌──────────────────────────────────────────┐ - │ git push feature-branch │ │ pull_request event │ - │ └─ ci-amd.yml (push trigger) │ │ └─ ci-amd.yml │ - │ gate job: fork -> run │ │ gate job: eligibility -> skip │ - │ build-info: fork mode │ │ (build-info and everything after │ - │ changed files from │ │ are skipped for gated PRs) │ - │ compare API vs upstream │ │ │ - │ public runners only │ │ fork-ci-monitor.yml (every 15 minutes) │ - └──────────────────────────────────┘ │ └─ breeze ci fork-ci-reconcile │ - ▲ │ discover new gated PRs -> label │ - │ GraphQL: checkSuites │ per labelled PR: fork state -> │ - │ REST: workflow state │ draft/undraft, labels, comment, │ - └────────────────────────│ commit status with run link, │ - │ close after 7 days or when the │ - │ PR tampered with the gate │ - │ unknown failures -> Slack │ - └──────────────────────────────────────────┘ -``` +![Fork CI architecture](images/diagram_fork_ci_architecture.svg) Four pieces, in dependency order: @@ -109,6 +90,34 @@ Four pieces, in dependency order: Plus `breeze ci audit` for the contributor's local setup, and documentation. +The diagrams in this document are SVG files in [`images/`](images/), generated from their +`diagram_fork_ci_*.py` Graphviz sources by the `generate-airflow-diagrams` prek hook whenever a +source changes. The ADIP-12 wiki page embeds the same files. Edit the source, not the SVG. + +### Pull request lifecycle + +What a gated pull request shows, and who acts next, from the moment it is opened until it is merged +or closed. The [state machine](#state-machine) below is the reconciler's view of the same flow; this +diagram is the contributor's and the maintainer's view. Pull requests of committers, collaborators +and exempted organisations are not gated and follow the usual project CI flow. + +![Pull request lifecycle](images/diagram_fork_ci_lifecycle.svg) + +- **Draft** is used only while the pull request waits for its author to set up fork CI or push. A + failed run leaves the draft state unchanged and removes `ready for maintainer review`, so a flaky + test after a rebase cannot draft an approved pull request or cancel its auto-merge. A pull + request its author opened as a draft, or converted to one, stays a draft when fork CI passes and + gets no `ready for maintainer review`: the author marks it ready for review next. +- **`CI passed`** is what merging waits for, by hand or with auto-merge. A green fork run sets it + only on a current base ([Tested base](#tested-base)) and only when the pull request does not change + CI-defining files ([Trusting a green fork run](#trusting-a-green-fork-run)). Otherwise project CI + sets it. +- **`use project ci`** moves a single pull request to project CI at any point; `use fork ci` moves + it back ([Override labels](#override-labels)). Adding the label does not start project CI: it + runs on the next push, or when a maintainer closes and reopens the pull request, and waits for a + maintainer's approval when the repository's approval setting covers the author. + + ## Configuration One file, `.github/fork-ci-gate.toml`, parsed with the standard library `tomllib`: @@ -126,7 +135,7 @@ enabled = false enabled_since = 2026-01-01T00:00:00Z # Days without any fork CI activity after which a gated pull request is closed. -close_after_days = 7 +close_after_days = 30 # Organisations exempted under AIP-120 §2.2. Keys are display names used in comments, # values are GitHub logins. Changes go through a reviewed pull request. @@ -168,7 +177,8 @@ The same applies to every other workflow with a `schedule` trigger: a fork that Actions gets all of their crons too. Their jobs carry a job-level `if: github.repository == 'apache/airflow'` guard, combined with `github.event_name != 'schedule'` when the workflow has other triggers that should still run -in a fork. A new scheduled workflow needs the same guard. The guard skips the jobs but not +in a fork. A new scheduled workflow needs the same guard, which the +`check-scheduled-workflows-skip-forks` prek hook enforces. The guard skips the jobs but not the run itself, so the frequent crons still leave an empty run in every enabled fork. Check 8 of `breeze ci audit` offers to disable these workflows in the fork. @@ -410,7 +420,8 @@ run's event, and fails when it cannot read them. Both rules use the same list as "Trusting a green fork run", so a gated pull request whose runs are approved automatically is also one whose green fork run is trusted, and one that changes CI is both -left waiting for approval and labelled `needs project ci` once its fork run is green. The gate +left waiting for approval and labelled `needs project ci` once its fork run is green, unless its +author keeps it a draft. The gate tampering check still applies to every run a maintainer approves by hand. The approval reads only API metadata and the git objects of the merge commit; it never checks out or @@ -489,7 +500,7 @@ reads too: the default branch's comes from the reconciler's checkout, any other from the contents API once per run. Those with a gated reason get the label. A base branch with neither gate file is never gated, as its gate job runs project CI (`base_without_gate`), and one with only one of the two files is reported as unknown. The -24-hour window overlaps many 15-minute cadences, so a lost scheduled run changes nothing. A +24-hour window overlaps many monitor cadences, so a lost scheduled run changes nothing. A pull request whose author later becomes a committer keeps the label until a maintainer removes it or adds `use project ci`; the reconciler does not un-gate on its own. @@ -503,8 +514,12 @@ state targetUrl description } } } } }`), and the reconciler's own state comment (`comments(first: 100)`, the earliest comment by `github-actions` with the marker, paging through the remaining comments of the rare pull request with more). The reconciler creates that comment on first contact and only edits it afterwards, so a marker posted by anyone else, or -echoed later by another workflow, is ignored. The fork state is then fetched in batches of 50 with one GraphQL -query per batch using aliases. Owner and name come from the pull request's +echoed later by another workflow, is ignored. The search index can take minutes to list a pull +request under the label pass 1 has just added, so the pull requests pass 1 labelled that the +search does not list yet are read by number with the same fields, 25 per query, and reconciled in +the same run. If that read fails, they are left for the next run. The fork state is then +fetched in batches of 50 with one GraphQL query per batch using aliases. Owner and name come +from the pull request's `headRepository { nameWithOwner }`, never a fixed `airflow`, because a fork can be renamed or created as `airflow-1` on a name collision: @@ -534,34 +549,64 @@ pull requests without a fork run for their head commit, not by the total. ### State machine -| Observed fork state | Draft | `ready for maintainer review` | Commit status | Comment | +![Fork CI reconciler states](images/diagram_fork_ci_states.svg) + +The draft state and the two review labels depend only on the state and on whether the pull +request is the author's draft: a draft whose last convert-to-draft event was not by +`github-actions[bot]`, or a pull request opened as a draft. `compute_projection` in +`fork_ci_reconcile.py` holds this table, and `test_projection_table` checks it. The table follows +one rule: a draft waits for the author, `ready for maintainer review` waits for a maintainer +review, and `needs project ci` waits for a maintainer to review the CI change and add +`use project ci`. Drafts do not count towards the open pull request limit, so marking a draft ready +needs a free slot: GitHub refuses it for an author at the limit, whoever tries, maintainers +included. The state comment of a green author's draft therefore tells the author to close or finish +one of their other open pull requests first. + +| State | Next | Draft | `ready for maintainer review` | `needs project ci` | |---|---|---|---|---| -| **setup_required**: workflow state is `disabled_fork`, `disabled_manually` or `disabled_inactivity`, or the fork has no `ci-amd.yml` | draft | remove | `error` (red), link to the fork's Actions page for `ci-amd.yml` | create or update: how to enable, with the exact `gh repo sync --branch main` and `gh workflow enable ci-amd.yml -R ` lines, and `breeze ci audit`; on transition from any other state but `closed` (the workflow stopped running after it worked): also post a **new** comment saying why it stopped | -| **awaiting_push**: enabled, no run for the head SHA | draft | remove | `pending` (yellow), link to the fork's Actions page | on transition from setup_required: post a **new** comment asking to push again (an edit would not notify) | -| **running**: run in progress | unchanged (a rebase of an approved pull request does not draft it; a failed run does) | remove | `pending` (yellow), link to the run | update state and run link | -| **failed**: conclusion is failure, cancelled or timed out | draft | remove | `failure` (red), link to the run | update with the run link | -| **green**: conclusion success and the pull request does not change CI-defining files | undraft\* | add | `success` (green), link to the run | update: ready for review, run link | -| **green_unverified**: conclusion success, but the pull request changes CI-defining files (see "Trusting a green fork run") | undraft\* | remove | `pending` (yellow), link to the run | update: a maintainer must run project CI; labelled `needs project ci` | -| setup_required or awaiting_push for `close_after_days` | unchanged | unchanged | unchanged | close with a new comment; reopening restarts the clock | -| **ungated**: the base branch has no gate, for example after retargeting to an older release branch | undraft\* | unchanged | `success` (green), link to the pull request's checks | update: project CI is in use; `fork ci required` is removed | -| **gate_modified**: the pull request changes the fork CI gate (see "Gate protection") | draft | remove | `error` (red), link to the pull request's changed files | post a **new** comment explaining why, then close; checked after gate tampering, before the fork state | -| **gate_bypassed**: a recorded project CI run of the pull request's branch bypassed the gate (see "Gate tampering") | draft | remove | `error` (red), link to the project CI run that should not have happened | post a **new** comment explaining why, then close; takes precedence over every row above | - -\* only if the last convert-to-draft event on the pull request was by `github-actions[bot]`; -a pull request the author drafted stays a draft. +| setup_required | Author enables the workflow | draft | remove | remove | +| awaiting_push | Author pushes | draft | remove | remove | +| failed | Author fixes and pushes | unchanged | remove | remove | +| running | Nobody, fork CI runs | unchanged | remove | remove | +| green | Maintainer reviews | undraft | add | remove | +| green, author's draft | Author marks it ready | unchanged | remove | remove | +| green_unverified | Maintainer reviews the CI change and adds `use project ci` | undraft | remove | add | +| green_unverified, author's draft | Author marks it ready | unchanged | remove | remove | +| released | Usual review with project CI | undraft, except the author's draft | unchanged | remove | +| ungated | Usual review with project CI | undraft, except the author's draft | unchanged | remove | +| closed | Author sets up fork CI and reopens | unchanged | remove | remove | +| gate_modified | Maintainer reopens and adds `use project ci` | unchanged | remove | remove | +| gate_bypassed | Maintainer reopens and adds `use project ci` | unchanged | remove | remove | + +The reconciler never undrafts the author's draft. It does not draft a closed pull request either, +since the draft-state job runs after the close and GitHub refuses to draft a closed pull request. + +| Observed fork state | Commit status | Comment | +|---|---|---| +| **setup_required**: workflow state is `disabled_fork`, `disabled_manually` or `disabled_inactivity`, or the fork has no `ci-amd.yml` | `error` (red), link to the fork's Actions page for `ci-amd.yml` | create or update: how to enable, with the exact `gh repo sync --branch main` and `gh workflow enable ci-amd.yml -R ` lines, and `breeze ci audit`; on transition from any other state but `closed` (the workflow stopped running after it worked): also post a **new** comment saying why it stopped | +| **awaiting_push**: enabled, no run for the head SHA | `pending` (yellow), link to the fork's Actions page | on transition from setup_required: post a **new** comment asking to push again (an edit would not notify) | +| **running**: run in progress | `pending` (yellow), link to the run | update state and run link | +| **failed**: conclusion is failure, cancelled, timed out or startup failure | `failure` (red), link to the run; for a startup failure the description says the run could not start | update with the run link and the jobs that did not pass; for a startup failure the text says no job ran and points to the fork's Actions settings | +| **green**: conclusion success and the pull request does not change CI-defining files | `success` (green), link to the run | update: ready for review, run link; on the author's draft: mark it ready for review when it is | +| **green_unverified**: conclusion success, but the pull request changes CI-defining files (see "Trusting a green fork run") | `pending` (yellow), link to the run | update: a maintainer must run project CI; on the author's draft: mark it ready for review, then a maintainer runs project CI | +| **closed**: setup_required or awaiting_push for `close_after_days` | unchanged | close with a new comment; reopening restarts the clock | +| **released**: the pull request carries `use project ci` | `success` (green), link to the pull request's checks | update: project CI is in use; `fork ci required` is removed | +| **ungated**: the base branch has no gate, for example after retargeting to an older release branch | `success` (green), link to the pull request's checks | update: project CI is in use; `fork ci required` is removed | +| **gate_modified**: the pull request changes the fork CI gate (see "Gate protection") | `error` (red), link to the pull request's changed files | post a **new** comment explaining why, then close; checked after gate tampering, before the fork state | +| **gate_bypassed**: a recorded project CI run of the pull request's branch bypassed the gate (see "Gate tampering") | `error` (red), link to the project CI run that should not have happened | post a **new** comment explaining why, then close; takes precedence over every row above | Rules that sit under the table: - **Gate tampering wins.** The `gate_bypassed` check is evaluated first for every pull request whose eligibility reason is `external` or `pilot`; when it holds, the fork state is not - consulted at all. A pull request that was already `green` and undrafted is drafted again, - loses the review label and is closed on the same run. + consulted at all. A pull request that was already `green` loses the review label and is + closed on the same run. - **A new push resets the row.** The head SHA is recorded in the state comment. A push to a green pull request is observed as `running` or `awaiting_push` on the next run, which removes the review label, so maintainers never see an unverified push in the queue. `awaiting_push` - also drafts it; `running` leaves the draft state alone, so rebasing an approved pull request - drafts it only if the new run fails. + also drafts it. `running` and `failed` leave the draft state alone, so a rebase of an approved + pull request that hits a flaky test does not draft it and cancel its auto-merge. - **Author drafts are respected.** "draft" in the table means the reconciler converts to draft if the pull request is not already one. On `green` it undrafts only when GitHub's timeline shows the last convert-to-draft actor was `github-actions[bot]`. A pull request the author @@ -609,8 +654,8 @@ decided or recorded for the old commit, and the next run decides for the new one seen, the state is `green_unverified` instead of `green`. The pull request is not drafted (a draft the reconciler made earlier is undone, as for `green`), but it gets no review label, the status is pending, and it is labelled `needs project ci` so maintainers can list these pull -requests, review the change and add `use project ci`. Without `CI passed` it cannot be merged -until then. It is not posted to Slack: +requests, review the change and add `use project ci`. A draft the author made gets no label +until the author marks it ready for review. Without `CI passed` it cannot be merged until then. It is not posted to Slack: the label already lists these pull requests, and a post for every new green commit would bury the CI breakages that channel is for. The reconciler removes `needs project ci` as soon as the state is anything else. A renamed file @@ -655,7 +700,7 @@ every event that runs the workflow files of a pull request's merge commit: `pull requests by head repository and branch. Looking at every recent run of the branch, rather than only at the head commit, is what stops a contributor from pushing a tampered commit and then a clean one to hide the tampered run. The 6-hour window spans many -15-minute cadences. Runs created before the most recent removal of `use project ci` are +monitor cadences. Runs created before the most recent removal of `use project ci` are ignored, because they ran legitimately while the label was on. So are runs created before the most recent reopen of the pull request by someone other than its author, which is how a maintainer brings back a pull request closed for tampering; reopening one whose gate is still @@ -712,8 +757,8 @@ contributor losing their collaborator association) with full runs in the last si (`TAMPER_WINDOW`) is closed. Signal 1 exists so that an external pull request that edits `ci-amd.yml` legitimately, and leaves the gate intact, is not closed for touching the file. -When both hold the reconciler records `gate_bypassed` in the state comment marker, drafts the -pull request, removes `ready for maintainer review`, posts an `error` status whose +When both hold the reconciler records `gate_bypassed` in the state comment marker, removes +`ready for maintainer review`, posts an `error` status whose `target_url` is the project CI run that should not have happened, posts a new comment and closes. The comment says which run was observed, that changes to `.github/workflows/` from a gated pull request cannot be verified in `apache/airflow` without a maintainer, and that a @@ -768,7 +813,7 @@ run checks the pull request again, with the gate configuration of its new base. retarget to a base without the gate in between could close a pull request that should be released to project CI. -The state is `gate_modified`: the pull request is drafted, loses `ready for maintainer review` +The state is `gate_modified`: the pull request loses `ready for maintainer review` and `needs project ci`, gets an `error` status linking its changed files, and the closure is reported to Slack as an attention item. Gate tampering is checked first and wins. As for gate tampering, only a reopen by someone other than the author counts: an open pull request whose @@ -795,7 +840,32 @@ All comment texts live in `dev/breeze/src/airflow_breeze/utils/fork_ci_messages. templates so wording can be reviewed in one place. Every comment ends with the AI-attribution footer required by `contributing-docs/25_maintainer_pr_triage.md`. The texts must say that CI is free for the contributor and finite for the project, must not characterise the recipient, -and must give the exact command to run. +and must give the exact command to run. They link the contributor documentation +(`contributing-docs/18_contribution_workflow.rst#ci-in-your-fork`) rather than this document, +which that section links for maintainers. + +A fork run that ends in `startup_failure` never ran a job: GitHub rejected the workflow before +running it, most often because the fork's Actions settings do not allow the actions or reusable +workflows it uses, sometimes because the workflow file is invalid. The run has no jobs, so there +are no job statuses either, and GitHub's error is only on the run page. It is handled as `failed`, +but `Fork CI / Tests (AMD)` says "Fork CI run could not start - check your fork's Actions +settings", and the state comment explains what to check instead of asking to fix failures. + +For a failed run that started, the state comment lists the jobs that did not pass, each linking +to the job in the fork, at most 10 of them, failed jobs before timed-out and cancelled ones; the +job statuses list them all. The jobs come from +`GET /repos//actions/runs//jobs?filter=latest`, reused from the job statuses when they +read them in the same run. A finished run's jobs do not change, so the marker records the run and +attempt the list belongs to (`failed_run`), and later runs do not read the jobs again while the +comment would otherwise stay the same: a failed pull request costs one jobs read per fork run, not +one per reconciler run. Job names come from the contributor's workflow, so each is shown in a code +span with backticks and line breaks replaced. When the jobs cannot be read and the state comment +already shows this head commit as `failed`, it is left as it is until the next run, rather than +dropping the list; otherwise the `failed` state is posted without the list, so the comment does +not keep describing a running or earlier run, and the next run tries the read again. A read that +failed after its retries is not repeated in +the same run, whether the job statuses or the state comment made it, so persistent server errors +cost each failed pull request one sequence of retries a run, not two. ### Commit status @@ -834,7 +904,7 @@ suites); its jobs come from `GET /repos//actions/runs//jobs?filter=lat - Only finished jobs are reported: a successful one as `success`, and a failed, timed-out, cancelled or unstartable one as `failure`. Queued and running jobs are not, so each job costs one write instead of two; `Fork CI / Tests (AMD)` shows the run as running, linking it, and with a - 15-minute cadence a per-job "running" status was behind the fork's own run page anyway. A job of + 5-minute cadence a per-job "running" status was behind the fork's own run page anyway. A job of a re-run in the fork keeps the result of the earlier attempt until it finishes. Skipped and neutral jobs, and the jobs every run has (`Fork CI gate`, `Check if the run is needed`, `Platform: AMD` and `CI passed`), are not reported. @@ -842,12 +912,12 @@ suites); its jobs come from `GET /repos//actions/runs//jobs?filter=lat (`GET /repos/apache/airflow/commits//status`), and only differences are posted, failures first. GraphQL has no mutation for commit statuses (`createCheckRun` exists, but see above for why check runs do not fit), so each one is a REST call, and they count towards GitHub's limits on - content-creating requests: a reconciler run posts at most `MAX_JOB_STATUSES_PER_RUN` (40) of + content-creating requests: a reconciler run posts at most `MAX_JOB_STATUSES_PER_RUN` (13) of them, each write charged as it is made. What does not fit is posted by the next run, and once the budget is spent no further pull request's jobs or statuses are read in that run. One fork run of the full matrix (about 130 jobs) needs more than a run's budget on its own, which the next run catches up on, so deferring alone is not a problem: the run log lists the deferred pull - requests, and only when three runs in a row deferred, about 45 minutes of the budget not + requests, and only when nine runs in a row deferred, about 45 minutes of the budget not keeping up, is a note added to the Slack message. The count is kept in the record the monitor carries between runs (`--reported-file`); without a readable record the note is added at once. Its wording is fixed, so it is posted once while deferrals persist. A pull request whose fork run @@ -867,7 +937,7 @@ suites); its jobs come from `GET /repos//actions/runs//jobs?filter=lat classified, so for example a `CI passed` that no longer holds is still revoked. Reading a fork run's jobs fails now and then on GitHub server errors (HTTP 502, a reset HTTP/2 stream) that the next run gets past, so each failure is only logged, and the pull request is listed with the - unknown items for Slack once its job statuses failed in three runs in a row. The per pull request + unknown items for Slack once its job statuses failed in nine runs in a row, about 45 minutes. The per pull request count is kept in the same record as that of deferred job statuses; without a readable record it starts again. The item's wording is fixed whatever the error, so it is posted once while the failure persists, and the run log has each run's error. @@ -882,7 +952,7 @@ records this for agents. Each run logs what is left of the token's REST and GraphQL quotas (`GET /rate_limit`, which does not count against them) and how many of its calls GitHub rate-limited; the client retries those with -backoff. Rate limiting in two runs in a row adds a note to the Slack message, with fixed wording so +backoff. Rate limiting in six runs in a row, about 30 minutes, adds a note to the Slack message, with fixed wording so it is posted once while it persists; the count is kept in the same record as that of deferred job statuses. @@ -911,14 +981,23 @@ requests a minute and 500 an hour. Assuming about 35 reported jobs for an averag monitor runs. - **Gate enabled**, assuming about half of the pull request runs come from gated pull requests (about 5-6 pushes an hour): about 200 writes an hour without a cap, 40% of the content-creating - limit and more at peaks. The cap of 40 writes a run, four runs an hour, holds job statuses to at - most 160 an hour: 32% of the content-creating limit, leaving the rest to the reconciler's other + limit and more at peaks. The cap of 13 writes a run, twelve runs an hour, holds job statuses to at + most 156 an hour: 31% of the content-creating limit, leaving the rest to the reconciler's other writes (labels, comments, the aggregate statuses) and to the other workflows that share it. Each - run posts at most 40 statuses at about one a second, well under the 80-a-minute limit. Job + run posts at most 13 statuses at about one a second, well under the 80-a-minute limit. Job statuses then lag behind at busy times, the aggregate status does not. The `CI run started` - statuses of pull requests whose CI runs in `apache/airflow` come out of the same 40 writes a run, - first and at most 20 of them (see "The `CI run started` check"), so they do not raise the total; - at about 11 pull request runs an hour they take about 3 writes a run. + statuses of pull requests whose CI runs in `apache/airflow` come out of the same 13 writes a run, + first and at most 6 of them (see "The `CI run started` check"), so they do not raise the total; + at about 11 pull request runs an hour they take about 1 write a run. + +The per-run caps and the run counts before a Slack note are derived from `MONITOR_INTERVAL` in +`fork_ci_reconcile.py`, which must match the monitor's cron (a test checks it), so changing the +cadence keeps the hourly totals above. The cadence was lowered from 15 to 5 minutes because fork CI +reaches a pull request only when the monitor runs: a fork run's start, failure or success showed up +to 15 minutes late, plus GitHub's delay of scheduled workflows. Unchanged pull requests cost no +writes, so the faster cadence adds reads, about 45 a run, well within the token's quotas, and +runner time: each run takes about two minutes on a GitHub-hosted runner, mostly installing +Breeze, so the monitor uses about 24 runner minutes an hour instead of 8. These are estimates: the share of gated runs, the jobs per push and the pushes per pull request are assumptions, not measurements. Watch the monitor's call counts in its run log after enabling @@ -946,6 +1025,8 @@ run. pending or failing one would say, and it never competes with project CI's own check. - `green_unverified` gets no `CI passed`: the fork run used CI files the pull request changed, so it is not evidence. Once a maintainer adds `use project ci`, project CI reports the check. +- A `green` run whose base is not current gets a `pending` `CI passed` asking to rebase, and one + whose base could not be compared gets none (see "Tested base"). - When the head commit leaves `green`, for example a re-run in the fork that fails, a `success` posted earlier is overwritten with `pending` so the pull request cannot merge on it. - A pull request carrying `use project ci` cannot merge on the fork result. GitHub requires both to @@ -968,7 +1049,7 @@ run. job has not succeeded, a `success` is overwritten with `pending`; once it has, the monitor posts `success` ("Project CI passed for this commit"). Adding the label does not start project CI (see "Override labels"): the pull request stays blocked until a maintainer starts it and it passes, - and then until the next monitor run posts the `success`, which can take about 15 minutes. + and then until the next monitor run posts the `success`, which can take about 5 minutes. Removing the label returns the pull request to the normal flow above: removing it updates the pull request, so discovery (open pull requests updated within the last 24 hours) labels it `fork ci required` again and the next run reports the fork result, replacing the waiting @@ -989,7 +1070,7 @@ run. - The protection is eventual, not atomic. Nothing reacts to the label when it is added (`pull_request_target` is not used, see "Goals and non-goals"), so a fork `success` posted before the label stays until the next monitor run that completes: normally within one cadence - (15 minutes), longer when GitHub delays or drops a scheduled run or a run fails, and one more + (5 minutes), longer when GitHub delays or drops a scheduled run or a run fails, and one more cycle when the label arrives after a run has taken its snapshot of the pull request. The pull request can merge on the fork result in that window. A maintainer who adds the label to a pull request whose fork CI is green waits for the `pending` before merging or enabling auto-merge, @@ -1007,7 +1088,7 @@ run. ### The `CI run started` check -Fork CI reports to the pull request only when the scheduled monitor runs, about every 15 minutes +Fork CI reports to the pull request only when the scheduled monitor runs, about every 5 minutes and later when GitHub delays scheduled workflows, and until then a gated pull request shows only skipped checks. The gate cannot say more itself: it runs on the contributor's `pull_request` event with a read-only token. Instead both CI rulesets in `.asf.yaml` also require a @@ -1033,8 +1114,8 @@ with state `success` linking the run: Folding the check into the scheduled monitor costs no runner per push, at the price of a pull request waiting for the next monitor run before it can merge, which a CI run takes longer than anyway. Each new head commit needs one write, taken first out of the per-run budget of job -statuses (see "Job statuses"), since merging waits for it; at most 20 such writes per run, so job -statuses keep at least half of the budget, and after the check became required the open pull +statuses (see "Job statuses"), since merging waits for it; at most `MAX_CI_RUN_STARTED_PER_RUN` (6) such +writes per run, so job statuses keep at least half of the budget, and after the check became required the open pull requests are caught up a batch at a time. A release branch has no monitor of its own, but `main`'s monitor covers pull requests to every base branch. @@ -1059,14 +1140,46 @@ conditions: completed, first attempt, its commit does not change `.github/workfl did project CI work by the tampering check's rules. Of its jobs only the gate and the jobs that also run in a gated run started, none failed, and at least one was cancelled. -### Far behind the base branch - -A fork run tests the pushed commit as it is, while project CI tests it merged with the base -branch. When a head commit turns green, the reconciler counts the commits it is behind its base -branch (`behind_by` of `GET /repos/apache/airflow/compare/...:`, once per -head commit, stored in the marker). From 200 commits behind, about four days of `main`, the -state comment says so and asks the contributor to rebase, since the merged result can still -fail. The count is advisory and does not change the state. +### Tested base + +A fork run tests the pushed commit as it is, while project CI's `pull_request` run tests GitHub's +merge commit of the pull request into its base branch. A fork run on an old base could otherwise go +green and merge although the merged result was never tested. So `CI passed` is only reported for a +fork run whose base is close enough to the base branch that the merge changes nothing the pull +request touches. + +When a head commit turns green, the reconciler compares it with the base branch, once per head +commit, and stores the result in the marker as `"base": {"sha", "behind", "overlap"}`: + +- `GET /repos/apache/airflow/compare/...:` gives the merge base, the commit + of the base branch the fork run tested on top of (`sha`), the commits the base branch has since + (`behind`), and the files the pull request changes. +- When `behind` is not zero, `GET /repos/apache/airflow/compare/...` gives the + files the base branch changed since. `overlap` is the paths in both lists, a renamed file + counting under both paths. + +`CI passed` is reported only when the base is current: fewer than `MAX_BEHIND_COMMITS` (200, about +four days of `main`) commits behind and no overlap. Otherwise the state stays `green` (the pull +request can be reviewed, and its status, labels and draft state are those of `green`), the state +comment lists the overlapping files or says how far behind the base is and gives the rebase +commands, and `CI passed` is posted as `pending` with a description saying to rebase. A rebase and +push starts a new fork run, which is compared again when it turns green. Every green state comment +names the base commit the run tested and how far behind it was, and the `CI passed` description +names it as well. + +The compare API lists at most 300 files. A pull request with more is listed with +`GET /repos/apache/airflow/pulls//files` instead (up to 3000); a base branch that changed 300 or +more files since the merge base cannot be checked and counts as overlapping, which a rebase +resolves. When the compare fails, `CI passed` is not reported, the failure is reported to Slack and +the next run compares again. A marker written before this check, with no `base`, is compared again +on the next run, so a pull request that already had `CI passed` loses it if its base turns out not +to be current. A retarget compares again, as it decides green trust again. + +This narrows the gap to project CI without closing it. A base change in other files can still break +the pull request, within at most 200 commits, which matches what project CI leaves open after its +run: a `pull_request` run is not repeated when the base branch moves, so the merged result at merge +time is not the one it tested either. Testing the merge commit in the fork itself was considered and +rejected; see "Known limitations". ### Unknown failures and attention items go to Slack @@ -1113,18 +1226,20 @@ Per run, with ~700 gated pull requests: ~28 GraphQL calls for pass 2, ~14 for fo ~1 for pass 1, plus one REST call per pull request in `setup_required`, a few paginated REST calls for the recent `pull_request` runs, one REST call per base branch for its workflow files, a jobs call and a compare call only per long `ci-amd.yml` run of a gated pull request, -one changed-files call per head commit that turns green, plus one mutation per actual state -change. Job statuses add a jobs call and a status call per pull request with a fork run in +one changed-files call and one or two compare calls per head commit that turns green, plus one +mutation per actual state change. Job statuses add a jobs call and a status call per pull request with a fork run in progress or a completed run not yet reported, and up to two status mutations per job, at most -`MAX_JOB_STATUSES_PER_RUN` per run; see "Write budget". Well inside -the `GITHUB_TOKEN` limits at a 15-minute cadence. The reconciler logs its call counts so the +`MAX_JOB_STATUSES_PER_RUN` per run; see "Write budget". Pull requests that pass 1 labelled and +the search does not list yet add one GraphQL call per 25. Well inside +the `GITHUB_TOKEN` limits at a 5-minute cadence. The reconciler logs its call counts so the budget stays observable. ## Monitor workflow `.github/workflows/fork-ci-monitor.yml`: -- `on: schedule: cron '*/15 * * * *'` and `workflow_dispatch` with a `dry-run` input. +- `on: schedule: cron '*/5 * * * *'` (`MONITOR_INTERVAL`) and `workflow_dispatch` with a `dry-run` + input. - Two jobs, each with its own `permissions` block and a justification comment; the workflow grants nothing by default. - The `reconcile` job: `actions: write, contents: read, pull-requests: write, statuses: write`. @@ -1296,10 +1411,11 @@ as a failed check rather than a crash. ## Known limitations -- Drafting, the setup comment and the commit status arrive within one cadence of the event - they reflect, not instantly. The cadence can be lowered, but the per-run write budgets are - sized for four runs an hour against the `GITHUB_TOKEN` limits shared by every workflow of the - repository, so they must be scaled down with it. +- Drafting, the setup comment and the commit status arrive within one cadence (5 minutes) of the + event they reflect, plus GitHub's delay of scheduled workflows, not instantly. The fork runs in + another repository, so no event in `apache/airflow` reports its start or end; a `workflow_run` + trigger on project CI's gate run would only cover a push, at the cost of a monitor run per + pull request push. - The workflows API lists workflows from the fork's default branch. A fork whose `main` predates `ci-amd.yml` reports it missing; the setup comment covers this with `gh repo sync`. @@ -1318,6 +1434,17 @@ as a failed check rather than a crash. - An external pull request that legitimately changes workflow files, was released with `use project ci` and then had that label removed is closed on the next run, because both signals hold. Keeping `use project ci` on such a pull request is the intended handling. +- Fork CI tests the head commit, not the pull request merged into its base branch; "Tested base" + only bounds the difference. Testing the merged result in the fork was rejected because: + every job of the fork run checks out the pushed commit (about 60 `actions/checkout` steps across + `ci-amd.yml` and its reusable workflows, plus the image builds and caches keyed by the commit), + and each would have to check out one merge commit fixed for the whole run; the first push of a + branch happens before its pull request exists, so there is no `refs/pull//merge` to test yet; + GitHub computes that ref asynchronously after each push and replaces it whenever the base branch + moves, so a run would have to wait for it and could find its commit gone mid-run; and the + reconciler would have to trust the run's own report of which merge it tested. Merging the base + branch into the head in every job instead needs the shared history of both, which shallow + checkouts do not have. ## Testing @@ -1334,7 +1461,8 @@ as a failed check rather than a crash. close), a fork with a name other than `airflow`, marker round-trips, status desired-vs-current comparison, the unknown-failure collection, green trust (a change to CI-defining files keeps the status pending without drafting the pull request, decided once - per commit) and the gate + per commit), the tested base (`CI passed` only on a current base, the marker and comment naming + the base, compared again for a new commit, a retarget or a marker without a base) and the gate tampering check: a tampered run on an earlier commit is caught after a clean push, each signal alone does not close, short runs and gate-only jobs do not count, an in-progress non-gate job, a removed, duplicated or long-running gate job and an injected workflow with a diff --git a/dev/breeze/doc/ci/images/diagram_fork_ci_architecture.md5sum b/dev/breeze/doc/ci/images/diagram_fork_ci_architecture.md5sum new file mode 100644 index 0000000000000..49c405e73eff9 --- /dev/null +++ b/dev/breeze/doc/ci/images/diagram_fork_ci_architecture.md5sum @@ -0,0 +1 @@ +0680ff8001f6f267b5b900af33ac16e3 diff --git a/dev/breeze/doc/ci/images/diagram_fork_ci_architecture.py b/dev/breeze/doc/ci/images/diagram_fork_ci_architecture.py new file mode 100644 index 0000000000000..6efe475797d6c --- /dev/null +++ b/dev/breeze/doc/ci/images/diagram_fork_ci_architecture.py @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "rich>=13.6.0", +# "graphviz>=0.20.1", +# ] +# /// +"""Where fork CI runs: the contributor fork and apache/airflow. Used in dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md and on the ADIP-12 wiki page.""" + +from __future__ import annotations + +from pathlib import Path + +import graphviz +from rich.console import Console + +MY_DIR = Path(__file__).parent +MY_FILENAME = Path(__file__).with_suffix("").name + +console = Console(width=400, color_system="standard") + +DOT = r""" +digraph arch { + graph [rankdir=LR, fontname="Helvetica", fontsize=11, nodesep=0.4, ranksep=0.7, compound=true, bgcolor="white"]; + node [shape=box, style="rounded,filled", fillcolor="#f2f4f7", color="#5b6b80", fontname="Helvetica", fontsize=11]; + edge [fontname="Helvetica", fontsize=9, color="#5b6b80", fontcolor="#33404d"]; + + subgraph cluster_fork { + label="contributor's fork (GitHub-hosted runners, contributor's token)"; style="rounded"; color="#3d8b57"; fontname="Helvetica"; + push [label="git push feature-branch"]; + ci [label="Tests (AMD) workflow\npush trigger"]; + gatef [label="gate job:\nfork -> run"]; + bi [label="build-info in fork mode\nchanged files vs upstream\n(compare API)"]; + tests [label="tests and checks\non public runners"]; + push -> ci -> gatef -> bi -> tests; + } + + subgraph cluster_apache { + label="apache/airflow"; style="rounded"; color="#5b6b80"; fontname="Helvetica"; + pr [label="pull_request event"]; + gatea [label="gate job:\neligibility -> skip\n(build-info and all later jobs skipped)"]; + mon [label="fork-ci-monitor.yml\n(scheduled, never runs PR code)"]; + rec [label="breeze ci fork-ci-reconcile"]; + out [label="per gated PR: draft state, labels,\nstate comment, commit statuses\n(Fork CI / Tests (AMD), CI passed),\nclose after 30 days or on gate tampering"]; + slack [label="unknown failures -> Slack", fillcolor="#eceff3"]; + pr -> gatea; + mon -> rec -> out; + rec -> slack [style=dashed]; + } + + tests -> rec [label="GraphQL: check suites\nREST: runs, jobs, workflow state\n(read by the monitor)", dir=back, style=dashed]; +} +""" + + +def generate_diagram() -> None: + image_file = MY_DIR / f"{MY_FILENAME}.svg" + console.print(f"[bright_blue]Generating {image_file}") + graphviz.Source(DOT).render(outfile=image_file, format="svg", cleanup=True) + console.print(f"[green]Generated {image_file}") + + +if __name__ == "__main__": + generate_diagram() diff --git a/dev/breeze/doc/ci/images/diagram_fork_ci_architecture.svg b/dev/breeze/doc/ci/images/diagram_fork_ci_architecture.svg new file mode 100644 index 0000000000000..2ce33697709c2 --- /dev/null +++ b/dev/breeze/doc/ci/images/diagram_fork_ci_architecture.svg @@ -0,0 +1,157 @@ + + + + + + +arch + + +cluster_fork + +contributor's fork (GitHub-hosted runners, contributor's token) + + +cluster_apache + +apache/airflow + + + +push + +git push feature-branch + + + +ci + +Tests (AMD) workflow +push trigger + + + +push->ci + + + + + +gatef + +gate job: +fork -> run + + + +ci->gatef + + + + + +bi + +build-info in fork mode +changed files vs upstream +(compare API) + + + +gatef->bi + + + + + +tests + +tests and checks +on public runners + + + +bi->tests + + + + + +rec + +breeze ci fork-ci-reconcile + + + +tests->rec + + +GraphQL: check suites +REST: runs, jobs, workflow state +(read by the monitor) + + + +pr + +pull_request event + + + +gatea + +gate job: +eligibility -> skip +(build-info and all later jobs skipped) + + + +pr->gatea + + + + + +mon + +fork-ci-monitor.yml +(scheduled, never runs PR code) + + + +mon->rec + + + + + +out + +per gated PR: draft state, labels, +state comment, commit statuses +(Fork CI / Tests (AMD), CI passed), +close after 30 days or on gate tampering + + + +rec->out + + + + + +slack + +unknown failures -> Slack + + + +rec->slack + + + + + diff --git a/dev/breeze/doc/ci/images/diagram_fork_ci_lifecycle.md5sum b/dev/breeze/doc/ci/images/diagram_fork_ci_lifecycle.md5sum new file mode 100644 index 0000000000000..d0118a174588e --- /dev/null +++ b/dev/breeze/doc/ci/images/diagram_fork_ci_lifecycle.md5sum @@ -0,0 +1 @@ +27566b64cbac2d256641b1dbf873fdc9 diff --git a/dev/breeze/doc/ci/images/diagram_fork_ci_lifecycle.py b/dev/breeze/doc/ci/images/diagram_fork_ci_lifecycle.py new file mode 100644 index 0000000000000..fcb7053e253ed --- /dev/null +++ b/dev/breeze/doc/ci/images/diagram_fork_ci_lifecycle.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "rich>=13.6.0", +# "graphviz>=0.20.1", +# ] +# /// +"""What a gated pull request shows, and who acts next, from opening to merge or close. Used in dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md and on the ADIP-12 wiki page.""" + +from __future__ import annotations + +from pathlib import Path + +import graphviz +from rich.console import Console + +MY_DIR = Path(__file__).parent +MY_FILENAME = Path(__file__).with_suffix("").name + +console = Console(width=400, color_system="standard") + +DOT = r""" +digraph lifecycle { + graph [rankdir=TB, fontname="Helvetica", fontsize=11, nodesep=0.5, ranksep=0.55, bgcolor="white"]; + node [shape=box, style="rounded,filled", fillcolor="#f2f4f7", color="#5b6b80", fontname="Helvetica", fontsize=11, margin="0.15,0.08"]; + edge [fontname="Helvetica", fontsize=9, color="#5b6b80", fontcolor="#33404d"]; + + start [shape=circle, label="", width=0.22, fillcolor="#33404d", color="#33404d"]; + done [shape=doublecircle, label="", width=0.18, fillcolor="#33404d", color="#33404d"]; + + waiting [label="Waiting for the author\n(draft: fork CI not set up,\nor no run for the head commit)"]; + running [label="Fork CI running"]; + failed [label="Fork CI failed\n(out of the review queue)", fillcolor="#fde8e8", color="#b54848"]; + ready [label="Green\nready for maintainer review\nCI passed", fillcolor="#e6f4ea", color="#3d8b57"]; + authordraft [label="Green, author's draft\nstays a draft, no review label\nuntil the author marks it ready", fillcolor="#e6f4ea", color="#3d8b57"]; + rebase [label="Green, base too old\nready for maintainer review\nCI passed pending until a rebase", fillcolor="#fff6db", color="#b38a1d"]; + needs [label="Green, CI files changed\nneeds project ci", fillcolor="#fff6db", color="#b38a1d"]; + project [label="Project CI\n(use project ci)\nruns on the next push\nor a close and reopen"]; + approved[label="Approved"]; + merged [label="Merged", fillcolor="#e6f4ea", color="#3d8b57"]; + closed [label="Closed after 30 days\nwithout fork CI activity", fillcolor="#eceff3", color="#8a96a3"]; + + start -> waiting [label="external contributor\nopens a pull request"]; + waiting -> running [label="author enables the fork\nworkflow and pushes"]; + running -> waiting [label="the fork workflow\nis disabled", style=dotted]; + running -> failed [label="a job fails, times out,\nis cancelled, or the\nrun cannot start"]; + running -> ready [label="passes on a\ncurrent base"]; + running -> authordraft [label="passes, the author\nmade it a draft"]; + authordraft -> ready [label="author marks it\nready for review"]; + running -> rebase [label="passes, base 200+ commits\nbehind or changed\nthe same files"]; + running -> needs [label="passes, changes\nCI-defining files"]; + failed -> running [label="author pushes a fix"]; + rebase -> running [label="author rebases\nand pushes"]; + ready -> running [label="author pushes", style=dashed]; + needs -> running [label="author pushes", style=dashed]; + approved -> running [label="author pushes\nor rebases", style=dashed]; + ready -> approved [label="maintainer approves"]; + rebase -> approved [label="maintainer approves;\nmerge waits for the rebase", style=dashed]; + ready -> project [label="maintainer adds\nuse project ci", style=dashed]; + needs -> project [label="maintainer reviews the\nCI change, adds use project ci"]; + project -> approved [label="project CI passes,\nmaintainer approves"]; + approved -> merged [label="merged, or auto-merged\nonce CI passed is green"]; + waiting -> closed [label="30 days without\nfork CI activity"]; + closed -> waiting [label="author reopens", style=dashed]; + merged -> done; +} +""" + + +def generate_diagram() -> None: + image_file = MY_DIR / f"{MY_FILENAME}.svg" + console.print(f"[bright_blue]Generating {image_file}") + graphviz.Source(DOT).render(outfile=image_file, format="svg", cleanup=True) + console.print(f"[green]Generated {image_file}") + + +if __name__ == "__main__": + generate_diagram() diff --git a/dev/breeze/doc/ci/images/diagram_fork_ci_lifecycle.svg b/dev/breeze/doc/ci/images/diagram_fork_ci_lifecycle.svg new file mode 100644 index 0000000000000..7b5cc6e9393df --- /dev/null +++ b/dev/breeze/doc/ci/images/diagram_fork_ci_lifecycle.svg @@ -0,0 +1,283 @@ + + + + + + +lifecycle + + + +start + + + + +waiting + +Waiting for the author +(draft: fork CI not set up, +or no run for the head commit) + + + +start->waiting + + +external contributor +opens a pull request + + + +done + + + + + +running + +Fork CI running + + + +waiting->running + + +author enables the fork +workflow and pushes + + + +closed + +Closed after 30 days +without fork CI activity + + + +waiting->closed + + +30 days without +fork CI activity + + + +running->waiting + + +the fork workflow +is disabled + + + +failed + +Fork CI failed +(out of the review queue) + + + +running->failed + + +a job fails, times out, +is cancelled, or the +run cannot start + + + +ready + +Green +ready for maintainer review +CI passed + + + +running->ready + + +passes on a +current base + + + +authordraft + +Green, author's draft +stays a draft, no review label +until the author marks it ready + + + +running->authordraft + + +passes, the author +made it a draft + + + +rebase + +Green, base too old +ready for maintainer review +CI passed pending until a rebase + + + +running->rebase + + +passes, base 200+ commits +behind or changed +the same files + + + +needs + +Green, CI files changed +needs project ci + + + +running->needs + + +passes, changes +CI-defining files + + + +failed->running + + +author pushes a fix + + + +ready->running + + +author pushes + + + +project + +Project CI +(use project ci) +runs on the next push +or a close and reopen + + + +ready->project + + +maintainer adds +use project ci + + + +approved + +Approved + + + +ready->approved + + +maintainer approves + + + +authordraft->ready + + +author marks it +ready for review + + + +rebase->running + + +author rebases +and pushes + + + +rebase->approved + + +maintainer approves; +merge waits for the rebase + + + +needs->running + + +author pushes + + + +needs->project + + +maintainer reviews the +CI change, adds use project ci + + + +project->approved + + +project CI passes, +maintainer approves + + + +approved->running + + +author pushes +or rebases + + + +merged + +Merged + + + +approved->merged + + +merged, or auto-merged +once CI passed is green + + + +merged->done + + + + + +closed->waiting + + +author reopens + + + diff --git a/dev/breeze/doc/ci/images/diagram_fork_ci_states.md5sum b/dev/breeze/doc/ci/images/diagram_fork_ci_states.md5sum new file mode 100644 index 0000000000000..7cd7ef394cfc2 --- /dev/null +++ b/dev/breeze/doc/ci/images/diagram_fork_ci_states.md5sum @@ -0,0 +1 @@ +e83d975c093d3a3c23d598f4be0077c3 diff --git a/dev/breeze/doc/ci/images/diagram_fork_ci_states.py b/dev/breeze/doc/ci/images/diagram_fork_ci_states.py new file mode 100644 index 0000000000000..47b3a9c66118a --- /dev/null +++ b/dev/breeze/doc/ci/images/diagram_fork_ci_states.py @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "rich>=13.6.0", +# "graphviz>=0.20.1", +# ] +# /// +"""The fork CI reconciler's states for a gated pull request. Used in dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md and on the ADIP-12 wiki page.""" + +from __future__ import annotations + +from pathlib import Path + +import graphviz +from rich.console import Console + +MY_DIR = Path(__file__).parent +MY_FILENAME = Path(__file__).with_suffix("").name + +console = Console(width=400, color_system="standard") + +DOT = r""" +digraph states { + graph [rankdir=TB, fontname="Helvetica", fontsize=11, nodesep=0.45, ranksep=0.5, compound=true, bgcolor="white"]; + node [shape=box, style="rounded,filled", fillcolor="#f2f4f7", color="#5b6b80", fontname="Helvetica", fontsize=11]; + edge [fontname="Helvetica", fontsize=9, color="#5b6b80", fontcolor="#33404d"]; + + start [shape=circle, label="", width=0.22, fillcolor="#33404d", color="#33404d"]; + + subgraph cluster_gated { + label="gated (label: fork ci required)"; fontname="Helvetica"; fontsize=12; style="rounded"; color="#8a96a3"; + setup_required [label="setup_required\nworkflow not enabled in the fork\n(draft)"]; + awaiting_push [label="awaiting_push\nworkflow enabled, no run\nfor the head commit (draft)"]; + running [label="running\nfork run in progress"]; + failed [label="failed\nrun failed, cancelled, timed out\nor could not start", fillcolor="#fde8e8", color="#b54848"]; + subgraph cluster_green { + label="green: no CI-defining change"; fontname="Helvetica"; fontsize=10; style="rounded,dashed"; color="#3d8b57"; + green [label="ready for maintainer review\n(not a draft, or the bot's draft)", fillcolor="#e6f4ea", color="#3d8b57"]; + green_author_draft [label="the author's draft\nauthor marks it ready", fillcolor="#e6f4ea", color="#3d8b57"]; + } + subgraph cluster_green_unverified { + label="green_unverified: CI-defining change"; fontname="Helvetica"; fontsize=10; style="rounded,dashed"; color="#b38a1d"; + green_unverified [label="needs project ci\n(not a draft, or the bot's draft)", fillcolor="#fff6db", color="#b38a1d"]; + unverified_author_draft [label="the author's draft\nauthor marks it ready", fillcolor="#fff6db", color="#b38a1d"]; + } + } + + closed [label="closed\nsetup_required or awaiting_push\nfor 30 days", fillcolor="#eceff3", color="#8a96a3"]; + released [label="released\nmaintainer added use project ci"]; + ungated [label="ungated\nbase branch without fork CI"]; + gate_modified [label="gate_modified\nPR changes the gate (closed)", fillcolor="#fde8e8", color="#b54848"]; + gate_bypassed [label="gate_bypassed\nproject CI ran without\nuse project ci (closed)", fillcolor="#fde8e8", color="#b54848"]; + + start -> setup_required [label="discovery adds\nfork ci required", lhead=cluster_gated]; + setup_required -> awaiting_push [label="author enables\nthe workflow"]; + awaiting_push -> running [label="author pushes"]; + running -> failed [label="run fails"]; + running -> green [label="run passes", lhead=cluster_green]; + running -> green_unverified [label="run passes,\nCI files changed", lhead=cluster_green_unverified]; + green -> green_author_draft [label="author converts\nto draft"]; + green_author_draft -> green [label="author marks\nit ready", style=dashed]; + green_unverified -> unverified_author_draft [label="author converts\nto draft"]; + unverified_author_draft -> green_unverified [label="author marks\nit ready", style=dashed]; + failed -> running [label="author pushes", style=dashed]; + green -> running [label="author pushes", style=dashed, ltail=cluster_green]; + green_unverified -> running [label="author pushes", style=dashed, ltail=cluster_green_unverified]; + running -> setup_required [label="workflow disabled", style=dotted]; + awaiting_push -> setup_required [label="workflow disabled", style=dotted]; + + awaiting_push -> closed [label="30 days", ltail=cluster_gated]; + closed -> awaiting_push [label="reopened", style=dashed, lhead=cluster_gated]; + green_unverified -> released [label="maintainer adds\nuse project ci", ltail=cluster_gated]; + green -> ungated [label="base without fork CI", ltail=cluster_gated]; + failed -> gate_modified [label="PR changes the gate", ltail=cluster_gated]; + running -> gate_bypassed [label="gate tampering", ltail=cluster_gated]; +} +""" + + +def generate_diagram() -> None: + image_file = MY_DIR / f"{MY_FILENAME}.svg" + console.print(f"[bright_blue]Generating {image_file}") + graphviz.Source(DOT).render(outfile=image_file, format="svg", cleanup=True) + console.print(f"[green]Generated {image_file}") + + +if __name__ == "__main__": + generate_diagram() diff --git a/dev/breeze/doc/ci/images/diagram_fork_ci_states.svg b/dev/breeze/doc/ci/images/diagram_fork_ci_states.svg new file mode 100644 index 0000000000000..2489a3881ce03 --- /dev/null +++ b/dev/breeze/doc/ci/images/diagram_fork_ci_states.svg @@ -0,0 +1,284 @@ + + + + + + +states + + +cluster_gated + +gated (label: fork ci required) + + +cluster_green + +green: no CI-defining change + + +cluster_green_unverified + +green_unverified: CI-defining change + + + +start + + + + +setup_required + +setup_required +workflow not enabled in the fork +(draft) + + + +start->setup_required + + +discovery adds +fork ci required + + + +awaiting_push + +awaiting_push +workflow enabled, no run +for the head commit (draft) + + + +setup_required->awaiting_push + + +author enables +the workflow + + + +awaiting_push->setup_required + + +workflow disabled + + + +running + +running +fork run in progress + + + +awaiting_push->running + + +author pushes + + + +closed + +closed +setup_required or awaiting_push +for 30 days + + + +awaiting_push->closed + + +30 days + + + +running->setup_required + + +workflow disabled + + + +failed + +failed +run failed, cancelled, timed out +or could not start + + + +running->failed + + +run fails + + + +green + +ready for maintainer review +(not a draft, or the bot's draft) + + + +running->green + + +run passes + + + +green_unverified + +needs project ci +(not a draft, or the bot's draft) + + + +running->green_unverified + + +run passes, +CI files changed + + + +gate_bypassed + +gate_bypassed +project CI ran without +use project ci (closed) + + + +running->gate_bypassed + + +gate tampering + + + +failed->running + + +author pushes + + + +gate_modified + +gate_modified +PR changes the gate (closed) + + + +failed->gate_modified + + +PR changes the gate + + + +green->running + + +author pushes + + + +green_author_draft + +the author's draft +author marks it ready + + + +green->green_author_draft + + +author converts +to draft + + + +ungated + +ungated +base branch without fork CI + + + +green->ungated + + +base without fork CI + + + +green_author_draft->green + + +author marks +it ready + + + +green_unverified->running + + +author pushes + + + +unverified_author_draft + +the author's draft +author marks it ready + + + +green_unverified->unverified_author_draft + + +author converts +to draft + + + +released + +released +maintainer added use project ci + + + +green_unverified->released + + +maintainer adds +use project ci + + + +unverified_author_draft->green_unverified + + +author marks +it ready + + + +closed->awaiting_push + + +reopened + + + diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_gate.py b/dev/breeze/src/airflow_breeze/utils/fork_ci_gate.py index 13d59238efb74..aa5f04039db3d 100644 --- a/dev/breeze/src/airflow_breeze/utils/fork_ci_gate.py +++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_gate.py @@ -52,7 +52,7 @@ LABEL_READY_FOR_REVIEW = "ready for maintainer review" LABEL_NEEDS_PROJECT_CI = "needs project ci" -DEFAULT_CLOSE_AFTER_DAYS = 7 +DEFAULT_CLOSE_AFTER_DAYS = 30 DEFAULT_ENABLED_SINCE = datetime(2026, 1, 1, tzinfo=UTC) # The release-prep / providers branches whose pushes run project CI in apache/airflow: the @@ -187,9 +187,12 @@ def parse_config(text: str) -> GateConfig: raise ValueError(f"allowlist entry {org!r} must be a list of GitHub logins, got {logins!r}") allowlist[org] = tuple(logins) + pilot_table = data.get("pilot", {}) + if not isinstance(pilot_table, dict): + raise ValueError(f"pilot must be a table of GitHub logins, got {pilot_table!r}") pilot = { login.casefold(): _require_aware_datetime(joined_at, f"pilot entry {login!r}") - for login, joined_at in data.get("pilot", {}).items() + for login, joined_at in pilot_table.items() } job_statuses = data.get("job_statuses", True) if not isinstance(job_statuses, bool): diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_github.py b/dev/breeze/src/airflow_breeze/utils/fork_ci_github.py index bc763c7e546e7..9d84bc89f1cd5 100644 --- a/dev/breeze/src/airflow_breeze/utils/fork_ci_github.py +++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_github.py @@ -45,6 +45,11 @@ # Network failures before the request reached GitHub, so repeating it is safe for any call. CONNECTION_ERROR = re.compile(r"TLS handshake timeout|dial tcp") SERVER_ERROR_WAIT_SECONDS = 10 +# GitHub answers HTTP 502 on every attempt when it lists 100 jobs per page of some runs with a few +# hundred jobs, re-run more than once, while it lists smaller pages of the same run. A smaller size is +# tried once the previous one keeps failing with server errors. +JOB_PAGE_SIZES = (100, 30, 15, 5) +JOB_PAGE_SIZE_SERVER_ERROR_ATTEMPTS = 2 # A hung `gh` would otherwise hold the job until its timeout-minutes. GH_TIMEOUT_SECONDS = 120 @@ -69,6 +74,7 @@ def _gh( *, allow_error_output: bool = False, retry_server_errors: bool = True, + max_server_error_attempts: int = MAX_ATTEMPTS, ) -> Any: """ :param allow_error_output: return the JSON output of a failed call when it has any. ``gh api @@ -77,6 +83,8 @@ def _gh( :param retry_server_errors: repeat a call that failed with a server error. Off for a call that may have taken effect anyway and must not be repeated, such as creating a comment. A call that failed to connect is repeated either way. + :param max_server_error_attempts: how many times the call is made before giving up on server + errors; rate limits and connection errors are retried up to ``MAX_ATTEMPTS`` times either way. """ payload = json.dumps(body) if body is not None else None rate_limited = False @@ -98,12 +106,10 @@ def _gh( rate_limited = any(marker in result.stderr.lower() for marker in RATE_LIMIT_MARKERS) self.rate_limited += rate_limited connection_error = bool(CONNECTION_ERROR.search(result.stderr)) - if ( - rate_limited - or connection_error - or (retry_server_errors and SERVER_ERROR.search(result.stderr)) - ): - if attempt == MAX_ATTEMPTS - 1: + server_error = not rate_limited and not connection_error and SERVER_ERROR.search(result.stderr) + if rate_limited or connection_error or (retry_server_errors and server_error): + last_attempt = max_server_error_attempts if server_error else MAX_ATTEMPTS + if attempt >= last_attempt - 1: break reason = ( "rate limit" @@ -123,9 +129,9 @@ def _gh( pass raise GitHubError(f"gh {' '.join(args)} failed: {result.stderr.strip()}") if rate_limited: - raise GitHubError(f"gh {' '.join(args)} still rate limited after {MAX_ATTEMPTS} attempts") + raise GitHubError(f"gh {' '.join(args)} still rate limited after {attempt + 1} attempts") raise GitHubError( - f"gh {' '.join(args)} still failing after {MAX_ATTEMPTS} attempts: {result.stderr.strip()}" + f"gh {' '.join(args)} still failing after {attempt + 1} attempts: {result.stderr.strip()}" ) def graphql( @@ -149,20 +155,53 @@ def graphql( console_print(f"[warning]Partial GraphQL result: {response['errors']}[/]") return response["data"] - def rest_get(self, path: str, *, paginate: bool = False, items_key: str | None = None) -> Any: + def rest_get( + self, + path: str, + *, + paginate: bool = False, + items_key: str | None = None, + max_server_error_attempts: int = MAX_ATTEMPTS, + ) -> Any: """ GET a REST path. :param paginate: follow every page and return the concatenated items. :param items_key: for endpoints whose pages are objects, the key holding the items (``workflow_runs``, ``jobs``, ``files``); pages are plain lists otherwise. + :param max_server_error_attempts: how many times the call is made before giving up on server + errors. """ args = ["api", path, *(["--paginate", "--slurp"] if paginate else [])] - result = self._gh(args, None, "rest") + result = self._gh(args, None, "rest", max_server_error_attempts=max_server_error_attempts) if paginate: return [item for page in result for item in (page[items_key] if items_key else page)] return result + def get_run_jobs(self, repo: str, run_id: int) -> list[dict[str, Any]]: + """The jobs of a workflow run's latest attempt, in smaller pages while GitHub fails to list them.""" + + def get_jobs(page_size: int, max_server_error_attempts: int) -> list[dict[str, Any]]: + return self.rest_get( + f"repos/{repo}/actions/runs/{run_id}/jobs?filter=latest&per_page={page_size}", + paginate=True, + items_key="jobs", + max_server_error_attempts=max_server_error_attempts, + ) + + *larger, smallest = JOB_PAGE_SIZES + for page_size in larger: + try: + return get_jobs(page_size, JOB_PAGE_SIZE_SERVER_ERROR_ATTEMPTS) + except GitHubError as e: + if not SERVER_ERROR.search(str(e)): + raise + console_print( + f"[warning]Listing the jobs of {repo} run {run_id} {page_size} per page keeps " + "failing, trying smaller pages[/]" + ) + return get_jobs(smallest, MAX_ATTEMPTS) + def rest_get_pages(self, path: str) -> list[Any]: """GET every page of a paginated REST path, keeping page-level fields such as ``total_count``.""" return self._gh(["api", path, "--paginate", "--slurp"], None, "rest") diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py b/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py index 521b4714ef86d..9d852c7a9fca4 100644 --- a/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py +++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py @@ -30,8 +30,10 @@ import json import re +from collections.abc import Sequence from dataclasses import dataclass from datetime import datetime +from typing import NamedTuple STATUS_CONTEXT = "Fork CI / Tests (AMD)" # Each job of the fork run is reported under this prefix. Job names come from the contributor's @@ -43,7 +45,6 @@ # CI's gate job reports it; for a gated pull request, which never runs project CI, the reconciler # reports it from the fork run. CI_PASSED_CONTEXT = "CI passed" -CI_PASSED_DESCRIPTION = "Fork CI passed for this commit" CI_PASSED_REVOKED_DESCRIPTION = "Fork CI result no longer trusted for this commit" # Replaces a fork `success` on a pull request carrying `use project ci` until project CI has passed. CI_PASSED_WAITING_PROJECT_CI_DESCRIPTION = "Waiting for project CI (use project ci)" @@ -54,16 +55,22 @@ CI_RUN_STARTED_CONTEXT = "CI run started" CI_RUN_STARTED_DESCRIPTION = "CI runs in the contributor's fork - details open the run" CI_RUN_STARTED_PROJECT_DESCRIPTION = "Project CI runs in apache/airflow - details open the run" -# A fork run tests the head commit as it is, not merged with the base branch. Further behind than -# this, it says little about the result after merge. -BEHIND_WARNING_COMMITS = 200 +# A fork run tests the head commit as it is, not merged with the base branch as project CI does. From +# this many commits behind, `CI passed` is withheld until the branch is rebased, as it is when the +# base branch changed any file the pull request changes. +MAX_BEHIND_COMMITS = 200 +# Overlapping files named in the state comment; the marker keeps them all. +OVERLAP_LISTED = 10 MARKER_PREFIX = "") -DESIGN_URL = ( - "https://github.com/apache/airflow/blob/main/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md" -) +# Texts on pull requests are for their authors, so they link the contributor documentation; it links +# the design document for maintainers. CONTRIBUTING_URL = ( "https://github.com/apache/airflow/blob/main/contributing-docs/18_contribution_workflow.rst" + "#ci-in-your-fork" +) +OPEN_PR_LIMIT_URL = ( + "https://github.com/apache/airflow/blob/main/contributing-docs/32_open_pull_request_limit.rst" ) WHY_AUTOMATED_URL = ( "https://github.com/apache/airflow/blob/main/contributing-docs/25_maintainer_pr_triage.md" @@ -85,15 +92,38 @@ ) +@dataclass(frozen=True) +class BaseCheck: + """ + The base a green fork run tested, compared with the base branch when the run turned green. + + :param sha: the merge base of the head commit and the base branch, the base the run tested. + :param behind: commits on the base branch since ``sha``. + :param overlap: paths that both the pull request and the base branch since ``sha`` change; + None when GitHub listed too many changed files on either side to tell. + """ + + sha: str + behind: int + overlap: tuple[str, ...] | None = () + + @property + def is_current(self) -> bool: + """Whether the run tested close enough to the merged result for `CI passed`.""" + return self.behind < MAX_BEHIND_COMMITS and self.overlap == () + + @dataclass(frozen=True) class Marker: state: str sha: str since: datetime - # Commits the head commit was behind its base branch when its fork CI turned green. - behind: int | None = None + # Set when the head commit's fork CI turned green and its base could be compared. + base: BaseCheck | None = None # The completed fork run (``::``) whose jobs are all reported. jobs: str | None = None + # The fork run (``:``) whose failed jobs the comment lists. + failed_run: str | None = None # Set only by a reconciler that ran the gate protection check for this head commit, so a # marker written before that check existed, or while it did not apply, never counts as one. gate_checked: bool = False @@ -101,10 +131,16 @@ class Marker: def render_marker(marker: Marker) -> str: data: dict[str, object] = {"state": marker.state, "sha": marker.sha, "since": marker.since.isoformat()} - if marker.behind is not None: - data["behind"] = marker.behind + if marker.base is not None: + data["base"] = { + "sha": marker.base.sha, + "behind": marker.base.behind, + "overlap": None if marker.base.overlap is None else list(marker.base.overlap), + } if marker.jobs is not None: data["jobs"] = marker.jobs + if marker.failed_run is not None: + data["failed_run"] = marker.failed_run if marker.gate_checked: data["gate_checked"] = True return f"{MARKER_PREFIX}{json.dumps(data, sort_keys=True)} -->" @@ -117,12 +153,21 @@ def parse_marker(body: str) -> Marker | None: return None try: data = json.loads(match.group("payload")) + # A marker written before the base was checked carries only "behind"; its base is checked again. + base = data.get("base") return Marker( state=data["state"], sha=data["sha"], since=datetime.fromisoformat(data["since"]), - behind=data.get("behind"), + base=None + if base is None + else BaseCheck( + sha=base["sha"], + behind=base["behind"], + overlap=None if base["overlap"] is None else tuple(base["overlap"]), + ), jobs=data.get("jobs"), + failed_run=data.get("failed_run"), gate_checked=data.get("gate_checked") is True, ) except (json.JSONDecodeError, KeyError, TypeError, ValueError) as e: @@ -141,27 +186,35 @@ def _setup_instructions(fork: str) -> str: f" ```bash\n gh repo sync {fork} --branch main\n ```\n\n" "2. Enable the workflow in your fork:\n\n" f" ```bash\n gh workflow enable ci-amd.yml -R {fork}\n ```\n\n" + f" *(Or in your browser: open `https://github.com/{fork}/actions` and click 'Enable workflows')*\n\n" "3. Push your branch again. Every push then runs CI in your fork.\n\n" "`breeze ci audit` checks your whole local and fork setup and prints the command that fixes " - f"anything missing. See [the contribution workflow]({CONTRIBUTING_URL})." + f"anything missing. See [CI in your fork]({CONTRIBUTING_URL})." ) +_CHANGES_CI_FILES = ( + "this pull request changes files that decide which CI jobs run and where (`.github/`, the breeze " + "sources, the breeze installation or `scripts/ci/prek/`), and the fork run used those changed files." +) + _STATE_TEXTS = { "setup_required": "**Fork CI: setup required.** CI for this pull request has not run in your fork yet, " "because the `Tests (AMD)` workflow is not enabled there.", - "awaiting_push": "**Fork CI: waiting for a push.** The workflow is enabled in your fork, but it has not " - "run for the latest commit of this pull request. Push the branch again to start it.", + "awaiting_push": ( + "**Fork CI: waiting for a push.** The workflow is enabled in your fork, but it has not " + "run for the latest commit of this pull request. Push the branch again to start it. " + "If you already pushed, verify in your fork's Actions tab that workflows are not disabled " + "by GitHub's fork banner." + ), "running": "**Fork CI: running.** CI for the latest commit is running in your fork.", "failed": "**Fork CI: failed.** CI for the latest commit failed in your fork. Fix the failures and " - "push again; this pull request stays a draft until the run is green.", + "push again.", "green": "**Fork CI: green.** CI for the latest commit passed in your fork. The pull request is " "ready for maintainer review.", "green_unverified": "**Fork CI: green, needs a maintainer.** CI for the latest commit passed in your " - "fork, but this pull request changes files that decide which CI jobs run and where (`.github/`, " - "the breeze sources, the breeze installation or `scripts/ci/prek/`), and the fork run used those " - "changed files. A maintainer will review the change and run project CI for it before it can be " - "merged.", + f"fork, but {_CHANGES_CI_FILES} A maintainer will review the change and run project CI for it before " + "it can be merged.", "closed": "**Fork CI: closed after inactivity.** CI did not run in your fork in time; see the " "comment below for how to continue.", "gate_bypassed": "**Fork CI: closed.** Project CI ran for this pull request although its CI runs in " @@ -169,28 +222,161 @@ def _setup_instructions(fork: str) -> str: "gate_modified": "**Fork CI: closed.** This pull request changes the fork CI gate; see the comment below.", } +# Drafts do not count towards the open pull request limit, so marking one ready needs a free slot. +# GitHub refuses it at the limit whoever tries, maintainers included. +_FREE_A_SLOT_TO_MARK_READY = ( + f"If GitHub does not let you mark it ready, you have reached the [limit of open pull requests]" + f"({OPEN_PR_LIMIT_URL}): drafts do not count towards it, so this one needs a free slot. Close or " + "finish one of your other open pull requests first; a maintainer cannot mark it ready for you either." +) + +_AUTHOR_DRAFT_TEXTS = { + "green": "**Fork CI: green.** CI for the latest commit passed in your fork. The pull request is a " + "draft, so maintainers do not review it yet. Mark it ready for review when it is. " + f"{_FREE_A_SLOT_TO_MARK_READY}", + "green_unverified": "**Fork CI: green.** CI for the latest commit passed in your fork, but " + f"{_CHANGES_CI_FILES} The pull request is a draft. Once you mark it ready for review, a maintainer " + f"will review the change and run project CI for it. {_FREE_A_SLOT_TO_MARK_READY}", +} + + +# A run that fails to start has no jobs and no log: GitHub rejected the workflow before running it, +# which the generic "failed" text, asking to fix the failures, does not explain. +STARTUP_FAILURE_TEXT = ( + "**Fork CI: could not start.** GitHub could not start the `Tests (AMD)` run for the latest commit " + "in your fork, so no job ran. This usually means that your fork's Actions settings block the " + "workflow: under **Settings > Actions > General** in your fork, GitHub Actions must be enabled and " + "allow all actions and reusable workflows. It can also mean that the workflow file is invalid. The " + "run page in your fork shows GitHub's error. Fix the cause and push again." +) + + +class FailedJob(NamedTuple): + """A job of a failed fork run that did not pass, as listed in the state comment.""" + + name: str + url: str + conclusion: str + + +# A full-matrix run can fail in dozens of jobs; the job statuses list them all. +MAX_LISTED_FAILED_JOBS = 10 +FAILED_JOB_LIST_HEADING = "Jobs that did not pass:" +_FAILED_JOB_OUTCOMES = { + "failure": "failed", + "timed_out": "timed out", + "cancelled": "cancelled", + "startup_failure": "could not start", +} + + +def build_failed_job_list(failed_jobs: Sequence[FailedJob]) -> str: + listed = [ + # Job names come from the contributor's workflow; a code span keeps them from posting + # Markdown, including mentions, as the bot. + f"- [`{_sanitize_path(job.name[:JOB_NAME_MAX_LENGTH])}`]({job.url}): " + f"{_FAILED_JOB_OUTCOMES.get(job.conclusion, 'failed')}" + for job in failed_jobs[:MAX_LISTED_FAILED_JOBS] + ] + if len(failed_jobs) > MAX_LISTED_FAILED_JOBS: + listed.append(f"- and {len(failed_jobs) - MAX_LISTED_FAILED_JOBS} more, see the run") + # One paragraph of the comment, so that ``strip_failed_job_list`` can find all of it. + return FAILED_JOB_LIST_HEADING + "\n" + "\n".join(listed) + + +def strip_failed_job_list(body: str) -> str: + """The state comment without its list of failed jobs.""" + return "\n\n".join(part for part in body.split("\n\n") if not part.startswith(FAILED_JOB_LIST_HEADING)) -def behind_warning(behind: int, *, base_ref: str) -> str: + +def tested_base_text(base: BaseCheck, *, base_ref: str) -> str: + text = ( + f"Fork CI tested this branch on top of `{base_ref}` at " + f"[`{base.sha[:10]}`](https://github.com/apache/airflow/commit/{base.sha}), " + f"{base.behind} commits behind `{base_ref}` when the run turned green. Fork CI tests the branch " + f"as it is, while project CI would test it merged with the current `{base_ref}`." + ) + if base.is_current: + return f"{text} No commit since changed a file this pull request changes." + if base.overlap is None: + reason = f"`{base_ref}` changed too many files since to tell whether it changed the same files." + elif base.overlap: + listed = ", ".join(f"`{path}`" for path in base.overlap[:OVERLAP_LISTED]) + more = len(base.overlap) - OVERLAP_LISTED + reason = f"`{base_ref}` changed files this pull request changes since: {listed}" + ( + f" and {more} more." if more > 0 else "." + ) + else: + reason = "That is too far behind to say much about the merged result." return ( - f"**This branch is {behind} commits behind `{base_ref}`.** Fork CI tests the branch as it is, " - f"while the merged result also contains those commits, so it can still fail after the merge. " - f"Rebase on `{base_ref}` and push to test it against the current code." + f"{text}\n\n**Rebase needed.** {reason} The `CI passed` check is not reported until fork CI is " + f"green for a branch rebased on `{base_ref}`:\n\n" + f"```bash\ngit fetch upstream\ngit rebase upstream/{base_ref}\ngit push --force-with-lease\n```" + ) + + +_FORK_CI_PASSED_PREFIX = "Fork CI passed" +_REBASE_NEEDED_PREFIX = "Rebase on " + + +def ci_passed_description(base: BaseCheck, *, base_ref: str) -> str: + return ( + f"{_FORK_CI_PASSED_PREFIX}; base {base_ref}@{base.sha[:10]} was {base.behind} commits behind, " + "no overlap" + ) + + +def rebase_needed_description(base: BaseCheck, *, base_ref: str) -> str: + if base.overlap is None: + why = f"{base_ref} changed too many files since to check" + elif base.overlap: + why = f"{base_ref} changed {len(base.overlap)} of the same files since" + else: + why = f"base is {base.behind} commits behind" + return f"{_REBASE_NEEDED_PREFIX}{base_ref} and push: {why}" + + +def is_fork_ci_passed_description(description: str | None) -> bool: + """ + Whether a `CI passed` status description is one the reconciler wrote from a fork run. + + The prefix also matches "Fork CI passed for this commit", written before the base was checked. + """ + if not description: + return False + return description.startswith((_FORK_CI_PASSED_PREFIX, _REBASE_NEEDED_PREFIX)) or description in ( + CI_PASSED_WAITING_PROJECT_CI_DESCRIPTION, + CI_PASSED_REVOKED_DESCRIPTION, ) -def state_comment(marker: Marker, *, fork: str, run_url: str | None, base_ref: str = "main") -> str: - parts = [render_marker(marker), _STATE_TEXTS[marker.state], WHY_FORK_CI] - if ( - marker.state in ("green", "green_unverified") - and marker.behind is not None - and marker.behind >= BEHIND_WARNING_COMMITS - ): - parts.append(behind_warning(marker.behind, base_ref=base_ref)) +def state_comment( + marker: Marker, + *, + fork: str, + run_url: str | None, + base_ref: str = "main", + author_draft: bool = False, + startup_failure: bool = False, + failed_jobs: Sequence[FailedJob] = (), +) -> str: + """ + :param startup_failure: the fork run of a failed state did not start, see ``STARTUP_FAILURE_TEXT``. + :param failed_jobs: the jobs of a failed state's fork run that did not pass. + """ + text = _AUTHOR_DRAFT_TEXTS.get(marker.state) if author_draft else None + if startup_failure: + text = STARTUP_FAILURE_TEXT + parts = [render_marker(marker), text or _STATE_TEXTS[marker.state], WHY_FORK_CI] + if marker.state in ("green", "green_unverified") and marker.base is not None: + parts.append(tested_base_text(marker.base, base_ref=base_ref)) if run_url: parts.append(f"Latest run: {run_url}") + if marker.state == "failed" and failed_jobs: + parts.append(build_failed_job_list(failed_jobs)) if marker.state == "setup_required": parts.append(_setup_instructions(fork)) - parts.append(f"How this works: [fork CI for external contributors]({DESIGN_URL}).") + parts.append(f"How this works: [CI in your fork]({CONTRIBUTING_URL}).") return "\n\n".join(parts) + FOOTER @@ -324,6 +510,8 @@ def ci_modification_note_comment(*, ci_files: list[str], gated: bool) -> str: "awaiting_push": "Fork CI enabled, waiting for a push - see the pull request comment", "running": "Fork CI running - see Details", "failed": "Fork CI failed - see Details", + # Not a state of its own: a failed state whose fork run did not start is handled as failed. + "startup_failure": "Fork CI run could not start - check your fork's Actions settings", "green": "Fork CI passed - see Details", "green_unverified": "Fork CI passed, but CI files changed - a maintainer must run project CI", "gate_bypassed": "Project CI ran for a gated pull request - see Details", @@ -363,6 +551,7 @@ def job_status_context(job_name: str) -> str: "awaiting_push": "pending", "running": "pending", "failed": "failure", + "startup_failure": "failure", "green": "success", "green_unverified": "pending", "gate_bypassed": "error", diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py index 151f33c0e829f..ed01ce7ad9890 100644 --- a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py +++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py @@ -59,7 +59,6 @@ from airflow_breeze.utils.fork_ci_github import GitHubClient, GitHubError from airflow_breeze.utils.fork_ci_messages import ( CI_PASSED_CONTEXT, - CI_PASSED_DESCRIPTION, CI_PASSED_PROJECT_DESCRIPTION, CI_PASSED_REVOKED_DESCRIPTION, CI_PASSED_WAITING_PROJECT_CI_DESCRIPTION, @@ -71,16 +70,22 @@ STALE_JOB_STATUS, STATUS_CONTEXT, STATUS_STATES, + BaseCheck, + FailedJob, Marker, + ci_passed_description, fork_actions_url, gate_modified_close_comment, inactivity_close_comment, + is_fork_ci_passed_description, job_status_context, nudge_comment, parse_marker, + rebase_needed_description, released_comment, state_comment, status_description, + strip_failed_job_list, tamper_close_comment, ungated_comment, workflow_disabled_comment, @@ -129,8 +134,19 @@ DISCOVERY_WINDOW = timedelta(hours=24) PASS_TWO_PAGE_SIZE = 25 FORK_BATCH_SIZE = 50 +# The cron interval of fork-ci-monitor.yml. Fork CI reaches a pull request only when the monitor +# runs, so this is how late it can be; the write budgets and alert thresholds below are per run and +# derived from it, to keep their hourly totals when it changes. +MONITOR_INTERVAL = timedelta(minutes=5) +RUNS_PER_HOUR = timedelta(hours=1) // MONITOR_INTERVAL + + +def count_runs_in(duration: timedelta) -> int: + return max(duration // MONITOR_INTERVAL, 1) + + # Project CI runs created this long ago or later are checked for gate tampering. Several hours -# overlap the 15-minute cadence, so a tampered run is seen even after the pull request was +# overlap the monitor's cadence, so a tampered run is seen even after the pull request was # pushed again with the gate restored. TAMPER_WINDOW = timedelta(hours=6) # A ci-amd.yml run in which only the gate executed finishes in about a minute; only longer runs @@ -171,30 +187,30 @@ RUNNING_JOB_STATUSES = frozenset({"queued", "waiting", "pending", "requested", "in_progress"}) JOB_STATE_RANKS = {"failure": 0, "pending": 1, "success": 2} # Commit statuses count towards GitHub's limit of 500 content-creating requests per hour, shared by -# every workflow of the repository. At four runs an hour this keeps job statuses to at most about -# half of it; the rest follow on the next run, failures first. See "Job statuses" in the design doc. -MAX_JOB_STATUSES_PER_RUN = 40 +# every workflow of the repository. This keeps job statuses to about a third of it; the rest follow +# on the next run, failures first. See "Job statuses" in the design doc. +MAX_JOB_STATUSES_PER_RUN = 160 // RUNS_PER_HOUR # `CI run started` statuses of pull requests whose CI runs in apache/airflow come out of that budget # first: one write per new head commit, which merging waits for, while job statuses are informational. # At most this many, so job statuses keep the rest; the first runs after the check became required # catch up on the open pull requests a batch at a time, most recently updated first. -MAX_CI_RUN_STARTED_PER_RUN = 20 +MAX_CI_RUN_STARTED_PER_RUN = 80 // RUNS_PER_HOUR # `CI passed` successes for pull requests carrying `use project ci`, written once per new project CI # pass, so far fewer than this per run. Revoking a success is never limited. -MAX_PROJECT_CI_PASSED_PER_RUN = 5 +MAX_PROJECT_CI_PASSED_PER_RUN = max(20 // RUNS_PER_HOUR, 1) # The GitHub Actions app, whose check suites are the workflow runs of a commit. GITHUB_ACTIONS_APP_ID = 15368 # A single large fork run can need more writes than one run's budget, which the next run catches up # on, so deferring is only reported when it lasts this many consecutive runs: then the budget does # not keep up with the pull requests. -DEFERRED_RUNS_BEFORE_ALERT = 3 +DEFERRED_RUNS_BEFORE_ALERT = count_runs_in(timedelta(minutes=45)) # GitHub answering with rate limits in this many consecutive runs is reported: a single burst is # retried, while a persisting one means the repository's shared limits are close to exhausted. -RATE_LIMITED_RUNS_BEFORE_ALERT = 2 +RATE_LIMITED_RUNS_BEFORE_ALERT = count_runs_in(timedelta(minutes=30)) # Reading a fork run's jobs fails now and then on GitHub server errors (HTTP 502, a reset HTTP/2 # stream) that the next run gets past, so a pull request whose job statuses failed is only reported # when it failed in this many consecutive runs. -JOB_STATUS_FAILURE_RUNS_BEFORE_ALERT = 3 +JOB_STATUS_FAILURE_RUNS_BEFORE_ALERT = count_runs_in(timedelta(minutes=45)) # Fixed text, so the item is posted to Slack once while the failure persists, whatever the error; # the run log has each run's error. JOB_STATUSES_FAILED_REASON = ( @@ -351,6 +367,70 @@ def _set_label(plan: Plan, labels: set[str], label: str, *, present: bool) -> No plan.remove_labels.append(label) +@dataclass(frozen=True) +class Projection: + """ + What a gated pull request shows in a state; None leaves its draft state or label as it is. + + :param draft: whether the pull request should be a draft. + :param ready_for_review: whether it carries `ready for maintainer review`. + :param needs_project_ci: whether it carries `needs project ci`. + """ + + draft: bool | None + ready_for_review: bool | None + needs_project_ci: bool + + +def compute_projection(state: ForkState, *, author_draft: bool) -> Projection: + """ + Return what a pull request shows in a state. + + :param author_draft: the pull request is a draft that the bot did not make, including one opened + as a draft. It is never undrafted and waits for its author, not for a maintainer. + """ + if state in (ForkState.GREEN, ForkState.GREEN_UNVERIFIED): + if author_draft: + return Projection(draft=None, ready_for_review=False, needs_project_ci=False) + return Projection( + draft=False, + ready_for_review=state == ForkState.GREEN, + needs_project_ci=state == ForkState.GREEN_UNVERIFIED, + ) + if state in (ForkState.RELEASED, ForkState.UNGATED): + return Projection( + draft=None if author_draft else False, ready_for_review=None, needs_project_ci=False + ) + if state in (ForkState.SETUP_REQUIRED, ForkState.AWAITING_PUSH): + return Projection(draft=True, ready_for_review=False, needs_project_ci=False) + # A running or failed pull request keeps its draft state, so a flaky test after a rebase does + # not draft an approved one. A closed one is not drafted, since the draft-state job runs after + # the close and GitHub refuses to draft a closed pull request. + if state in ( + ForkState.RUNNING, + ForkState.FAILED, + ForkState.CLOSED, + ForkState.GATE_BYPASSED, + ForkState.GATE_MODIFIED, + ): + return Projection(draft=None, ready_for_review=False, needs_project_ci=False) + raise ValueError(f"No projection for fork CI state {state.value!r}") + + +def _is_author_draft(pr: PullRequestSnapshot) -> bool: + return pr.is_draft and not _is_bot(pr.last_draft_actor) + + +def _enter_state(plan: Plan, pr: PullRequestSnapshot, labels: set[str], state: ForkState) -> None: + plan.state = state + projection = compute_projection(state, author_draft=_is_author_draft(pr)) + if projection.draft is not None and projection.draft != pr.is_draft: + plan.draft = projection.draft + if projection.ready_for_review is not None: + _set_label(plan, labels, LABEL_READY_FOR_REVIEW, present=projection.ready_for_review) + _set_label(plan, labels, LABEL_NEEDS_PROJECT_CI, present=projection.needs_project_ci) + + def _workflow_file(resource_path: str | None) -> str: return (resource_path or "").rsplit("/", 1)[-1] @@ -523,17 +603,24 @@ def _set_status( plan.status = desired -def _set_ci_passed(plan: Plan, pr: PullRequestSnapshot, *, passed: bool, target_url: str) -> None: +def _set_ci_passed( + plan: Plan, pr: PullRequestSnapshot, state: ForkState, base: BaseCheck | None, *, target_url: str +) -> None: """ Report fork CI to the `CI passed` check the merge ruleset requires. - Only success is ever posted: a gated pull request without the status cannot merge, which is - what a pending or failing one would say. A success that no longer holds for the head commit, - such as a failed re-run in the fork, is overwritten so the pull request cannot merge on it. + Success is posted only for a trusted green run whose base is current enough; a gated pull + request without the status cannot merge. A green run on an old base gets a pending status that + says to rebase. A success that no longer holds for the head commit, such as a failed re-run in + the fork, is overwritten so the pull request cannot merge on it. """ - if passed and pr.ci_passed_state != "success": - plan.ci_passed = ("success", target_url, CI_PASSED_DESCRIPTION) - elif not passed and pr.ci_passed_state == "success": + if state == ForkState.GREEN and base is not None and base.is_current: + if pr.ci_passed_state != "success": + plan.ci_passed = ("success", target_url, ci_passed_description(base, base_ref=pr.base_ref)) + elif state == ForkState.GREEN and base is not None: + if pr.ci_passed_state != "pending": + plan.ci_passed = ("pending", target_url, rebase_needed_description(base, base_ref=pr.base_ref)) + elif pr.ci_passed_state == "success": plan.ci_passed = ("pending", target_url, CI_PASSED_REVOKED_DESCRIPTION) @@ -557,10 +644,11 @@ def classify( repo: str, find_bypass_run: Callable[[], str | None], changes_ci_files: Callable[[], bool | None], - count_behind: Callable[[], int | None] = lambda: None, + compare_base: Callable[[], BaseCheck | None] = lambda: None, jobs_reported: str | None = None, head_shared_across_bases: bool = False, find_gate_change: Callable[[], str | None] = lambda: None, + find_failed_jobs: Callable[[], list[FailedJob] | None] = lambda: [], ) -> Plan: """ Decide the mutations for one gated pull request. @@ -573,8 +661,9 @@ def classify( cannot be told for the snapshot's head commit; only called when a new head commit turns green in the fork. On None nothing is changed, since the decision would be recorded on a commit the answer may not belong to. - :param count_behind: how many commits the head commit is behind the base branch, None when - unknown; called at the same time as ``changes_ci_files``. + :param compare_base: compares the head commit's base with the base branch, None when that + cannot be read; called when a new head commit turns green, alongside ``changes_ci_files``, + and on later runs until it succeeds. :param jobs_reported: the completed fork run whose jobs are all reported as statuses, recorded in the marker so that later runs do not read its jobs again. :param head_shared_across_bases: another gated pull request has the same head commit and a @@ -582,6 +671,9 @@ def classify( :param find_gate_change: describes how the pull request changes the fork CI gate, or returns None; only called for pull requests that the gate tampering check applies to, once per head commit. + :param find_failed_jobs: the jobs of the fork run that did not pass, None when they cannot be + read; only called for a failed run that started. On None the state comment is left for the + next run, so that a failed read does not drop the list from it. """ plan = Plan() labels = set(pr.labels) @@ -591,11 +683,8 @@ def classify( marker = None def release(state: ForkState, comment: Callable[[Marker], str]) -> Plan: - plan.state = state plan.remove_labels.append(LABEL_FORK_CI_REQUIRED) - _set_label(plan, labels, LABEL_NEEDS_PROJECT_CI, present=False) - if pr.is_draft and _is_bot(pr.last_draft_actor): - plan.draft = False + _enter_state(plan, pr, labels, state) since = marker.since if marker and marker.state == state.value else now _set_marker_comment(plan, pr, comment(Marker(state.value, pr.head_sha, since))) checks_url = f"https://github.com/{repo}/pull/{pr.number}/checks" @@ -628,12 +717,7 @@ def release(state: ForkState, comment: Callable[[Marker], str]) -> Plan: if reason in TAMPER_CHECKED_REASONS: bypass_run = find_bypass_run() if bypass_run is not None: - plan.state = ForkState.GATE_BYPASSED - if not pr.is_draft: - plan.draft = True - if LABEL_READY_FOR_REVIEW in labels: - plan.remove_labels.append(LABEL_READY_FOR_REVIEW) - _set_label(plan, labels, LABEL_NEEDS_PROJECT_CI, present=False) + _enter_state(plan, pr, labels, ForkState.GATE_BYPASSED) _set_status(plan, pr, ForkState.GATE_BYPASSED.value, bypass_run) _set_marker_comment( plan, @@ -660,8 +744,7 @@ def release(state: ForkState, comment: Callable[[Marker], str]) -> Plan: if marker and marker.state in reopened_since_closing: reopened = reopened_since_closing[marker.state] if reopened is None or reopened <= marker.since: - plan.state = ForkState(marker.state) - _set_label(plan, labels, LABEL_NEEDS_PROJECT_CI, present=False) + _enter_state(plan, pr, labels, ForkState(marker.state)) if plan.state == ForkState.CLOSED: comment = inactivity_close_comment(days=config.close_after_days, fork=pr.head_repo) elif plan.state == ForkState.GATE_MODIFIED: @@ -694,11 +777,7 @@ def release(state: ForkState, comment: Callable[[Marker], str]) -> Plan: if gate_checked and not already_checked: change = find_gate_change() if change is not None: - plan.state = ForkState.GATE_MODIFIED - if not pr.is_draft: - plan.draft = True - _set_label(plan, labels, LABEL_READY_FOR_REVIEW, present=False) - _set_label(plan, labels, LABEL_NEEDS_PROJECT_CI, present=False) + _enter_state(plan, pr, labels, ForkState.GATE_MODIFIED) _set_status( plan, pr, ForkState.GATE_MODIFIED.value, f"https://github.com/{repo}/pull/{pr.number}/files" ) @@ -721,7 +800,7 @@ def release(state: ForkState, comment: Callable[[Marker], str]) -> Plan: plan.unknown.append(unknown_reason or "unknown fork state") return plan refresh_status = False - behind = marker.behind if marker and marker.sha == pr.head_sha else None + base = marker.base if marker and marker.sha == pr.head_sha else None if state == ForkState.GREEN: # The decision is read back from the commit status on the head commit, which only # apache/airflow's own tokens can write, never from a comment. The changed files depend @@ -751,9 +830,16 @@ def release(state: ForkState, comment: Callable[[Marker], str]) -> Plan: if changes is None: return Plan() unverified = changes - behind = count_behind() + base = None if unverified: state = ForkState.GREEN_UNVERIFIED + if base is None: + base = compare_base() + if base is None and state == ForkState.GREEN: + plan.unknown.append( + f"fork CI is green, but the head commit could not be compared with `{pr.base_ref}`, " + "so `CI passed` waits for the next run" + ) plan.state = state since = marker.since if marker and marker.state == state.value else now @@ -771,8 +857,7 @@ def release(state: ForkState, comment: Callable[[Marker], str]) -> Plan: ) if state in CLOSABLE_STATES and now - since >= timedelta(days=config.close_after_days): - plan.state = ForkState.CLOSED - _set_label(plan, labels, LABEL_NEEDS_PROJECT_CI, present=False) + _enter_state(plan, pr, labels, ForkState.CLOSED) _set_marker_comment( plan, pr, @@ -786,49 +871,57 @@ def release(state: ForkState, comment: Callable[[Marker], str]) -> Plan: plan.close = True return plan - if state in (ForkState.GREEN, ForkState.GREEN_UNVERIFIED): - # A green run that changes CI files still needs a maintainer's review, which a draft would - # hide; the `needs project ci` label marks it instead of the review label. - if pr.is_draft and _is_bot(pr.last_draft_actor): - plan.draft = False - ready = not pr.is_draft or plan.draft is False - if state == ForkState.GREEN and ready and LABEL_READY_FOR_REVIEW not in labels: - plan.add_labels.append(LABEL_READY_FOR_REVIEW) - elif state == ForkState.GREEN_UNVERIFIED and LABEL_READY_FOR_REVIEW in labels: - plan.remove_labels.append(LABEL_READY_FOR_REVIEW) - else: - # A run in progress needs nothing from the author, and drafting on it would draft every - # approved pull request the moment it is rebased; it is drafted only if the run then fails. - if not pr.is_draft and state != ForkState.RUNNING: - plan.draft = True - if LABEL_READY_FOR_REVIEW in labels: - plan.remove_labels.append(LABEL_READY_FOR_REVIEW) - # Lets maintainers list the pull requests that only wait for them to run project CI, which - # otherwise look like any other pull request with a pending fork CI status. - _set_label(plan, labels, LABEL_NEEDS_PROJECT_CI, present=state == ForkState.GREEN_UNVERIFIED) + _enter_state(plan, pr, labels, state) target_url = fork.run_url or fork_actions_url(pr.head_repo) - _set_status(plan, pr, state.value, target_url, force=refresh_status) - _set_ci_passed(plan, pr, passed=state == ForkState.GREEN, target_url=target_url) + startup_failure = state == ForkState.FAILED and fork.suite_conclusion == "STARTUP_FAILURE" + _set_status( + plan, pr, "startup_failure" if startup_failure else state.value, target_url, force=refresh_status + ) + _set_ci_passed(plan, pr, state, base, target_url=target_url) _set_ci_run_started(plan, pr, fork) green = state in (ForkState.GREEN, ForkState.GREEN_UNVERIFIED) - _set_marker_comment( - plan, - pr, - state_comment( - Marker( - state.value, - pr.head_sha, - since, - behind=behind if green else None, - jobs=jobs_reported, - gate_checked=gate_checked, - ), + lists_failed_jobs = state == ForkState.FAILED and not startup_failure and fork.run_id is not None + new_marker = Marker( + state.value, + pr.head_sha, + since, + base=base if green else None, + jobs=jobs_reported, + failed_run=f"{fork.run_id}:{fork.run_attempt}" if lists_failed_jobs else None, + gate_checked=gate_checked, + ) + + def render(marker: Marker, failed_jobs: list[FailedJob]) -> str: + return state_comment( + marker, fork=pr.head_repo, run_url=fork.run_url, base_ref=pr.base_ref, - ), - ) + author_draft=_is_author_draft(pr), + startup_failure=startup_failure, + failed_jobs=failed_jobs, + ) + + failed_jobs: list[FailedJob] | None = [] + if lists_failed_jobs: + # A finished run's jobs do not change, so the list already in the comment still holds. + if ( + marker == new_marker + and pr.marker_comment_body is not None + and strip_failed_job_list(pr.marker_comment_body) == render(new_marker, []) + ): + return plan + failed_jobs = find_failed_jobs() + if failed_jobs is None: + # Only a comment that already shows this head as failed is worth keeping; anything else + # describes a state the pull request is no longer in. + if marker and marker.state == ForkState.FAILED.value and marker.sha == pr.head_sha: + return plan + failed_jobs = [] + # Without a list, the next run reads the jobs again. + new_marker = replace(new_marker, failed_run=None) + _set_marker_comment(plan, pr, render(new_marker, failed_jobs)) return plan @@ -854,11 +947,8 @@ def release(state: ForkState, comment: Callable[[Marker], str]) -> Plan: } """ -PASS_TWO_QUERY = """ -query($q: String!, $cursor: String) { - search(type: ISSUE, first: %(page)d, after: $cursor, query: $q) { - pageInfo { hasNextPage endCursor } - nodes { ... on PullRequest { +PASS_TWO_FIELDS = """ +fragment PassTwoFields on PullRequest { number id isDraft createdAt authorAssociation baseRefName headRefName headRefOid author { login } headRepository { nameWithOwner } @@ -887,10 +977,35 @@ def release(state: ForkState, comment: Callable[[Marker], str]) -> Plan: pageInfo { hasNextPage endCursor } nodes { databaseId body author { login } } } - } } +} +""" + +PASS_TWO_QUERY = ( + """ +query($q: String!, $cursor: String) { + search(type: ISSUE, first: %(page)d, after: $cursor, query: $q) { + pageInfo { hasNextPage endCursor } + nodes { ...PassTwoFields } } } """ + + PASS_TWO_FIELDS +) + +PULL_REQUESTS_BY_NUMBER_QUERY = ( + """ +query($owner: String!, $name: String!) { + repository(owner: $owner, name: $name) { %(pull_requests)s } +} +""" + + PASS_TWO_FIELDS +) + +PASS_TWO_CONTEXTS = { + "context": STATUS_CONTEXT, + "ci_passed": CI_PASSED_CONTEXT, + "ci_run_started": CI_RUN_STARTED_CONTEXT, +} def _labels(node: dict[str, Any]) -> tuple[str, ...]: @@ -1351,6 +1466,35 @@ def _search(client: GitHubClient, query: str, search_query: str) -> Iterable[dic cursor = result["pageInfo"]["endCursor"] +def fetch_open_pull_requests( + client: GitHubClient, numbers: Sequence[int], *, repo: str +) -> list[dict[str, Any]]: + """ + Return the pass 2 nodes of the pull requests among these that are still open. + + Reads them by number rather than through the search index, which can take minutes to list a + pull request under a label added moments before. + """ + owner, name = repo.split("/", 1) + nodes: list[dict[str, Any]] = [] + for start in range(0, len(numbers), PASS_TWO_PAGE_SIZE): + batch = numbers[start : start + PASS_TWO_PAGE_SIZE] + pull_requests = " ".join( + f"pr{index}: pullRequest(number: {number}) {{ state ...PassTwoFields }}" + for index, number in enumerate(batch) + ) + result = ( + client.graphql( + PULL_REQUESTS_BY_NUMBER_QUERY % {"pull_requests": pull_requests, **PASS_TWO_CONTEXTS}, + {"owner": owner, "name": name}, + allow_partial=True, + )["repository"] + or {} + ) + nodes.extend(node for node in result.values() if node and node["state"] == "OPEN") + return nodes + + def _is_not_found(error: GitHubError) -> bool: return "404" in str(error) or "Not Found" in str(error) @@ -1563,12 +1707,7 @@ def report_orphaned_ci_passed( status = read_ci_passed_status(client, repo, node["headRefOid"]) if ( (status.get("state") or "").lower() in ("", "success") - or status.get("description") - not in ( - CI_PASSED_DESCRIPTION, - CI_PASSED_WAITING_PROJECT_CI_DESCRIPTION, - CI_PASSED_REVOKED_DESCRIPTION, - ) + or not is_fork_ci_passed_description(status.get("description")) or len(written) >= MAX_PROJECT_CI_PASSED_PER_RUN ): return @@ -1661,6 +1800,53 @@ def build_job_statuses(jobs: Iterable[dict[str, Any]]) -> dict[str, tuple[str, s return statuses +def find_failed_jobs(jobs: Iterable[dict[str, Any]]) -> list[FailedJob]: + """The finished jobs of a fork run that did not pass, by name, leaving out the jobs every run has.""" + failed: dict[str, FailedJob] = {} + for job in jobs: + if job["name"] in UNREPORTED_JOB_NAMES or job.get("status") in RUNNING_JOB_STATUSES: + continue + conclusion = job.get("conclusion") or "" + if JOB_STATUSES.get(conclusion, ("",))[0] != "failure": + continue + listed = failed.get(job["name"]) + # Of jobs sharing a name, one that failed is listed rather than one cancelled or timed out. + if listed is None or (listed.conclusion != "failure" and conclusion == "failure"): + failed[job["name"]] = FailedJob(job["name"], job["html_url"], conclusion) + # Failures first: the cap on the listed jobs must not hide them behind cancelled or timed-out ones. + return sorted(failed.values(), key=lambda job: (job.conclusion != "failure", job.name)) + + +# The jobs of fork runs read in this reconciler run, by run id. None records a read that failed after +# its retries, so the next reader gives up at once instead of repeating them; the next run reads again. +ReadJobs = dict[int, list[dict[str, Any]] | None] + + +def read_failed_jobs( + client: GitHubClient, + pr: PullRequestSnapshot, + fork: ForkObservation, + read_jobs: ReadJobs, +) -> list[FailedJob] | None: + """ + ``find_failed_jobs`` of the pull request's fork run; None when its jobs cannot be read. + + :param read_jobs: the jobs already read in this reconciler run; a read made here is recorded too. + """ + if fork.run_id is None or not pr.head_repo: + return [] + if fork.run_id in read_jobs: + jobs = read_jobs[fork.run_id] + else: + try: + jobs = client.get_run_jobs(pr.head_repo, fork.run_id) + except GitHubError as e: + console_print(f"[warning]#{pr.number}: could not read the jobs of {fork.run_url}: {e}[/]") + jobs = None + read_jobs[fork.run_id] = jobs + return None if jobs is None else find_failed_jobs(jobs) + + class JobStatusReporter: """Reports each job of a pull request's fork run as a commit status, within a per-run budget.""" @@ -1671,12 +1857,17 @@ def __init__( repo: str, budget: int = MAX_JOB_STATUSES_PER_RUN, now: datetime | None = None, + read_jobs: ReadJobs | None = None, ): - """:param now: when given, deferring the statuses of a run that finished long ago is reported.""" + """ + :param now: when given, deferring the statuses of a run that finished long ago is reported. + :param read_jobs: where the jobs read, or a failed read, are kept, for the state comment to reuse. + """ self._client = client self._repo = repo self._now = now self.remaining = budget + self.read_jobs = {} if read_jobs is None else read_jobs self.deferred: list[int] = [] # (pull request, attention item) for runs whose statuses are deferred long after they finished. self.delayed: list[tuple[int, str]] = [] @@ -1721,11 +1912,12 @@ def report(self, pr: PullRequestSnapshot, fork: ForkObservation) -> str | None: if self.remaining == 0: self._defer(pr, fork) return None - jobs = self._client.rest_get( - f"repos/{pr.head_repo}/actions/runs/{fork.run_id}/jobs?filter=latest&per_page=100", - paginate=True, - items_key="jobs", - ) + try: + jobs = self._client.get_run_jobs(pr.head_repo, fork.run_id) + except Exception: + self.read_jobs[fork.run_id] = None + raise + self.read_jobs[fork.run_id] = jobs # By the case-folded context, as GitHub compares them. current = { status["context"].casefold(): ( @@ -1878,11 +2070,7 @@ def __init__( def _get_jobs(self, run: WorkflowRun) -> list[dict[str, Any]]: if run.id not in self._jobs: - self._jobs[run.id] = self._client.rest_get( - f"repos/{self._repo}/actions/runs/{run.id}/jobs?filter=latest&per_page=100", - paginate=True, - items_key="jobs", - ) + self._jobs[run.id] = self._client.get_run_jobs(self._repo, run.id) return self._jobs[run.id] def _did_run_project_ci(self, run: WorkflowRun) -> bool: @@ -2288,17 +2476,37 @@ def pull_request_changes_ci_files(client: GitHubClient, repo: str, number: int, return changes_ci_definition(changed_paths(files), file_count=len(files)) -def count_commits_behind(client: GitHubClient, repo: str, pr: PullRequestSnapshot) -> int | None: - """Commits on the base branch that the head commit does not have; None when not readable.""" +def compare_with_base(client: GitHubClient, repo: str, pr: PullRequestSnapshot) -> BaseCheck | None: + """ + Compare the base a fork run tested with the base branch; None when not readable. + + The fork run tests the head commit, so its base is the merge base with the base branch. Two + three-dot compares from it give the files the pull request changes and the files the base branch + changed since; a file in both means the merged result differs from what the fork run tested + in a file the pull request changes. Compares list at most 300 files (on their first page, which + ``per_page`` does not limit): a pull request with more is listed with the files endpoint instead, + while a base branch that changed more cannot be checked until the branch is rebased. + """ owner = (pr.head_repo or "").split("/", 1)[0] + base_ref = quote(pr.base_ref, safe="") try: - compare = client.rest_get( - f"repos/{repo}/compare/{quote(pr.base_ref, safe='')}...{owner}:{pr.head_sha}?per_page=1" + head_side = client.rest_get(f"repos/{repo}/compare/{base_ref}...{owner}:{pr.head_sha}?per_page=1") + merge_base = head_side["merge_base_commit"]["sha"] + behind = head_side["behind_by"] + base_side = ( + client.rest_get(f"repos/{repo}/compare/{merge_base}...{base_ref}?per_page=1") if behind else {} ) - except GitHubError as e: - console_print(f"[warning]#{pr.number}: could not count commits behind {pr.base_ref}: {e}[/]") + head_files = head_side.get("files") or [] + if len(head_files) >= COMPARE_FILES_CAP: + head_files = client.rest_get(f"repos/{repo}/pulls/{pr.number}/files?per_page=100", paginate=True) + except (GitHubError, KeyError, TypeError) as e: + console_print(f"[warning]#{pr.number}: could not compare with {pr.base_ref}: {e}[/]") return None - return compare.get("behind_by") + base_files = base_side.get("files") or [] + if len(head_files) >= PULL_REQUEST_FILES_CAP or len(base_files) >= COMPARE_FILES_CAP: + return BaseCheck(merge_base, behind, None) + overlap = set(changed_paths(head_files)) & set(changed_paths(base_files)) + return BaseCheck(merge_base, behind, tuple(sorted(overlap))) def find_heads_shared_across_bases(prs: Iterable[PullRequestSnapshot]) -> set[str]: @@ -2550,16 +2758,16 @@ def run_reconcile( parse_pull_request(node) for node in _search( client, - PASS_TWO_QUERY - % { - "page": PASS_TWO_PAGE_SIZE, - "context": STATUS_CONTEXT, - "ci_passed": CI_PASSED_CONTEXT, - "ci_run_started": CI_RUN_STARTED_CONTEXT, - }, + PASS_TWO_QUERY % {"page": PASS_TWO_PAGE_SIZE, **PASS_TWO_CONTEXTS}, f'repo:{repo} is:pr is:open label:"{LABEL_FORK_CI_REQUIRED}"', ) ] + found = {pr.number for pr in prs} + unlisted = [number for number in labelled if number not in found] + try: + prs.extend(parse_pull_request(node) for node in fetch_open_pull_requests(client, unlisted, repo=repo)) + except GitHubError as e: + console_print(f"[warning]Pull requests {unlisted} are left for the next run: {e}[/]") observations = fetch_fork_observations(client, prs) def is_gated_base(base_ref: str) -> bool: @@ -2603,11 +2811,13 @@ def is_gated_base(base_ref: str) -> bool: ci_passed_notes.append( f"listing the pull requests with `{LABEL_USE_PROJECT_CI}` failed: {type(e).__name__}" ) + read_jobs: ReadJobs = {} job_reporter = JobStatusReporter( client, repo=repo, budget=max(MAX_JOB_STATUSES_PER_RUN - ci_run_started_writes - ci_passed_reserved, 0), now=now, + read_jobs=read_jobs, ) previous_job_status_failure_runs = read_job_status_failure_runs(reported_file) @@ -2653,10 +2863,11 @@ def is_gated_base(base_ref: str) -> bool: changes_ci_files=functools.partial( pull_request_changes_ci_files, client, repo, pr.number, pr.head_sha ), - count_behind=functools.partial(count_commits_behind, client, repo, pr), + compare_base=functools.partial(compare_with_base, client, repo, pr), jobs_reported=jobs_reported, head_shared_across_bases=pr.head_sha in shared_heads, find_gate_change=functools.partial(gate_change_finder.find, pr), + find_failed_jobs=functools.partial(read_failed_jobs, client, pr, fork, read_jobs), ) # A revocation to `pending` always goes through: it is idempotent and must not depend on # `report_project_ci_passed` succeeding later in this run. diff --git a/dev/breeze/tests/test_fork_ci_gate.py b/dev/breeze/tests/test_fork_ci_gate.py index 29b4d5c820c57..63ae34d8f62c7 100644 --- a/dev/breeze/tests/test_fork_ci_gate.py +++ b/dev/breeze/tests/test_fork_ci_gate.py @@ -341,7 +341,7 @@ def test_parse_config_defaults(): config = parse_config("") assert config.enabled is False - assert config.close_after_days == 7 + assert config.close_after_days == 30 assert config.allowlist == {} assert config.pilot == {} assert config.job_statuses is True @@ -374,6 +374,7 @@ def test_parse_config_lowercases_pilot_logins(): pytest.param('[allowlist]\n"Org" = "login"\n', "allowlist entry 'Org'", id="allowlist-not-list"), pytest.param('[pilot]\n"login" = "2026-10-20"\n', "pilot entry 'login'", id="pilot-not-datetime"), pytest.param('[pilot]\n"login" = 2026-10-20T00:00:00\n', "pilot entry 'login'", id="pilot-naive"), + pytest.param('pilot = ["login"]\n', "pilot must be a table", id="pilot-not-table"), pytest.param("enabled = \n", "Invalid fork CI gate configuration", id="invalid-toml"), ], ) diff --git a/dev/breeze/tests/test_fork_ci_messages.py b/dev/breeze/tests/test_fork_ci_messages.py index 6bf940c69df82..d945dfe35766c 100644 --- a/dev/breeze/tests/test_fork_ci_messages.py +++ b/dev/breeze/tests/test_fork_ci_messages.py @@ -25,27 +25,36 @@ from airflow_breeze.utils.fork_ci_messages import ( _STATE_TEXTS, _STATUS_DESCRIPTIONS, - BEHIND_WARNING_COMMITS, CI_CHANGE_MARKER, CI_MODIFICATION_MARKER, - CI_PASSED_DESCRIPTION, CI_PASSED_REVOKED_DESCRIPTION, + CONTRIBUTING_URL, FOOTER, GATED_NOTE_SENTENCE, JOB_NAME_MAX_LENGTH, JOB_STATUS_CONTEXT_PREFIX, + MAX_BEHIND_COMMITS, MAX_LISTED_CI_FILES, + MAX_LISTED_FAILED_JOBS, + OPEN_PR_LIMIT_URL, + OVERLAP_LISTED, + BaseCheck, + FailedJob, Marker, ci_change_approval_comment, ci_modification_note_comment, + ci_passed_description, gate_modified_close_comment, inactivity_close_comment, + is_fork_ci_passed_description, job_status_context, nudge_comment, parse_marker, + rebase_needed_description, released_comment, render_marker, state_comment, + strip_failed_job_list, tamper_close_comment, ungated_comment, workflow_disabled_comment, @@ -57,9 +66,11 @@ @pytest.mark.parametrize( "marker", [ - pytest.param(MARKER, id="without-behind"), - pytest.param(replace(MARKER, behind=12), id="behind"), + pytest.param(MARKER, id="without-base"), + pytest.param(replace(MARKER, base=BaseCheck("def456", 12, ("a.py",))), id="base"), + pytest.param(replace(MARKER, base=BaseCheck("def456", 12, None)), id="base-unchecked"), pytest.param(replace(MARKER, jobs="123:SUCCESS"), id="jobs"), + pytest.param(replace(MARKER, failed_run="123:2"), id="failed-run"), pytest.param(replace(MARKER, gate_checked=True), id="gate-checked"), ], ) @@ -67,27 +78,168 @@ def test_marker_round_trip(marker): assert parse_marker(f"text before\n{render_marker(marker)}\ntext after") == marker -def test_marker_without_behind_keeps_its_rendering(): - assert "behind" not in render_marker(MARKER) +def test_marker_without_base_keeps_its_rendering(): + assert "base" not in render_marker(MARKER) assert "jobs" not in render_marker(MARKER) +def test_marker_from_before_the_base_check_has_no_base(): + marker = parse_marker( + '' + ) + + assert marker is not None + assert marker.base is None + + +@pytest.mark.parametrize( + ("base", "current"), + [ + pytest.param(BaseCheck("d", MAX_BEHIND_COMMITS - 1), True, id="close-no-overlap"), + pytest.param(BaseCheck("d", MAX_BEHIND_COMMITS), False, id="far-behind"), + pytest.param(BaseCheck("d", 1, ("a.py",)), False, id="overlap"), + pytest.param(BaseCheck("d", 1, None), False, id="unchecked"), + ], +) +def test_base_check_is_current(base, current): + assert base.is_current is current + + +@pytest.mark.parametrize( + ("state", "base", "expected", "unexpected"), + [ + pytest.param( + "green", + BaseCheck("d" * 40, 3), + [ + "`v3-1-test` at [`dddddddddd`](https://github.com/apache/airflow/commit/" + "d" * 40, + "3 commits", + ], + ["Rebase needed"], + id="current", + ), + pytest.param( + "green", + BaseCheck("d", MAX_BEHIND_COMMITS), + ["Rebase needed", "too far behind", "git rebase upstream/v3-1-test"], + [], + id="far-behind", + ), + pytest.param( + "green_unverified", + BaseCheck("d", 4, ("a.py", "b.py")), + ["Rebase needed", "`a.py`, `b.py`."], + [], + id="overlap", + ), + pytest.param( + "green", + BaseCheck("d", 4, tuple(f"f{i:02}.py" for i in range(OVERLAP_LISTED + 2))), + [f"`f{OVERLAP_LISTED - 1:02}.py` and 2 more."], + [f"`f{OVERLAP_LISTED:02}.py`"], + id="overlap-shortened", + ), + pytest.param( + "green", BaseCheck("d", 4, None), ["Rebase needed", "too many files"], [], id="unchecked" + ), + pytest.param("failed", BaseCheck("d", MAX_BEHIND_COMMITS), [], ["tested this branch"], id="failed"), + pytest.param("green", None, [], ["tested this branch"], id="not-compared"), + ], +) +def test_state_comment_says_which_base_was_tested(state, base, expected, unexpected): + text = state_comment( + Marker(state, "a", MARKER.since, base=base), fork="u/airflow", run_url=None, base_ref="v3-1-test" + ) + + assert all(part in text for part in expected) + assert not any(part in text for part in unexpected) + + @pytest.mark.parametrize( - ("state", "behind", "warned"), + ("state", "author_draft", "asks_to_mark_ready"), [ - pytest.param("green", BEHIND_WARNING_COMMITS, True, id="green-far-behind"), - pytest.param("green_unverified", BEHIND_WARNING_COMMITS + 1, True, id="unverified-far-behind"), - pytest.param("green", BEHIND_WARNING_COMMITS - 1, False, id="green-close"), - pytest.param("green", None, False, id="green-not-counted"), - pytest.param("failed", BEHIND_WARNING_COMMITS, False, id="failed"), + pytest.param("green", True, True, id="green-author-draft"), + pytest.param("green_unverified", True, True, id="unverified-author-draft"), + pytest.param("green", False, False, id="green-ready"), + pytest.param("failed", True, False, id="failed-author-draft"), ], ) -def test_state_comment_warns_about_a_branch_far_behind(state, behind, warned): +def test_state_comment_asks_the_author_to_mark_a_green_draft_ready(state, author_draft, asks_to_mark_ready): text = state_comment( - Marker(state, "a", MARKER.since, behind=behind), fork="u/airflow", run_url=None, base_ref="v3-1-test" + Marker(state, "a", MARKER.since), fork="u/airflow", run_url=None, author_draft=author_draft ) - assert ("commits behind `v3-1-test`" in text) is warned + assert ("mark it ready for review" in text.lower()) is asks_to_mark_ready + assert ("Close or finish one of your other open pull requests first" in text) is asks_to_mark_ready + assert (OPEN_PR_LIMIT_URL in text) is asks_to_mark_ready + assert "ask a maintainer to mark it ready" not in text + + +@pytest.mark.parametrize("startup_failure", [False, True]) +def test_state_comment_links_the_contributor_documentation(startup_failure): + text = state_comment( + Marker("failed", "a", MARKER.since), fork="u/airflow", run_url=None, startup_failure=startup_failure + ) + + assert f"How this works: [CI in your fork]({CONTRIBUTING_URL})." in text + assert CONTRIBUTING_URL.endswith("18_contribution_workflow.rst#ci-in-your-fork") + assert "08_fork_ci_for_external_contributors" not in text + assert text.endswith(FOOTER) + + +def test_failed_job_list_is_safe_markdown_and_capped(): + jobs = [ + FailedJob("Tests `evil`\n\n@someone", "https://job/0", "timed_out"), + *( + FailedJob(f"Job {i}", f"https://job/{i + 1}", "failure") + for i in range(MAX_LISTED_FAILED_JOBS + 1) + ), + ] + + text = state_comment( + Marker("failed", "a", MARKER.since), fork="u/airflow", run_url=None, failed_jobs=jobs + ) + + assert "- [`Tests ?evil???@someone`](https://job/0): timed out" in text + assert "\n@someone" not in text + assert "- [`Job 0`](https://job/1): failed" in text + assert "- and 2 more, see the run" in text + + +def test_failed_jobs_are_listed_only_for_a_failed_state(): + jobs = [FailedJob("Static checks", "https://job/1", "failure")] + + text = state_comment( + Marker("running", "a", MARKER.since), fork="u/airflow", run_url=None, failed_jobs=jobs + ) + + assert "Static checks" not in text + + +def test_strip_failed_job_list_leaves_the_comment_without_the_list(): + marker = Marker("failed", "a", MARKER.since) + jobs = [FailedJob(f"Job {i}", f"https://job/{i}", "failure") for i in range(MAX_LISTED_FAILED_JOBS + 1)] + + listed = state_comment(marker, fork="u/airflow", run_url="https://run", failed_jobs=jobs) + + assert strip_failed_job_list(listed) == state_comment(marker, fork="u/airflow", run_url="https://run") + + +@pytest.mark.parametrize( + ("description", "expected"), + [ + pytest.param("Fork CI passed for this commit", True, id="before-the-base-check"), + pytest.param(ci_passed_description(BaseCheck("a" * 40, 3), base_ref="main"), True, id="current-base"), + pytest.param( + rebase_needed_description(BaseCheck("a" * 40, 300), base_ref="main"), True, id="rebase-needed" + ), + pytest.param(CI_PASSED_REVOKED_DESCRIPTION, True, id="revoked"), + pytest.param("Project CI passed for this commit", False, id="project-ci"), + pytest.param(None, False, id="none"), + ], +) +def test_is_fork_ci_passed_description(description, expected): + assert is_fork_ci_passed_description(description) is expected def test_parse_marker_absent(): @@ -147,7 +299,16 @@ def test_setup_texts_give_the_exact_command(text): def test_status_descriptions_fit_github_limit(): - descriptions = [*_STATUS_DESCRIPTIONS.values(), CI_PASSED_DESCRIPTION, CI_PASSED_REVOKED_DESCRIPTION] + base_ref = "providers-cncf-kubernetes/v10-22" + descriptions = [ + *_STATUS_DESCRIPTIONS.values(), + CI_PASSED_REVOKED_DESCRIPTION, + ci_passed_description(BaseCheck("d" * 40, 199), base_ref=base_ref), + *( + rebase_needed_description(BaseCheck("d" * 40, 9999, overlap), base_ref=base_ref) + for overlap in ((), ("a",) * 300, None) + ), + ] assert all(len(description) < 140 for description in descriptions) diff --git a/dev/breeze/tests/test_fork_ci_reconcile.py b/dev/breeze/tests/test_fork_ci_reconcile.py index c7b50800756fd..fa1d432977e2a 100644 --- a/dev/breeze/tests/test_fork_ci_reconcile.py +++ b/dev/breeze/tests/test_fork_ci_reconcile.py @@ -24,6 +24,7 @@ from urllib.parse import quote import pytest +import yaml from click.testing import CliRunner from airflow_breeze.commands.ci_commands import fork_ci_reconcile @@ -31,7 +32,6 @@ from airflow_breeze.utils.fork_ci_github import GitHubClient, GitHubError from airflow_breeze.utils.fork_ci_messages import ( CI_PASSED_CONTEXT, - CI_PASSED_DESCRIPTION, CI_PASSED_PROJECT_DESCRIPTION, CI_PASSED_WAITING_PROJECT_CI_DESCRIPTION, CI_RUN_STARTED_CONTEXT, @@ -40,14 +40,23 @@ JOB_STATUS_CONTEXT_PREFIX, STALE_JOB_STATUS, STATUS_CONTEXT, + BaseCheck, + FailedJob, Marker, parse_marker, render_marker, ) from airflow_breeze.utils.fork_ci_reconcile import ( + DEFERRED_RUNS_BEFORE_ALERT, + JOB_STATUS_FAILURE_RUNS_BEFORE_ALERT, JOB_STATUSES_DEFERRED_NOTE, JOB_STATUSES_FAILED_REASON, + MAX_CI_RUN_STARTED_PER_RUN, + MAX_JOB_STATUSES_PER_RUN, + MAX_PROJECT_CI_PASSED_PER_RUN, + MONITOR_INTERVAL, RATE_LIMITED_NOTE, + RATE_LIMITED_RUNS_BEFORE_ALERT, BaseGateConfigs, BypassFinder, DraftChange, @@ -56,6 +65,7 @@ GateChangeFinder, JobStatusReporter, Plan, + Projection, PullRequestChangedError, PullRequestSnapshot, apply_plan, @@ -63,12 +73,15 @@ build_slack_payload, changes_ci_definition, classify, - count_commits_behind, + compare_with_base, + compute_projection, discover, drop_reported, fetch_fork_observations, + fetch_open_pull_requests, fetch_workflow_state, find_ci_run_since, + find_failed_jobs, find_heads_shared_across_bases, find_project_ci_label_heads, find_project_ci_passed_url, @@ -84,6 +97,7 @@ parse_pull_request, pull_request_changes_ci_files, read_ci_run_at, + read_failed_jobs, read_head_status, read_job_status_failure_runs, read_project_ci_passed_url, @@ -100,9 +114,13 @@ JOINED = CREATED + timedelta(days=2) CONFIG = GateConfig(enabled=True, enabled_since=datetime(2026, 10, 1, tzinfo=UTC), close_after_days=7) RUN_URL = "https://github.com/someone/airflow/actions/runs/1" +# A `CI passed` success the reconciler wrote for a green fork run. +FORK_CI_PASSED_DESCRIPTION = "Fork CI passed; base main@0123456789 was 3 commits behind, no overlap" PROJECT_RUN_URL = "https://github.com/apache/airflow/actions/runs/2" PROJECT_PASSED_URL = "https://github.com/apache/airflow/actions/runs/5" GREEN = ForkObservation(suite_status="COMPLETED", suite_conclusion="SUCCESS", run_url=RUN_URL) +MERGE_BASE = "def4567890aa" +CURRENT_BASE = BaseCheck(MERGE_BASE, 3) def _pr(**overrides) -> PullRequestSnapshot: @@ -135,7 +153,7 @@ def _classify( bypass_run=None, changes_ci_files=False, config=CONFIG, - behind=None, + base=CURRENT_BASE, head_shared_across_bases=False, gate_change=None, ): @@ -149,7 +167,7 @@ def _classify( repo=REPO, find_bypass_run=find_bypass, changes_ci_files=changes, - count_behind=mock.Mock(return_value=behind), + compare_base=mock.Mock(return_value=base), head_shared_across_bases=head_shared_across_bases, find_gate_change=lambda: gate_change, ) @@ -238,19 +256,46 @@ def test_setup_required_comment_gives_the_command(): assert "gh workflow enable ci-amd.yml -R someone/airflow" in plan.marker_comment +@pytest.mark.parametrize( + ("conclusion", "expected_description", "explains_settings"), + [ + pytest.param( + "STARTUP_FAILURE", + "Fork CI run could not start - check your fork's Actions settings", + True, + id="startup-failure", + ), + pytest.param("FAILURE", "Fork CI failed - see Details", False, id="failure"), + ], +) +def test_a_run_that_could_not_start_points_to_the_fork_settings( + conclusion, expected_description, explains_settings +): + fork = ForkObservation(suite_status="COMPLETED", suite_conclusion=conclusion, run_url=RUN_URL) + + plan, _, _ = _classify(_pr(is_draft=False), fork) + + assert plan.state == ForkState.FAILED + assert plan.draft is None + assert plan.status == ("failure", RUN_URL, expected_description) + assert ("**Fork CI: could not start.**" in plan.marker_comment) is explains_settings + assert ("Settings > Actions > General" in plan.marker_comment) is explains_settings + assert '"state": "failed"' in plan.marker_comment + + @pytest.mark.parametrize( ("fork", "draft"), [ pytest.param(ForkObservation(suite_status="IN_PROGRESS", run_url=RUN_URL), None, id="running"), pytest.param( ForkObservation(suite_status="COMPLETED", suite_conclusion="FAILURE", run_url=RUN_URL), - True, + None, id="failed", ), pytest.param(ForkObservation(workflow_state="disabled_fork"), True, id="setup-required"), ], ) -def test_non_green_removes_review_label_and_drafts_only_when_the_author_must_act(fork, draft): +def test_non_green_removes_review_label_and_drafts_only_without_a_fork_run(fork, draft): pr = _pr(is_draft=False, labels=("fork ci required", "ready for maintainer review")) plan, _, _ = _classify(pr, fork) @@ -259,6 +304,105 @@ def test_non_green_removes_review_label_and_drafts_only_when_the_author_must_act assert plan.remove_labels == ["ready for maintainer review"] +@pytest.mark.parametrize( + ("state", "author_draft", "expected"), + [ + pytest.param( + ForkState.SETUP_REQUIRED, + False, + Projection(draft=True, ready_for_review=False, needs_project_ci=False), + id="setup-required", + ), + pytest.param( + ForkState.AWAITING_PUSH, + False, + Projection(draft=True, ready_for_review=False, needs_project_ci=False), + id="awaiting-push", + ), + pytest.param( + ForkState.FAILED, + False, + Projection(draft=None, ready_for_review=False, needs_project_ci=False), + id="failed", + ), + pytest.param( + ForkState.FAILED, + True, + Projection(draft=None, ready_for_review=False, needs_project_ci=False), + id="failed-author-draft", + ), + pytest.param( + ForkState.RUNNING, + False, + Projection(draft=None, ready_for_review=False, needs_project_ci=False), + id="running", + ), + pytest.param( + ForkState.GREEN, + False, + Projection(draft=False, ready_for_review=True, needs_project_ci=False), + id="green", + ), + pytest.param( + ForkState.GREEN, + True, + Projection(draft=None, ready_for_review=False, needs_project_ci=False), + id="green-author-draft", + ), + pytest.param( + ForkState.GREEN_UNVERIFIED, + False, + Projection(draft=False, ready_for_review=False, needs_project_ci=True), + id="unverified", + ), + pytest.param( + ForkState.GREEN_UNVERIFIED, + True, + Projection(draft=None, ready_for_review=False, needs_project_ci=False), + id="unverified-author-draft", + ), + pytest.param( + ForkState.RELEASED, + False, + Projection(draft=False, ready_for_review=None, needs_project_ci=False), + id="released", + ), + pytest.param( + ForkState.RELEASED, + True, + Projection(draft=None, ready_for_review=None, needs_project_ci=False), + id="released-author-draft", + ), + pytest.param( + ForkState.UNGATED, + False, + Projection(draft=False, ready_for_review=None, needs_project_ci=False), + id="ungated", + ), + pytest.param( + ForkState.CLOSED, + False, + Projection(draft=None, ready_for_review=False, needs_project_ci=False), + id="closed", + ), + pytest.param( + ForkState.GATE_BYPASSED, + False, + Projection(draft=None, ready_for_review=False, needs_project_ci=False), + id="gate-bypassed", + ), + pytest.param( + ForkState.GATE_MODIFIED, + False, + Projection(draft=None, ready_for_review=False, needs_project_ci=False), + id="gate-modified", + ), + ], +) +def test_projection_table(state, author_draft, expected): + assert compute_projection(state, author_draft=author_draft) == expected + + def test_green_undrafts_what_the_bot_drafted(): plan, _, _ = _classify(_pr(last_draft_actor="github-actions"), GREEN) @@ -266,11 +410,22 @@ def test_green_undrafts_what_the_bot_drafted(): assert plan.add_labels == ["ready for maintainer review"] -def test_green_keeps_an_author_draft(): - plan, _, _ = _classify(_pr(last_draft_actor="someone"), GREEN) +@pytest.mark.parametrize( + "last_draft_actor", + [ + pytest.param("someone", id="drafted-by-author"), + pytest.param(None, id="opened-as-draft"), + ], +) +def test_green_keeps_an_author_draft_out_of_the_review_queue(last_draft_actor): + pr = _pr(last_draft_actor=last_draft_actor, labels=("fork ci required", "ready for maintainer review")) + + plan, _, _ = _classify(pr, GREEN) assert plan.draft is None assert plan.add_labels == [] + assert plan.remove_labels == ["ready for maintainer review"] + assert "Mark it ready for review when it is." in plan.marker_comment def test_green_labels_a_ready_pull_request(): @@ -300,18 +455,27 @@ def test_green_is_not_trusted_when_ci_files_change(): @pytest.mark.parametrize( - ("last_draft_actor", "expected_draft"), + ("last_draft_actor", "expected_draft", "expected_labels", "expected_text"), [ - pytest.param("github-actions[bot]", False, id="bot-draft-undrafted"), - pytest.param("someone", None, id="author-draft-kept"), + pytest.param( + "github-actions[bot]", + False, + ["needs project ci"], + "run project CI for it before it can be merged", + id="bot-draft-undrafted", + ), + pytest.param("someone", None, [], "Once you mark it ready for review", id="author-draft-kept"), ], ) -def test_green_unverified_does_not_keep_a_bot_draft(last_draft_actor, expected_draft): +def test_green_unverified_does_not_keep_a_bot_draft( + last_draft_actor, expected_draft, expected_labels, expected_text +): plan, _, _ = _classify(_pr(last_draft_actor=last_draft_actor), GREEN, changes_ci_files=True) assert plan.state == ForkState.GREEN_UNVERIFIED assert plan.draft is expected_draft - assert plan.add_labels == ["needs project ci"] + assert plan.add_labels == expected_labels + assert expected_text in plan.marker_comment LABELLED = ("fork ci required", "needs project ci") @@ -359,66 +523,116 @@ def test_needs_project_ci_label_is_removed_outside_green_unverified( ) -CI_PASSED = ("success", RUN_URL, "Fork CI passed for this commit") +CI_PASSED = ("success", RUN_URL, "Fork CI passed; base main@def4567890 was 3 commits behind, no overlap") CI_PASSED_REVOKED = ("pending", RUN_URL, "Fork CI result no longer trusted for this commit") +OVERLAPPING_BASE = BaseCheck(MERGE_BASE, 3, ("a.py",)) +REBASE_NEEDED = ("pending", RUN_URL, "Rebase on main and push: main changed 1 of the same files since") @pytest.mark.parametrize( - ("fork", "changes_ci_files", "ci_passed_state", "expected"), + ("fork", "changes_ci_files", "base", "ci_passed_state", "expected"), [ - pytest.param(GREEN, False, None, CI_PASSED, id="green-posts-success"), - pytest.param(GREEN, False, "success", None, id="green-already-posted"), - pytest.param(GREEN, False, "pending", CI_PASSED, id="green-after-revoke"), - pytest.param(GREEN, True, None, None, id="unverified-gets-none"), + pytest.param(GREEN, False, CURRENT_BASE, None, CI_PASSED, id="green-posts-success"), + pytest.param(GREEN, False, CURRENT_BASE, "success", None, id="green-already-posted"), + pytest.param(GREEN, False, CURRENT_BASE, "pending", CI_PASSED, id="green-after-revoke"), + pytest.param(GREEN, True, CURRENT_BASE, None, None, id="unverified-gets-none"), + pytest.param(GREEN, False, OVERLAPPING_BASE, None, REBASE_NEEDED, id="old-base-asks-to-rebase"), + pytest.param(GREEN, False, OVERLAPPING_BASE, "success", REBASE_NEEDED, id="old-base-revokes"), + pytest.param(GREEN, False, OVERLAPPING_BASE, "pending", None, id="old-base-already-pending"), + pytest.param(GREEN, False, None, None, None, id="base-not-compared-gets-none"), + pytest.param(GREEN, False, None, "success", CI_PASSED_REVOKED, id="base-not-compared-revokes"), pytest.param( ForkObservation(suite_status="COMPLETED", suite_conclusion="FAILURE", run_url=RUN_URL), False, + CURRENT_BASE, "success", CI_PASSED_REVOKED, id="failed-rerun-revokes", ), - pytest.param(GREEN, True, "success", CI_PASSED_REVOKED, id="unverified-after-retarget-revokes"), + pytest.param( + GREEN, True, CURRENT_BASE, "success", CI_PASSED_REVOKED, id="unverified-after-retarget-revokes" + ), pytest.param( ForkObservation(suite_status="IN_PROGRESS", run_url=RUN_URL), False, + CURRENT_BASE, None, None, id="running-gets-none", ), ], ) -def test_ci_passed_reports_trusted_green_only(fork, changes_ci_files, ci_passed_state, expected): - plan, _, _ = _classify(_pr(ci_passed_state=ci_passed_state), fork, changes_ci_files=changes_ci_files) +def test_ci_passed_reports_trusted_green_on_a_current_base_only( + fork, changes_ci_files, base, ci_passed_state, expected +): + plan, _, _ = _classify( + _pr(ci_passed_state=ci_passed_state), fork, changes_ci_files=changes_ci_files, base=base + ) assert plan.ci_passed == expected +def test_green_whose_base_cannot_be_compared_is_reported(): + plan, _, _ = _classify(_pr(), GREEN, base=None) + + assert plan.state == ForkState.GREEN + assert any("could not be compared with `main`" in item for item in plan.unknown) + + +KEPT_BASE = BaseCheck("kept", 300, None) + + @pytest.mark.parametrize( - ("pr_overrides", "behind", "expected_behind", "warned"), + ("pr_overrides", "compared", "expected_base"), [ - pytest.param({}, 250, 250, True, id="counted-on-decision"), - pytest.param({}, 3, 3, False, id="close-to-base"), + pytest.param({}, True, CURRENT_BASE, id="compared-on-decision"), pytest.param( - {"current_status": GREEN_STATUS, "marker": Marker("green", "abc123", NOW, behind=300)}, - None, - 300, - True, + {"current_status": GREEN_STATUS, "marker": Marker("green", "abc123", NOW, base=KEPT_BASE)}, + False, + KEPT_BASE, id="kept-from-the-marker", ), pytest.param( - {"current_status": GREEN_STATUS, "marker": Marker("green", "old-sha", NOW, behind=300)}, - None, - None, - False, + {"current_status": GREEN_STATUS, "marker": Marker("green", "old-sha", NOW, base=KEPT_BASE)}, + True, + CURRENT_BASE, id="not-kept-for-another-commit", ), + pytest.param( + {"current_status": GREEN_STATUS, "marker": Marker("green", "abc123", NOW)}, + True, + CURRENT_BASE, + id="compared-for-a-marker-without-base", + ), + pytest.param( + { + "current_status": GREEN_STATUS, + "marker": Marker("green", "abc123", NOW, base=KEPT_BASE), + "last_base_change_at": NOW, + "status_created_at": NOW - timedelta(hours=1), + }, + True, + CURRENT_BASE, + id="compared-again-after-retarget", + ), ], ) -def test_green_records_how_far_behind_the_base_branch(pr_overrides, behind, expected_behind, warned): - plan, _, _ = _classify(_pr(**pr_overrides), GREEN, behind=behind) +def test_green_records_the_base_it_tested(pr_overrides, compared, expected_base): + compare_base = mock.Mock(return_value=CURRENT_BASE) + plan = classify( + _pr(**pr_overrides), + GREEN, + config=CONFIG, + now=NOW, + repo=REPO, + find_bypass_run=lambda: None, + changes_ci_files=lambda: False, + compare_base=compare_base, + ) - assert parse_marker(plan.marker_comment).behind == expected_behind - assert ("commits behind `main`" in plan.marker_comment) is warned + assert compare_base.called is compared + assert parse_marker(plan.marker_comment).base == expected_base + assert f"[`{expected_base.sha[:10]}`]" in plan.marker_comment @pytest.mark.parametrize( @@ -693,7 +907,7 @@ def test_release_with_use_project_ci(): @pytest.mark.parametrize( ("ci_passed_state", "ci_passed_description", "revoked"), [ - pytest.param("success", CI_PASSED_DESCRIPTION, True, id="fork-success-is-revoked"), + pytest.param("success", FORK_CI_PASSED_DESCRIPTION, True, id="fork-success-is-revoked"), pytest.param("success", None, True, id="success-without-description-is-revoked"), pytest.param("success", CI_PASSED_PROJECT_DESCRIPTION, False, id="project-success-is-kept"), pytest.param("pending", CI_PASSED_WAITING_PROJECT_CI_DESCRIPTION, False, id="pending-is-kept"), @@ -801,7 +1015,7 @@ def test_gate_tampering_closes(pr_overrides, checked): assert plan.close is checked if checked: assert plan.state == ForkState.GATE_BYPASSED - assert plan.draft is True + assert plan.draft is None assert plan.status == ( "error", PROJECT_RUN_URL, @@ -999,8 +1213,6 @@ def rest_get_pages(path): return [{"total_count": count, "workflow_runs": event_runs}] def rest_get(path, paginate=False, items_key=None): - if path.endswith("jobs?filter=latest&per_page=100"): - return list(jobs) if "/compare/" in path: assert not paginate, "the compare API returns files only on its first page" return {"files": [{"filename": name} for name in compare_files]} @@ -1010,6 +1222,7 @@ def rest_get(path, paginate=False, items_key=None): client.rest_get_pages.side_effect = rest_get_pages client.rest_get.side_effect = rest_get + client.get_run_jobs.return_value = list(jobs) return BypassFinder(client, repo=REPO, now=NOW, base_triggers=BASE_TRIGGERS, is_gated_base=is_gated_base) @@ -1214,9 +1427,7 @@ def test_finder_reports_a_run_another_pull_request_may_own(client, other_pulls): def test_finder_catches_a_workflow_moved_away(client): finder = _finder(client, [_run(7)], jobs=TAMPERED_JOBS) client.rest_get.side_effect = lambda path, paginate=False, items_key=None: ( - TAMPERED_JOBS - if path.endswith("jobs?filter=latest&per_page=100") - else [{"number": 42, "closed_at": None}] + [{"number": 42, "closed_at": None}] if "/pulls?head=" in path else {"files": [{"filename": "docs/x.yml", "previous_filename": ".github/workflows/x.yml"}]} ) @@ -1916,12 +2127,14 @@ def _ci_passed_commit(state, description, oid="sha1"): id="ungated-waiting-project-ci-passed", ), pytest.param( - False, "PENDING", CI_PASSED_DESCRIPTION, PROJECT_PASSED_URL, [1], id="ungated-fork-status" + False, "PENDING", FORK_CI_PASSED_DESCRIPTION, PROJECT_PASSED_URL, [1], id="ungated-fork-status" ), pytest.param( False, "PENDING", CI_PASSED_WAITING_PROJECT_CI_DESCRIPTION, None, [], id="ungated-not-passed" ), - pytest.param(False, "SUCCESS", CI_PASSED_DESCRIPTION, PROJECT_PASSED_URL, [], id="already-success"), + pytest.param( + False, "SUCCESS", FORK_CI_PASSED_DESCRIPTION, PROJECT_PASSED_URL, [], id="already-success" + ), pytest.param(False, "PENDING", "someone else", PROJECT_PASSED_URL, [], id="foreign-status"), pytest.param( True, @@ -1983,7 +2196,7 @@ def test_discover_reads_the_status_of_a_head_commit_the_search_did_not_list(mock **_pr_node(1), "headRefOid": "sha1", "labels": {"nodes": []}, - "commits": _ci_passed_commit("SUCCESS", CI_PASSED_DESCRIPTION, oid="older"), + "commits": _ci_passed_commit("SUCCESS", FORK_CI_PASSED_DESCRIPTION, oid="older"), } mock_read.return_value = PROJECT_PASSED_URL client.graphql.side_effect = [_search_result(node), _search_result()] @@ -2331,13 +2544,12 @@ def test_run_reconcile_deletes_gated_runs(mock_classify, client, tmp_path, state def rest_get(path, paginate=False, items_key=None): if path.endswith("/actions/workflows/ci-amd.yml"): return {"state": "active"} - if path.endswith("jobs?filter=latest&per_page=100"): - return GATED_JOBS if "/compare/" in path: return {"files": [{"filename": "README.md"}]} raise AssertionError(path) client.rest_get.side_effect = rest_get + client.get_run_jobs.return_value = GATED_JOBS (tmp_path / "ci-amd.yml").write_text("on: [pull_request]\n") run_reconcile( @@ -2583,18 +2795,64 @@ def test_apply_plan_posts_ci_passed_under_its_own_context(client): assert mock.call(42, "abc123", *CI_PASSED, context="CI passed") in client.set_status.call_args_list +def _compare(behind, files, merge_base=MERGE_BASE): + return {"merge_base_commit": {"sha": merge_base}, "behind_by": behind, "files": files} + + +def _files(*paths): + return [{"filename": path} for path in paths] + + +HEAD_COMPARE = f"repos/{REPO}/compare/providers-fab%2Fv1-5...someone:abc123?per_page=1" +BASE_COMPARE = f"repos/{REPO}/compare/{MERGE_BASE}...providers-fab%2Fv1-5?per_page=1" +PR_FILES = f"repos/{REPO}/pulls/42/files?per_page=100" + + @pytest.mark.parametrize( - ("side_effect", "expected"), + ("responses", "expected", "expected_calls"), [ - pytest.param([{"behind_by": 17}], 17, id="counted"), - pytest.param(GitHubError("gh api failed: HTTP 500"), None, id="unreadable"), + pytest.param( + [_compare(0, _files("a.py"))], BaseCheck(MERGE_BASE, 0, ()), [HEAD_COMPARE], id="up-to-date" + ), + pytest.param( + [_compare(5, _files("a.py", "b.py")), _compare(0, _files("c.py"))], + BaseCheck(MERGE_BASE, 5, ()), + [HEAD_COMPARE, BASE_COMPARE], + id="no-overlap", + ), + pytest.param( + [ + _compare(5, [*_files("b.py"), {"filename": "new.py", "previous_filename": "a.py"}]), + _compare(0, _files("b.py", "a.py", "c.py")), + ], + BaseCheck(MERGE_BASE, 5, ("a.py", "b.py")), + [HEAD_COMPARE, BASE_COMPARE], + id="overlap-with-rename", + ), + pytest.param( + [_compare(5, _files("a.py")), _compare(0, _files(*(f"f{i}" for i in range(300))))], + BaseCheck(MERGE_BASE, 5, None), + [HEAD_COMPARE, BASE_COMPARE], + id="base-side-at-cap", + ), + pytest.param( + [ + _compare(5, _files(*(f"f{i}" for i in range(300)))), + _compare(0, _files("g.py", "f301")), + _files(*(f"f{i}" for i in range(400))), + ], + BaseCheck(MERGE_BASE, 5, ("f301",)), + [HEAD_COMPARE, BASE_COMPARE, PR_FILES], + id="large-pull-request-listed-in-full", + ), + pytest.param(GitHubError("gh api failed: HTTP 500"), None, [HEAD_COMPARE], id="unreadable"), ], ) -def test_count_commits_behind(client, side_effect, expected): - client.rest_get.side_effect = side_effect +def test_compare_with_base(client, responses, expected, expected_calls): + client.rest_get.side_effect = responses - assert count_commits_behind(client, REPO, _pr(base_ref="v3-1-test")) == expected - assert client.rest_get.call_args.args[0] == f"repos/{REPO}/compare/v3-1-test...someone:abc123?per_page=1" + assert compare_with_base(client, REPO, _pr(base_ref="providers-fab/v1-5")) == expected + assert [call.args[0] for call in client.rest_get.call_args_list] == expected_calls def _fork_job(name, status="completed", conclusion="success", url=None): @@ -2658,13 +2916,14 @@ def test_build_job_statuses_cannot_post_the_reconciler_contexts(name): def _reporter_client(client, jobs, current=()): def rest_get(path, paginate=False, items_key=None): - if path.startswith("repos/someone/airflow/actions/runs/1/jobs"): - return jobs if path.startswith(f"repos/{REPO}/commits/abc123/status"): return list(current) raise AssertionError(path) client.rest_get.side_effect = rest_get + client.get_run_jobs.side_effect = lambda repo, run_id: ( + jobs if (repo, run_id) == ("someone/airflow", 1) else pytest.fail(f"{repo} run {run_id}") + ) return client @@ -2783,6 +3042,9 @@ def test_run_reconcile_reports_job_statuses_before_classifying( (pr, fork), _ = mock_reporter.return_value.report.call_args assert pr.number == 1 assert mock_classify.call_args.kwargs["jobs_reported"] == "7:SUCCESS" + find_failed = mock_classify.call_args.kwargs["find_failed_jobs"] + assert find_failed.func is read_failed_jobs + assert find_failed.args == (client, pr, fork, mock_reporter.call_args.kwargs["read_jobs"]) @pytest.mark.parametrize("status", ["queued", "in_progress", "waiting"]) @@ -2889,23 +3151,23 @@ def test_run_reconcile_reports_failing_job_statuses_once_when_failing_persists( reported_file = tmp_path / "reported.json" reported_file.write_text("{}") alerts = [] + failures = [GitHubError("HTTP 502"), GitHubError("stream ID 1; CANCEL")] errors = [ GitHubError("HTTP 502"), GitHubError("HTTP 502"), None, - GitHubError("HTTP 502"), - GitHubError("stream ID 1; CANCEL"), - GitHubError("HTTP 502"), - GitHubError("stream ID 1; CANCEL"), + *(failures[run % 2] for run in range(JOB_STATUS_FAILURE_RUNS_BEFORE_ALERT + 1)), ] for error in errors: mock_reporter.return_value.report.side_effect = error mock_reporter.return_value.report.return_value = None alerts.append(_reconcile_with_reporter(client, tmp_path, reported_file=reported_file)) - assert alerts == [False, False, False, False, False, True, False] + assert alerts == [*[False] * (JOB_STATUS_FAILURE_RUNS_BEFORE_ALERT + 2), True, False] assert JOB_STATUSES_FAILED_REASON in (tmp_path / "s.json").read_text() - assert json.loads(reported_file.read_text())["job_status_failure_runs"] == {"1": 4} + assert json.loads(reported_file.read_text())["job_status_failure_runs"] == { + "1": JOB_STATUS_FAILURE_RUNS_BEFORE_ALERT + 1 + } @pytest.mark.parametrize( @@ -2980,14 +3242,14 @@ def test_run_reconcile_reports_deferred_job_statuses_only_when_deferring_persist reported_file = tmp_path / "reported.json" reported_file.write_text("{}") alerts = [] - for deferred in ([1], [1], [], [1], [1], [1]): + for deferred in ([1], [1], [], *[[1]] * DEFERRED_RUNS_BEFORE_ALERT): mock_reporter.return_value.deferred = deferred mock_reporter.return_value.delayed = [] alerts.append(_reconcile_with_reporter(client, tmp_path, reported_file=reported_file)) - assert alerts == [False, False, False, False, False, True] + assert alerts == [*[False] * (DEFERRED_RUNS_BEFORE_ALERT + 2), True] assert JOB_STATUSES_DEFERRED_NOTE in (tmp_path / "s.json").read_text() - assert json.loads(reported_file.read_text())["deferred_runs"] == 3 + assert json.loads(reported_file.read_text())["deferred_runs"] == DEFERRED_RUNS_BEFORE_ALERT def test_build_job_statuses_treats_contexts_case_insensitively(): @@ -3103,7 +3365,7 @@ def test_a_change_to_the_gate_closes(): assert plan.state == ForkState.GATE_MODIFIED assert plan.close is True - assert plan.draft is True + assert plan.draft is None assert plan.remove_labels == ["ready for maintainer review"] assert plan.status == ( "error", @@ -3463,6 +3725,109 @@ def test_gate_change_finder_aborts_when_the_pull_request_changed(client, head_sh finder.find(_pr()) +NEWLY_GATED = {**_pr_node(1), "labels": {"nodes": []}} + + +def _graphql_after_gating_pull_request_1(*, listed: list[dict], by_number: dict | Exception | None): + def graphql(query, variables=None, *, allow_partial=False): + if "pullRequest(number:" in query: + if by_number is None: + raise AssertionError("read by number although the search listed it") + if isinstance(by_number, Exception): + raise by_number + return by_number + if "q" not in variables: + return {f"pr{key[1:]}": {"object": None} for key in variables if key.startswith("s")} + if "updated:>=" in variables["q"]: + return _search_result(NEWLY_GATED) + if variables["q"] == f'repo:{REPO} is:pr is:open label:"fork ci required"': + return _search_result(*listed) + return _search_result() + + return graphql + + +def _rest_get_for_pull_request_1(path, paginate=False, items_key=None): + if path.endswith("/actions/workflows/ci-amd.yml"): + return {"state": "active"} + if path == f"repos/{REPO}/pulls/1/files?per_page=100": + return [] + if path == f"repos/{REPO}/pulls/1": + return {"head": {"sha": "sha1"}, "base": {"ref": "main"}} + raise AssertionError(path) + + +@pytest.mark.parametrize( + ("listed", "by_number", "commented"), + [ + pytest.param( + [], {"repository": {"pr0": {**_pr_node(1), "state": "OPEN"}}}, [1], id="not-yet-listed-by-search" + ), + pytest.param([_pr_node(1)], None, [1], id="listed-by-search"), + pytest.param( + [], {"repository": {"pr0": {**_pr_node(1), "state": "CLOSED"}}}, [], id="closed-since-labelled" + ), + pytest.param([], {"repository": {"pr0": None}}, [], id="deleted-since-labelled"), + pytest.param([], {"repository": None}, [], id="repository-not-resolved"), + ], +) +def test_run_reconcile_reconciles_pull_requests_labelled_in_the_same_run( + client, tmp_path, listed, by_number, commented +): + client.graphql.side_effect = _graphql_after_gating_pull_request_1(listed=listed, by_number=by_number) + client.rest_get_pages.return_value = [{"total_count": 0, "workflow_runs": []}] + client.rest_get.side_effect = _rest_get_for_pull_request_1 + + has_alerts = run_reconcile( + client, + CONFIG, + now=NOW, + run_url="https://run", + output_file=tmp_path / "s.json", + workflows_dir=tmp_path, + ) + + assert has_alerts is False + client.add_labels.assert_called_once_with(1, ["fork ci required"]) + assert [call.args[0] for call in client.create_comment.call_args_list] == commented + + +def test_run_reconcile_leaves_pull_requests_it_cannot_read_for_the_next_run(client, tmp_path): + client.graphql.side_effect = _graphql_after_gating_pull_request_1( + listed=[], by_number=GitHubError("gh api failed: HTTP 502") + ) + client.rest_get_pages.return_value = [{"total_count": 0, "workflow_runs": []}] + client.rest_get.side_effect = _rest_get_for_pull_request_1 + + has_alerts = run_reconcile( + client, + CONFIG, + now=NOW, + run_url="https://run", + output_file=tmp_path / "s.json", + workflows_dir=tmp_path, + ) + + assert has_alerts is False + client.create_comment.assert_not_called() + + +@mock.patch("airflow_breeze.utils.fork_ci_reconcile.PASS_TWO_PAGE_SIZE", 2) +def test_fetch_open_pull_requests_reads_in_batches(client): + client.graphql.side_effect = [ + {"repository": {"pr0": {"number": 1, "state": "OPEN"}, "pr1": {"number": 2, "state": "OPEN"}}}, + {"repository": {"pr0": {"number": 3, "state": "OPEN"}}}, + ] + + nodes = fetch_open_pull_requests(client, [1, 2, 3], repo=REPO) + + assert [node["number"] for node in nodes] == [1, 2, 3] + queries = [call.args[0] for call in client.graphql.call_args_list] + assert "pr1: pullRequest(number: 2)" in queries[0] + assert "pr0: pullRequest(number: 3)" in queries[1] + assert "number: 3" not in queries[0] + + def test_run_reconcile_retries_a_pull_request_whose_head_moved(client, tmp_path): output = tmp_path / "slack-message.json" client.graphql.side_effect = [ @@ -3496,7 +3861,10 @@ def rest_get(path, paginate=False, items_key=None): @pytest.mark.parametrize( ("fails", "job_status_budget"), - [pytest.param(False, 38, id="reported"), pytest.param(True, 20, id="fails")], + [ + pytest.param(False, MAX_JOB_STATUSES_PER_RUN - 2, id="reported"), + pytest.param(True, MAX_JOB_STATUSES_PER_RUN - MAX_CI_RUN_STARTED_PER_RUN, id="fails"), + ], ) @mock.patch("airflow_breeze.utils.fork_ci_reconcile.JobStatusReporter", autospec=True) @mock.patch("airflow_breeze.utils.fork_ci_reconcile.report_project_ci_runs_started", autospec=True) @@ -3533,7 +3901,7 @@ def rest_get(path, paginate=False, items_key=None): assert mock_classify.call_args.args[0].ci_run_at == datetime(2026, 11, 2, tzinfo=UTC) mock_report.assert_called_once_with(client, repo=REPO, skip={1}) - mock_reporter.assert_called_once_with(client, repo=REPO, budget=job_status_budget, now=NOW) + mock_reporter.assert_called_once_with(client, repo=REPO, budget=job_status_budget, now=NOW, read_jobs={}) assert has_alerts is fails if fails: assert "reporting `CI run started` for project CI failed: GitHubError" in output.read_text() @@ -3614,13 +3982,13 @@ def test_run_reconcile_reports_rate_limiting_only_when_it_persists( reported_file = tmp_path / "reported.json" reported_file.write_text("{}") alerts = [] - for rate_limited in (1, 0, 3, 2): + for rate_limited in (1, 0, *[3] * RATE_LIMITED_RUNS_BEFORE_ALERT): client.rate_limited = rate_limited alerts.append(_reconcile_with_reporter(client, tmp_path, reported_file=reported_file)) - assert alerts == [False, False, False, True] + assert alerts == [*[False] * (RATE_LIMITED_RUNS_BEFORE_ALERT + 1), True] assert RATE_LIMITED_NOTE in (tmp_path / "s.json").read_text() - assert json.loads(reported_file.read_text())["rate_limited_runs"] == 2 + assert json.loads(reported_file.read_text())["rate_limited_runs"] == RATE_LIMITED_RUNS_BEFORE_ALERT @pytest.mark.parametrize( @@ -3711,7 +4079,7 @@ def _passed_suite( ("node", "expected"), [ pytest.param( - _passed_node(state="SUCCESS", description=CI_PASSED_DESCRIPTION), + _passed_node(state="SUCCESS", description=FORK_CI_PASSED_DESCRIPTION), WAITING, id="fork-success-revoked", ), @@ -3721,7 +4089,9 @@ def _passed_suite( ), pytest.param( _passed_node( - state="SUCCESS", description=CI_PASSED_DESCRIPTION, suites=[_passed_suite(5, cancelled=1)] + state="SUCCESS", + description=FORK_CI_PASSED_DESCRIPTION, + suites=[_passed_suite(5, cancelled=1)], ), PASSED, id="rerun-passed-next-to-the-marker-of-its-first-attempt", @@ -3729,7 +4099,7 @@ def _passed_suite( pytest.param( _passed_node( state="SUCCESS", - description=CI_PASSED_DESCRIPTION, + description=FORK_CI_PASSED_DESCRIPTION, suites=[ _passed_suite(4, created="2026-11-10T09:00:00Z"), _passed_suite(6, conclusion=None, running=True), @@ -3741,7 +4111,7 @@ def _passed_suite( pytest.param( _passed_node( state="SUCCESS", - description=CI_PASSED_DESCRIPTION, + description=FORK_CI_PASSED_DESCRIPTION, suites=[ _passed_suite(5, created="2026-11-10T09:00:00Z"), _passed_suite(6, conclusion=None, suite_cancelled=True), @@ -3753,7 +4123,7 @@ def _passed_suite( pytest.param( _passed_node( state="SUCCESS", - description=CI_PASSED_DESCRIPTION, + description=FORK_CI_PASSED_DESCRIPTION, suites=[ _passed_suite(5, created="2026-11-10T09:00:00Z"), _passed_suite(6, cancelled=1, conclusion=None), @@ -3765,7 +4135,9 @@ def _passed_suite( pytest.param(_passed_node(state="PENDING", description="x"), None, id="pending-left-alone"), pytest.param( _passed_node( - state="SUCCESS", description=CI_PASSED_DESCRIPTION, suites=[_passed_suite(5, conclusion=None)] + state="SUCCESS", + description=FORK_CI_PASSED_DESCRIPTION, + suites=[_passed_suite(5, conclusion=None)], ), WAITING, id="project-ci-still-running", @@ -3773,7 +4145,7 @@ def _passed_suite( pytest.param( _passed_node( state="SUCCESS", - description=CI_PASSED_DESCRIPTION, + description=FORK_CI_PASSED_DESCRIPTION, suites=[_passed_suite(5, cancelled=1, conclusion=None)], ), WAITING, @@ -3782,7 +4154,7 @@ def _passed_suite( pytest.param( _passed_node( state="SUCCESS", - description=CI_PASSED_DESCRIPTION, + description=FORK_CI_PASSED_DESCRIPTION, suites=[_passed_suite(5, not_run=1, conclusion=None)], ), WAITING, @@ -3791,7 +4163,7 @@ def _passed_suite( pytest.param( _passed_node( state="SUCCESS", - description=CI_PASSED_DESCRIPTION, + description=FORK_CI_PASSED_DESCRIPTION, suites=[_passed_suite(5, event="push"), _passed_suite(6, path=".github/workflows/other.yml")], ), WAITING, @@ -3800,7 +4172,7 @@ def _passed_suite( pytest.param( _passed_node( state="SUCCESS", - description=CI_PASSED_DESCRIPTION, + description=FORK_CI_PASSED_DESCRIPTION, suites=[ _passed_suite(4, created="2026-11-10T09:00:00Z"), _passed_suite(5, conclusion="FAILURE"), @@ -3810,7 +4182,7 @@ def _passed_suite( id="latest-run-failed", ), pytest.param( - _passed_node(state="SUCCESS", description=CI_PASSED_DESCRIPTION, suites=[_passed_suite(5)]), + _passed_node(state="SUCCESS", description=FORK_CI_PASSED_DESCRIPTION, suites=[_passed_suite(5)]), PASSED, id="project-ci-passed", ), @@ -3851,7 +4223,7 @@ def test_report_project_ci_passed(client, node, expected): def test_report_project_ci_passed_limits_successes_but_not_revocations(client): client.graphql.return_value = _search_result( *(_passed_node(n, state="PENDING", description="x", suites=[_passed_suite(5)]) for n in (1, 2, 3)), - _passed_node(4, state="SUCCESS", description=CI_PASSED_DESCRIPTION), + _passed_node(4, state="SUCCESS", description=FORK_CI_PASSED_DESCRIPTION), ) assert report_project_ci_passed(client, repo=REPO, attention=[], budget=2) == [1, 2, 4] @@ -3860,7 +4232,7 @@ def test_report_project_ci_passed_limits_successes_but_not_revocations(client): def test_report_project_ci_passed_keeps_going_after_an_error(client): client.graphql.return_value = _search_result( - *(_passed_node(n, state="SUCCESS", description=CI_PASSED_DESCRIPTION) for n in (1, 2)) + *(_passed_node(n, state="SUCCESS", description=FORK_CI_PASSED_DESCRIPTION) for n in (1, 2)) ) client.set_status.side_effect = [GitHubError("gh api failed: HTTP 502"), None] attention: list[tuple[int, str]] = [] @@ -3955,10 +4327,10 @@ def test_report_project_ci_passed_reads_a_head_commit_the_search_did_not_list( ): mock_read.return_value = PROJECT_PASSED_URL if passed else None client.graphql.return_value = _search_result( - _passed_node(state="SUCCESS", description=CI_PASSED_DESCRIPTION, oid="older") + _passed_node(state="SUCCESS", description=FORK_CI_PASSED_DESCRIPTION, oid="older") ) client.rest_get.return_value = [ - {"context": CI_PASSED_CONTEXT, "state": "success", "description": CI_PASSED_DESCRIPTION} + {"context": CI_PASSED_CONTEXT, "state": "success", "description": FORK_CI_PASSED_DESCRIPTION} ] report_project_ci_passed(client, repo=REPO, attention=[]) @@ -3982,7 +4354,7 @@ def _run_reconcile_with_green_pull_requests( *numbers, live=LIVE_PULL_REQUEST, same_branch=(), - ci_passed=("success", RUN_URL, CI_PASSED_DESCRIPTION), + ci_passed=("success", RUN_URL, FORK_CI_PASSED_DESCRIPTION), state=ForkState.GREEN, ): """Reconcile gated, fork-green pull requests that all have head commit ``shared``.""" @@ -4123,7 +4495,7 @@ def test_report_project_ci_passed_revokes_a_pull_request_the_heads_search_missed _search_result(), _search_result(), _search_result(), - _search_result(_passed_node(state="SUCCESS", description=CI_PASSED_DESCRIPTION)), + _search_result(_passed_node(state="SUCCESS", description=FORK_CI_PASSED_DESCRIPTION)), ] client.rest_get_pages.return_value = [{"total_count": 0, "workflow_runs": []}] @@ -4179,7 +4551,7 @@ def test_run_reconcile_reports_project_ci_passed_after_the_loop( names = [c[0] for c in order.mock_calls] assert names.index("report") > names.index("set_status") - assert mock_report_project_ci_passed.call_args.kwargs["budget"] == 5 + assert mock_report_project_ci_passed.call_args.kwargs["budget"] == MAX_PROJECT_CI_PASSED_PER_RUN def test_run_reconcile_reports_project_ci_passed_failure(mock_report_project_ci_passed, client, tmp_path): @@ -4213,3 +4585,176 @@ def test_report_project_ci_passed_counts_only_the_successes_that_were_written(cl client.set_status.side_effect = [GitHubError("HTTP 502"), None, None] assert report_project_ci_passed(client, repo=REPO, attention=[], budget=1) == [2] + + +def test_find_failed_jobs_lists_finished_jobs_that_did_not_pass(): + jobs = [ + _fork_job("Static checks"), + _fork_job("Postgres tests", conclusion="failure"), + _fork_job("Helm tests", status="in_progress", conclusion=None), + _fork_job("Docs", conclusion="timed_out"), + _fork_job("Java SDK tests", conclusion="skipped"), + _fork_job("Platform: AMD", conclusion="failure"), + _fork_job("Postgres tests", conclusion="cancelled", url="https://job/second"), + _fork_job("MySQL tests", conclusion="cancelled", url="https://job/cancelled"), + _fork_job("MySQL tests", conclusion="failure"), + ] + + assert find_failed_jobs(jobs) == [ + FailedJob("MySQL tests", "https://job/MySQL tests", "failure"), + FailedJob("Postgres tests", "https://job/Postgres tests", "failure"), + FailedJob("Docs", "https://job/Docs", "timed_out"), + ] + + +@pytest.mark.parametrize( + ("read_jobs", "side_effect", "expected", "reads"), + [ + pytest.param({1: [_fork_job("A", conclusion="failure")]}, None, ["A"], False, id="read-this-run"), + pytest.param({}, [[_fork_job("B", conclusion="failure")]], ["B"], True, id="not-read-yet"), + pytest.param({}, GitHubError("HTTP 502"), None, True, id="unreadable"), + pytest.param({1: None}, None, None, False, id="read-failed-this-run"), + ], +) +def test_read_failed_jobs(client, read_jobs, side_effect, expected, reads): + client.get_run_jobs.side_effect = side_effect + + failed = read_failed_jobs(client, _pr(), COMPLETED_RUN, read_jobs) + + assert (None if failed is None else [job.name for job in failed]) == expected + assert client.get_run_jobs.called is reads + assert 1 in read_jobs + + +def test_a_failed_jobs_read_is_not_repeated_for_the_state_comment(client): + client.rest_get.return_value = [] + client.get_run_jobs.side_effect = GitHubError("HTTP 502") + read_jobs: dict = {} + reporter = JobStatusReporter(client, repo=REPO, budget=10, read_jobs=read_jobs) + + with pytest.raises(GitHubError): + reporter.report(_pr(), COMPLETED_RUN) + + assert read_failed_jobs(client, _pr(), COMPLETED_RUN, read_jobs) is None + client.get_run_jobs.assert_called_once_with("someone/airflow", 1) + + +def test_job_status_reporter_keeps_the_jobs_it_read(client): + jobs = [_fork_job("Static checks", conclusion="failure")] + reporter = JobStatusReporter(_reporter_client(client, jobs), repo=REPO, budget=10) + + reporter.report(_pr(), COMPLETED_RUN) + + assert reporter.read_jobs == {1: jobs} + + +FAILED_JOB = FailedJob("Static checks", "https://job/1", "failure") + + +@pytest.mark.parametrize( + ("fork", "failed_jobs", "called", "listed"), + [ + pytest.param(COMPLETED_RUN, [FAILED_JOB], True, True, id="failed"), + pytest.param( + dataclasses.replace(COMPLETED_RUN, suite_conclusion="STARTUP_FAILURE"), + [FAILED_JOB], + False, + False, + id="startup-failure", + ), + pytest.param(RUNNING_RUN, [FAILED_JOB], False, False, id="running"), + ], +) +def test_state_comment_lists_the_jobs_of_a_failed_run(fork, failed_jobs, called, listed): + find_failed = mock.Mock(return_value=failed_jobs) + + plan = classify( + _pr(), + fork, + config=CONFIG, + now=NOW, + repo=REPO, + find_bypass_run=lambda: None, + changes_ci_files=lambda: False, + find_failed_jobs=find_failed, + ) + + assert find_failed.called is called + assert ("- [`Static checks`](https://job/1): failed" in plan.marker_comment) is listed + + +@pytest.mark.parametrize( + ("marker", "kept"), + [ + pytest.param(_marker("failed"), True, id="failed-this-head"), + pytest.param(_marker("failed", sha="older"), False, id="failed-earlier-head"), + pytest.param(_marker("running"), False, id="running"), + pytest.param(None, False, id="no-comment"), + ], +) +def test_state_comment_is_left_for_the_next_run_only_when_it_shows_the_head_failed(marker, kept): + plan = classify( + _pr(is_draft=False, labels=("fork ci required", "ready for maintainer review"), marker=marker), + COMPLETED_RUN, + config=CONFIG, + now=NOW, + repo=REPO, + find_bypass_run=lambda: None, + changes_ci_files=lambda: False, + find_failed_jobs=lambda: None, + ) + + assert plan.state == ForkState.FAILED + assert plan.status == ("failure", RUN_URL, "Fork CI failed - see Details") + assert plan.remove_labels == ["ready for maintainer review"] + if kept: + assert plan.marker_comment is None + else: + assert parse_marker(plan.marker_comment).state == "failed" + assert parse_marker(plan.marker_comment).failed_run is None + assert "Jobs that did not pass" not in plan.marker_comment + + +def _classify_failed(pr, fork=COMPLETED_RUN, find_failed=None): + return classify( + pr, + fork, + config=CONFIG, + now=NOW, + repo=REPO, + find_bypass_run=lambda: None, + changes_ci_files=lambda: False, + find_failed_jobs=find_failed or (lambda: [FAILED_JOB]), + ) + + +@pytest.mark.parametrize( + ("fork", "marker_overrides", "reads"), + [ + pytest.param(COMPLETED_RUN, {}, False, id="unchanged"), + pytest.param(dataclasses.replace(COMPLETED_RUN, run_attempt=2), {}, True, id="re-run"), + pytest.param(COMPLETED_RUN, {"failed_run": None}, True, id="not-listed-yet"), + ], +) +def test_jobs_of_a_listed_failed_run_are_not_read_again(fork, marker_overrides, reads): + pr = _pr(is_draft=False, labels=("fork ci required",)) + listed = _classify_failed(pr).marker_comment + marker = dataclasses.replace(parse_marker(listed), **marker_overrides) + find_failed = mock.Mock(return_value=[FAILED_JOB]) + + plan = _classify_failed( + dataclasses.replace(pr, marker=marker, marker_comment_body=listed), fork, find_failed + ) + + assert parse_marker(listed).failed_run == "1:1" + assert find_failed.called is reads + if not reads: + assert plan.marker_comment is None + + +def test_monitor_interval_is_the_monitor_workflow_cadence(): + workflow = yaml.safe_load((AIRFLOW_ROOT_PATH / ".github/workflows/fork-ci-monitor.yml").read_text()) + + crons = [entry["cron"] for entry in workflow[True]["schedule"]] + + assert crons == [f"*/{MONITOR_INTERVAL // timedelta(minutes=1)} * * * *"] diff --git a/scripts/ci/prek/check_fork_ci_gate_config.py b/scripts/ci/prek/check_fork_ci_gate_config.py index 2bb9401641f05..ab1c8a1713a3e 100755 --- a/scripts/ci/prek/check_fork_ci_gate_config.py +++ b/scripts/ci/prek/check_fork_ci_gate_config.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. """ -Validate ``.github/fork-ci-gate.toml`` (AIP-120 fork CI gate). +Validate ``.github/fork-ci-gate.toml`` (AIP-120 fork CI gate) and ``.asf.yaml``. The file must parse with the same rules the gate job uses, and every ``[pilot]`` join instant and ``enabled_since`` value that is new or changed relative to ``main`` must lie in the future. @@ -24,6 +24,9 @@ while the instant is compared with pull request creation: an instant before the merge would gate pull requests that already ran project CI. See "Contributor pilot" in dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md. + +Additionally, ``.asf.yaml``'s ``github.pull_requests.creation_cap.bypass_users`` must stay strictly in sync +with ``[pilot]`` participants in ``.github/fork-ci-gate.toml``. """ from __future__ import annotations @@ -32,12 +35,16 @@ import os import subprocess import sys +import tomllib +from collections.abc import Iterable from datetime import UTC, datetime from types import ModuleType +import yaml from common_prek_utils import AIRFLOW_ROOT_PATH CONFIG_PATH = ".github/fork-ci-gate.toml" +ASF_YAML_PATH = ".asf.yaml" UPSTREAM_URL = "https://github.com/apache/airflow.git" GATE_MODULE_PATH = ( AIRFLOW_ROOT_PATH / "dev" / "breeze" / "src" / "airflow_breeze" / "utils" / "fork_ci_gate.py" @@ -97,6 +104,68 @@ def find_violations(base_text: str | None, new_text: str, *, now: datetime) -> l return violations +def find_asf_yaml_sync_violations(asf_yaml_text: str, pilot_users: Iterable[str]) -> list[str]: + try: + data = yaml.safe_load(asf_yaml_text) + except yaml.YAMLError as e: + return [f"failed to parse YAML: {e}"] + if not isinstance(data, dict): + return ["root must be a YAML mapping"] + + github = data.get("github") + if not isinstance(github, dict): + return ["github must be a mapping"] + pull_requests = github.get("pull_requests") + if not isinstance(pull_requests, dict): + return ["github.pull_requests must be a mapping"] + creation_cap = pull_requests.get("creation_cap") + if not isinstance(creation_cap, dict): + return ["github.pull_requests.creation_cap must be a mapping"] + + bypass_users = creation_cap.get("bypass_users") + if bypass_users is None: + return ["github.pull_requests.creation_cap.bypass_users is missing"] + if not isinstance(bypass_users, list): + return ["github.pull_requests.creation_cap.bypass_users must be a list"] + if any(not isinstance(u, str) for u in bypass_users): + return ["all entries in bypass_users must be strings"] + + violations = [] + seen_casefolded: set[str] = set() + duplicates: list[str] = [] + seen_duplicates: set[str] = set() + for u in bypass_users: + folded = u.casefold() + if folded in seen_casefolded and folded not in seen_duplicates: + duplicates.append(u) + seen_duplicates.add(folded) + seen_casefolded.add(folded) + if duplicates: + violations.append(f"duplicate users in bypass_users: {', '.join(sorted(duplicates))}") + + if bypass_users != sorted(bypass_users, key=str.casefold): + violations.append("bypass_users must be sorted alphabetically (case-insensitive)") + + asf_map = {u.casefold(): u for u in bypass_users} + pilot_map = {u.casefold(): u for u in pilot_users} + missing = [pilot_map[k] for k in sorted(set(pilot_map) - set(asf_map))] + extra = [asf_map[k] for k in sorted(set(asf_map) - set(pilot_map))] + if missing: + violations.append(f"missing pilot users in bypass_users: {', '.join(missing)}") + if extra: + violations.append(f"extra users in bypass_users not in [pilot]: {', '.join(extra)}") + + casing = [ + f"{asf_map[k]} in .asf.yaml vs {pilot_map[k]} in [pilot]" + for k in sorted(set(asf_map) & set(pilot_map)) + if asf_map[k] != pilot_map[k] + ] + if casing: + violations.append(f"casing mismatch for pilot users: {', '.join(casing)}") + + return violations + + def _git(*args: str) -> str | None: result = subprocess.run( ["git", "-C", str(AIRFLOW_ROOT_PATH), *args], capture_output=True, text=True, check=False @@ -142,7 +211,25 @@ def main() -> int: violations = find_violations(base_text, new_text, now=datetime.now(tz=UTC)) for violation in violations: print(f"{CONFIG_PATH}: {violation}", file=sys.stderr) - return 1 if violations else 0 + + asf_yaml_path = AIRFLOW_ROOT_PATH / ASF_YAML_PATH + if not asf_yaml_path.is_file(): + print(f"{ASF_YAML_PATH}: file does not exist", file=sys.stderr) + return 1 + + try: + load_gate_module().parse_config(new_text) + except ValueError: + # find_violations already reported the parse error. + return 1 + # Read the logins with tomllib rather than the gate's parser, which casefolds them, so a + # casing difference between the two files is still reported. + pilot_users = list(tomllib.loads(new_text).get("pilot", {})) + asf_violations = find_asf_yaml_sync_violations(asf_yaml_path.read_text(), pilot_users) + for violation in asf_violations: + print(f"{ASF_YAML_PATH}: {violation}", file=sys.stderr) + + return 1 if (violations or asf_violations) else 0 if __name__ == "__main__": diff --git a/scripts/ci/prek/check_scheduled_workflows_skip_forks.py b/scripts/ci/prek/check_scheduled_workflows_skip_forks.py new file mode 100755 index 0000000000000..aed634c146a72 --- /dev/null +++ b/scripts/ci/prek/check_scheduled_workflows_skip_forks.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.11,<3.12" +# dependencies = [ +# "pyyaml>=6.0.3", +# "rich>=13.6.0", +# ] +# /// +"""Check that every job in a scheduled workflow skips its cron runs in forks. + +Forks that enable Actions for fork CI also get every cron. A job passes when its ``if:`` has the +``apache/airflow`` guard, or when it needs a job that passes and does not override the skip with +``always()``, ``cancelled()`` or ``failure()`` - unless its ``if:`` still requires such a job to +succeed with ``needs..result == 'success'``. ``ci-amd.yml`` and ``ci-arm.yml`` are exempt: their +``check-run-needed`` job skips scheduled runs in forks, and ``ci-amd.yml`` must still run there on push. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path +from typing import Any + +import yaml +from rich.console import Console + +console = Console(color_system="standard", width=200) + +EXEMPTIONS = frozenset({"ci-amd.yml", "ci-arm.yml"}) +GUARD = "github.repository == 'apache/airflow'" +STATUS_CHECK_FUNCTIONS = frozenset({"always()", "cancelled()", "failure()"}) +REQUIRED_SUCCESS = re.compile(r"needs\.([\w-]+)\.result\s*==\s*'success'") + + +def is_covered(jobs: dict[str, Any], job_name: str) -> bool: + """Whether the job skips scheduled runs in forks, through its own guard or a job it needs.""" + job = jobs[job_name] + if_condition = job.get("if") or "" + if GUARD in if_condition: + return True + needs = job.get("needs") or [] + if isinstance(needs, str): + needs = [needs] + if any(status_check in if_condition for status_check in STATUS_CHECK_FUNCTIONS): + # A status function runs the job even when its needs were skipped, unless the condition + # still requires one of them to have succeeded. + required = set(REQUIRED_SUCCESS.findall(if_condition)) + return any(is_covered(jobs, parent) for parent in needs if parent in required) + return any(is_covered(jobs, parent) for parent in needs) + + +def check_file(file_path: Path) -> list[str]: + """Return the jobs of a scheduled workflow that would still run on schedule in forks.""" + if file_path.name in EXEMPTIONS: + return [] + workflow = yaml.safe_load(file_path.read_text()) + # PyYAML reads the bare `on:` key as boolean True. + triggers = workflow.get("on", workflow.get(True)) + if not (isinstance(triggers, dict) and "schedule" in triggers): + return [] + jobs = workflow["jobs"] + return [job_name for job_name in jobs if not is_covered(jobs, job_name)] + + +def main(file_paths: list[str]) -> int: + errors = [] + for file_path in file_paths: + path = Path(file_path) + for job_name in check_file(path): + errors.append( + f"[red]{path}[/]: job [yellow]{job_name}[/] runs on schedule in forks. " + f"Add `if: {GUARD}` to it, or make it need a job that has it." + ) + if not errors: + return 0 + for error in errors: + console.print(error) + console.print( + "\nWorkflows with other triggers should use " + f"`if: github.event_name != 'schedule' || {GUARD}`\n" + "so manual, push and pull request runs still work in forks." + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/tests/ci/prek/test_check_fork_ci_gate_config.py b/scripts/tests/ci/prek/test_check_fork_ci_gate_config.py index 73a85340f8948..0e825df219d20 100644 --- a/scripts/tests/ci/prek/test_check_fork_ci_gate_config.py +++ b/scripts/tests/ci/prek/test_check_fork_ci_gate_config.py @@ -20,10 +20,19 @@ from unittest import mock import pytest -from check_fork_ci_gate_config import find_violations, main, resolve_base_ref +from check_fork_ci_gate_config import find_asf_yaml_sync_violations, find_violations, main, resolve_base_ref NOW = datetime(2026, 10, 15, 12, 0, tzinfo=UTC) BASE = 'enabled_since = 2026-01-01T00:00:00Z\n[pilot]\n"old-user" = 2026-09-01T00:00:00Z\n' +VALID_ASF_YAML = """ +github: + pull_requests: + creation_cap: + bypass_users: + - alice + - Bob + - charlie +""" @pytest.mark.parametrize( @@ -113,3 +122,207 @@ def test_resolve_base_ref(mock_git, monkeypatch, base_ref_env, git_results, expe def test_main_fails_without_a_base(mock_resolve, capsys): assert main() == 1 assert "no base revision to compare with" in capsys.readouterr().err + + +@pytest.mark.parametrize( + ("asf_yaml", "pilot_users"), + [ + pytest.param(VALID_ASF_YAML, ["alice", "Bob", "charlie"], id="exact-sync"), + pytest.param(VALID_ASF_YAML, ["charlie", "alice", "Bob"], id="unordered-pilot-set-matches"), + pytest.param( + """ +github: + pull_requests: + creation_cap: + bypass_users: [] +""", + [], + id="empty-lists-in-sync", + ), + ], +) +def test_asf_yaml_sync_accepts(asf_yaml, pilot_users): + assert find_asf_yaml_sync_violations(asf_yaml, pilot_users) == [] + + +@pytest.mark.parametrize( + ("asf_yaml", "pilot_users", "expected_message"), + [ + pytest.param( + VALID_ASF_YAML, + ["alice", "Bob", "charlie", "david"], + "missing pilot users in bypass_users: david", + id="pilot-missing-in-asf-yaml", + ), + pytest.param( + VALID_ASF_YAML, + ["alice", "Bob"], + "extra users in bypass_users not in [pilot]: charlie", + id="extra-user-in-asf-yaml", + ), + pytest.param( + """ +github: + pull_requests: + creation_cap: + bypass_users: + - charlie + - alice +""", + ["alice", "charlie"], + "bypass_users must be sorted alphabetically (case-insensitive)", + id="unsorted-bypass-users", + ), + pytest.param( + """ +github: + pull_requests: + creation_cap: + bypass_users: + - alice + - alice +""", + ["alice"], + "duplicate users in bypass_users: alice", + id="duplicate-bypass-user", + ), + pytest.param( + """ +github: + pull_requests: + creation_cap: + bypass_users: + - Alice + - alice +""", + ["Alice"], + "duplicate users in bypass_users: alice", + id="duplicate-bypass-user-case-insensitive", + ), + pytest.param( + "not: [valid: yaml", + ["alice"], + "failed to parse YAML", + id="malformed-yaml", + ), + pytest.param( + "42", + ["alice"], + "root must be a YAML mapping", + id="root-not-mapping", + ), + pytest.param( + "github: null", + ["alice"], + "github must be a mapping", + id="github-null", + ), + pytest.param( + "github:\n pull_requests: null", + ["alice"], + "github.pull_requests must be a mapping", + id="pull-requests-null", + ), + pytest.param( + "github:\n pull_requests:\n creation_cap: null", + ["alice"], + "github.pull_requests.creation_cap must be a mapping", + id="creation-cap-null", + ), + pytest.param( + "github:\n pull_requests:\n creation_cap: {}", + ["alice"], + "github.pull_requests.creation_cap.bypass_users is missing", + id="bypass-users-missing", + ), + pytest.param( + "github:\n pull_requests:\n creation_cap:\n bypass_users: 'alice'", + ["alice"], + "github.pull_requests.creation_cap.bypass_users must be a list", + id="bypass-users-not-list", + ), + pytest.param( + "github:\n pull_requests:\n creation_cap:\n bypass_users:\n - 123", + ["123"], + "all entries in bypass_users must be strings", + id="bypass-users-non-string-entries", + ), + pytest.param( + """ +github: + pull_requests: + creation_cap: + bypass_users: + - Alice +""", + ["alice"], + "casing mismatch for pilot users: Alice in .asf.yaml vs alice in [pilot]", + id="casing-mismatch", + ), + ], +) +def test_asf_yaml_sync_rejects(asf_yaml, pilot_users, expected_message): + violations = find_asf_yaml_sync_violations(asf_yaml, pilot_users) + assert any(expected_message in v for v in violations) + + +@mock.patch("check_fork_ci_gate_config.resolve_base_ref", autospec=True, return_value="HEAD") +@mock.patch("check_fork_ci_gate_config._git", autospec=True) +def test_main_passes_when_gate_and_asf_yaml_are_in_sync(mock_git, mock_resolve, tmp_path, monkeypatch): + monkeypatch.setattr("check_fork_ci_gate_config.AIRFLOW_ROOT_PATH", tmp_path) + mock_git.return_value = 'enabled_since = 2026-01-01T00:00:00Z\n[pilot]\n"alice" = 2026-01-01T00:00:00Z\n' + (tmp_path / ".github").mkdir(parents=True) + (tmp_path / ".github" / "fork-ci-gate.toml").write_text( + 'enabled_since = 2026-01-01T00:00:00Z\n[pilot]\n"alice" = 2026-01-01T00:00:00Z\n' + ) + (tmp_path / ".asf.yaml").write_text( + "github:\n pull_requests:\n creation_cap:\n bypass_users:\n - alice\n" + ) + assert main() == 0 + + +@mock.patch("check_fork_ci_gate_config.resolve_base_ref", autospec=True, return_value="HEAD") +@mock.patch("check_fork_ci_gate_config._git", autospec=True) +def test_main_fails_when_asf_yaml_is_out_of_sync(mock_git, mock_resolve, tmp_path, monkeypatch, capsys): + monkeypatch.setattr("check_fork_ci_gate_config.AIRFLOW_ROOT_PATH", tmp_path) + mock_git.return_value = 'enabled_since = 2026-01-01T00:00:00Z\n[pilot]\n"alice" = 2026-01-01T00:00:00Z\n' + (tmp_path / ".github").mkdir(parents=True) + (tmp_path / ".github" / "fork-ci-gate.toml").write_text( + 'enabled_since = 2026-01-01T00:00:00Z\n[pilot]\n"alice" = 2026-01-01T00:00:00Z\n' + ) + (tmp_path / ".asf.yaml").write_text( + "github:\n pull_requests:\n creation_cap:\n bypass_users:\n - bob\n" + ) + assert main() == 1 + err = capsys.readouterr().err + assert ".asf.yaml: missing pilot users in bypass_users: alice" in err + assert ".asf.yaml: extra users in bypass_users not in [pilot]: bob" in err + + +@mock.patch("check_fork_ci_gate_config.resolve_base_ref", autospec=True, return_value="HEAD") +@mock.patch("check_fork_ci_gate_config._git", autospec=True) +def test_main_fails_when_asf_yaml_is_missing(mock_git, mock_resolve, tmp_path, monkeypatch, capsys): + monkeypatch.setattr("check_fork_ci_gate_config.AIRFLOW_ROOT_PATH", tmp_path) + mock_git.return_value = 'enabled_since = 2026-01-01T00:00:00Z\n[pilot]\n"alice" = 2026-01-01T00:00:00Z\n' + (tmp_path / ".github").mkdir(parents=True) + (tmp_path / ".github" / "fork-ci-gate.toml").write_text( + 'enabled_since = 2026-01-01T00:00:00Z\n[pilot]\n"alice" = 2026-01-01T00:00:00Z\n' + ) + assert main() == 1 + assert ".asf.yaml: file does not exist" in capsys.readouterr().err + + +@mock.patch("check_fork_ci_gate_config.resolve_base_ref", autospec=True, return_value="HEAD") +@mock.patch("check_fork_ci_gate_config._git", autospec=True) +def test_main_reports_a_pilot_that_is_not_a_table(mock_git, mock_resolve, tmp_path, monkeypatch, capsys): + monkeypatch.setattr("check_fork_ci_gate_config.AIRFLOW_ROOT_PATH", tmp_path) + mock_git.return_value = None + (tmp_path / ".github").mkdir(parents=True) + (tmp_path / ".github" / "fork-ci-gate.toml").write_text('pilot = ["alice"]\n') + (tmp_path / ".asf.yaml").write_text( + "github:\n pull_requests:\n creation_cap:\n bypass_users:\n - alice\n" + ) + assert main() == 1 + err = capsys.readouterr().err + assert ".github/fork-ci-gate.toml: pilot must be a table" in err + assert ".asf.yaml" not in err diff --git a/scripts/tests/ci/prek/test_check_scheduled_workflows_skip_forks.py b/scripts/tests/ci/prek/test_check_scheduled_workflows_skip_forks.py new file mode 100644 index 0000000000000..131bd2b6950e5 --- /dev/null +++ b/scripts/tests/ci/prek/test_check_scheduled_workflows_skip_forks.py @@ -0,0 +1,138 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest +import yaml +from check_scheduled_workflows_skip_forks import GUARD, check_file, is_covered + +SCHEDULE_ONLY = {"schedule": [{"cron": "0 0 * * *"}]} + + +@pytest.mark.parametrize( + ("jobs", "job_name", "expected"), + [ + pytest.param({"job": {"if": GUARD}}, "job", True, id="own-guard"), + pytest.param({"job": {}}, "job", False, id="no-guard-no-needs"), + pytest.param( + {"parent": {"if": GUARD}, "job": {"needs": "parent"}}, "job", True, id="needs-as-string" + ), + pytest.param( + {"root": {"if": GUARD}, "middle": {"needs": "root"}, "job": {"needs": "middle"}}, + "job", + True, + id="chain", + ), + pytest.param( + {"first": {}, "second": {"if": GUARD}, "job": {"needs": ["first", "second"]}}, + "job", + True, + id="one-of-several-parents-covered", + ), + pytest.param( + {"first": {}, "second": {}, "job": {"needs": ["first", "second"]}}, + "job", + False, + id="no-parent-covered", + ), + pytest.param( + {"parent": {"if": GUARD}, "job": {"needs": "parent", "if": "!cancelled()"}}, + "job", + False, + id="status-function-overrides-covered-parent", + ), + pytest.param( + {"parent": {"if": GUARD}, "job": {"needs": "parent", "if": f"!cancelled() && {GUARD}"}}, + "job", + True, + id="status-function-with-own-guard", + ), + pytest.param( + { + "parent": {"if": GUARD}, + "other": {}, + "job": { + "needs": ["parent", "other"], + "if": "!cancelled() && needs.parent.result == 'success'", + }, + }, + "job", + True, + id="status-function-requiring-a-covered-parent", + ), + pytest.param( + { + "parent": {"if": GUARD}, + "other": {}, + "job": { + "needs": ["parent", "other"], + "if": "!cancelled() && needs.other.result == 'success'", + }, + }, + "job", + False, + id="status-function-requiring-an-uncovered-parent", + ), + pytest.param( + { + "parent": {"if": GUARD}, + "job": {"needs": [], "if": "always() && needs.parent.result == 'success'"}, + }, + "job", + False, + id="status-function-requiring-a-job-it-does-not-need", + ), + ], +) +def test_is_covered(jobs, job_name, expected): + assert is_covered(jobs, job_name) is expected + + +@pytest.mark.parametrize( + ("workflow", "expected"), + [ + pytest.param( + {"on": {"push": None}, "jobs": {"job": {}}}, + [], + id="not-scheduled", + ), + pytest.param( + {"on": "push", "jobs": {"job": {}}}, + [], + id="trigger-as-string", + ), + pytest.param( + {"on": SCHEDULE_ONLY, "jobs": {"guarded": {"if": GUARD}, "unguarded": {}}}, + ["unguarded"], + id="scheduled-with-unguarded-job", + ), + # Real workflows load `on:` as the key True, so this case uses True instead of "on". + pytest.param( + {True: SCHEDULE_ONLY, "jobs": {"job": {}}}, + ["job"], + id="bare-on-key", + ), + ], +) +def test_check_file(write_workflow_file, workflow, expected): + assert check_file(write_workflow_file(workflow)) == expected + + +def test_check_file_skips_exempt_workflow(tmp_path): + path = tmp_path / "ci-amd.yml" + path.write_text(yaml.dump({"on": SCHEDULE_ONLY, "jobs": {"job": {}}})) + assert check_file(path) == [] From 89b2ff5e6f4dd787ebe1984bf09cbb0724c9336c Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 15 Nov 2026 14:52:11 +0900 Subject: [PATCH 2/3] [v3-4-test] Say the fork CI monitor runs every 5 minutes in the CI tasks doc The sentence changed with #76836 on main and was missed by the sync. Generated-by: Claude Opus 5 --- dev/breeze/doc/08_ci_tasks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/breeze/doc/08_ci_tasks.rst b/dev/breeze/doc/08_ci_tasks.rst index 03d61fed826f5..c4001b658c00e 100644 --- a/dev/breeze/doc/08_ci_tasks.rst +++ b/dev/breeze/doc/08_ci_tasks.rst @@ -185,7 +185,7 @@ Reconciling pull requests gated to fork CI ------------------------------------------ Pull requests from external contributors can run their CI in the contributor's fork instead of in -``apache/airflow`` (AIP-120). The ``fork-ci-reconcile`` command, run every 15 minutes by the +``apache/airflow`` (AIP-120). The ``fork-ci-reconcile`` command, run every 5 minutes by the ``fork-ci-monitor.yml`` workflow, is the only place that changes those pull requests: it labels newly gated pull requests ``fork ci required``, keeps them in draft until the fork run is green, maintains one state comment and a ``Fork CI / Tests (AMD)`` commit status linking to the run in the fork, closes From 3bb1b30e757bd1b060b55bf6224288f6b9562299 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 15 Nov 2026 14:52:37 +0900 Subject: [PATCH 3/3] [v3-4-test] Bring the fork CI GitHub client tests in line with main The tests for the smaller job listing pages (#76327) were missed by the sync; the client code they cover is already in it. Generated-by: Claude Opus 5 --- dev/breeze/tests/test_fork_ci_github.py | 69 +++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/dev/breeze/tests/test_fork_ci_github.py b/dev/breeze/tests/test_fork_ci_github.py index 25ae3f105c6fd..ab497dd3b34cd 100644 --- a/dev/breeze/tests/test_fork_ci_github.py +++ b/dev/breeze/tests/test_fork_ci_github.py @@ -238,3 +238,72 @@ def test_set_status_context(mock_run, kwargs, context): GitHubClient().set_status(1, "abc", "success", "https://x", "desc", **kwargs) assert json.loads(mock_run.call_args.kwargs["input"])["context"] == context + + +def _jobs_page(*names: str) -> str: + return json.dumps([{"total_count": len(names), "jobs": [{"name": name} for name in names]}]) + + +def _jobs_path(page_size: int) -> str: + return f"repos/someone/airflow/actions/runs/7/jobs?filter=latest&per_page={page_size}" + + +BAD_GATEWAY = "gh: Server Error (HTTP 502)" + + +@mock.patch("airflow_breeze.utils.fork_ci_github.time.sleep", autospec=True) +@mock.patch("airflow_breeze.utils.fork_ci_github.subprocess.run", autospec=True) +def test_lists_run_jobs_in_smaller_pages_while_github_fails_to_list_them(mock_run, mock_sleep): + mock_run.side_effect = [ + *[_completed(1, stderr=BAD_GATEWAY)] * 4, + _completed(0, stdout=_jobs_page("Tests", "Docs")), + ] + + jobs = GitHubClient().get_run_jobs("someone/airflow", 7) + + assert jobs == [{"name": "Tests"}, {"name": "Docs"}] + assert [c.args[0][2] for c in mock_run.call_args_list] == [ + _jobs_path(100), + _jobs_path(100), + _jobs_path(30), + _jobs_path(30), + _jobs_path(15), + ] + + +@mock.patch("airflow_breeze.utils.fork_ci_github.time.sleep", autospec=True) +@mock.patch("airflow_breeze.utils.fork_ci_github.subprocess.run", autospec=True) +def test_gives_up_listing_run_jobs_when_the_smallest_pages_keep_failing(mock_run, mock_sleep): + mock_run.return_value = _completed(1, stderr=BAD_GATEWAY) + + with pytest.raises(GitHubError, match=r"per_page=5 --paginate --slurp still failing after 4 attempts"): + GitHubClient().get_run_jobs("someone/airflow", 7) + assert [c.args[0][2] for c in mock_run.call_args_list] == [ + *[_jobs_path(100)] * 2, + *[_jobs_path(30)] * 2, + *[_jobs_path(15)] * 2, + *[_jobs_path(5)] * 4, + ] + + +@mock.patch("airflow_breeze.utils.fork_ci_github.time.sleep", autospec=True) +@mock.patch("airflow_breeze.utils.fork_ci_github.subprocess.run", autospec=True) +def test_listing_run_jobs_retries_a_rate_limit_as_often_as_any_call(mock_run, mock_sleep): + mock_run.side_effect = [ + *[_completed(1, stderr="API rate limit exceeded")] * 3, + _completed(0, stdout=_jobs_page("Tests")), + ] + + assert GitHubClient().get_run_jobs("someone/airflow", 7) == [{"name": "Tests"}] + assert [c.args[0][2] for c in mock_run.call_args_list] == [_jobs_path(100)] * 4 + assert [c.args[0] for c in mock_sleep.call_args_list] == [30, 60, 120] + + +@mock.patch("airflow_breeze.utils.fork_ci_github.time.sleep", autospec=True) +@mock.patch("airflow_breeze.utils.fork_ci_github.subprocess.run", autospec=True) +def test_does_not_shrink_run_job_pages_for_errors_other_than_server_errors(mock_run, mock_sleep): + mock_run.return_value = _completed(1, stderr="gh: Not Found (HTTP 404)") + + with pytest.raises(GitHubError, match="HTTP 404"): + GitHubClient().get_run_jobs("someone/airflow", 7) + mock_run.assert_called_once()