diff --git a/.github/actions/merge-with-base/action.yml b/.github/actions/merge-with-base/action.yml new file mode 100644 index 0000000000000..2451ba616ecf3 --- /dev/null +++ b/.github/actions/merge-with-base/action.yml @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: 'Merge with the base branch' +description: >- + Replace the checked-out pushed commit of a fork CI run with its merge into the base branch, the + commit a pull_request run of project CI tests (AIP-120). Callers run it only when merge-with is + set: the action is read from the checkout, which an older ref or a skipped checkout lacks. +inputs: + merge-with: + description: >- + The base branch commit and the merge base with the pushed commit, space-separated, as + build-info resolved them for the whole run; empty outside fork runs. + required: true +outputs: + merge-sha: + description: "The merge commit now checked out; empty when nothing was merged." + value: ${{ steps.merge.outputs.merge-sha }} +runs: + using: "composite" + steps: + - name: "Merge the pushed commit with the base branch" + id: merge + if: inputs.merge-with != '' + shell: bash + env: + MERGE_WITH: ${{ inputs.merge-with }} + # The merge base is given, so the three commits are all that is fetched: no history between + # them is needed. Identity and dates are fixed, so every job of a run makes the same commit. + # The base branch is the first parent, as in GitHub's refs/pull//merge, so `^` + # is the base and `^..` is what the pull request changes. + run: | + set -euo pipefail + read -r base_sha merge_base <<< "${MERGE_WITH}" + head_sha="$(git rev-parse HEAD)" + for attempt in 1 2 3; do + if git fetch --no-tags --depth=1 https://github.com/apache/airflow.git \ + "${base_sha}" "${merge_base}"; then + break + fi + if [[ "${attempt}" -eq 3 ]]; then + echo "::error::Could not fetch ${base_sha} and ${merge_base} from apache/airflow" + exit 1 + fi + sleep $((attempt * 5)) + done + if ! tree="$(git merge-tree --write-tree --merge-base "${merge_base}" \ + "${base_sha}" "${head_sha}")"; then + echo "::error::The pushed commit conflicts with apache/airflow ${base_sha}; rebase it." + exit 1 + fi + commit_date="$(git show -s --format=%cI "${head_sha}")" + merge_sha="$( + GIT_AUTHOR_NAME="Fork CI" GIT_AUTHOR_EMAIL="dev@airflow.apache.org" \ + GIT_COMMITTER_NAME="Fork CI" GIT_COMMITTER_EMAIL="dev@airflow.apache.org" \ + GIT_AUTHOR_DATE="${commit_date}" GIT_COMMITTER_DATE="${commit_date}" \ + git commit-tree "${tree}" -p "${base_sha}" -p "${head_sha}" \ + -m "Merge ${head_sha} into apache/airflow ${base_sha}" + )" + git checkout --quiet --detach "${merge_sha}" + echo "merge-sha=${merge_sha}" >> "${GITHUB_OUTPUT}" + echo "Testing ${merge_sha}: ${head_sha} merged into apache/airflow ${base_sha}" diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index e1dbdd2a4e8e9..aff261cb0f0b6 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -20,6 +20,13 @@ name: Additional CI image checks on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -81,6 +88,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Check that image builds quickly" diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index aca622255be8a..505d56d2649a0 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -20,6 +20,13 @@ name: Additional PROD image tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -137,6 +144,7 @@ jobs: name: PROD image extra checks (main) uses: ./.github/workflows/prod-image-extra-checks.yml with: + merge-with: ${{ inputs.merge-with }} runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} python-versions: "[ '${{ inputs.default-python-version }}' ]" @@ -154,6 +162,7 @@ jobs: name: PROD image extra checks (release) uses: ./.github/workflows/prod-image-extra-checks.yml with: + merge-with: ${{ inputs.merge-with }} runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} python-versions: "[ '${{ inputs.default-python-version }}' ]" @@ -186,6 +195,11 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -225,6 +239,11 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -256,6 +275,11 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -274,6 +298,7 @@ jobs: if: inputs.run-generic-prod-image-tests == 'true' uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Regular e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -284,6 +309,7 @@ jobs: name: "Remote logging tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Remote logging e2e test" runners: ${{ format(inputs.runners, '4+8', '16+32') }} platform: ${{ inputs.platform }} @@ -298,6 +324,7 @@ jobs: name: "Elasticsearch remote logging tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Elasticsearch remote logging e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -312,6 +339,7 @@ jobs: name: "OpenSearch remote logging tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "OpenSearch remote logging e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -327,6 +355,7 @@ jobs: if: inputs.run-generic-prod-image-tests == 'true' uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "XCom object storage backend e2e test" runners: ${{ format(inputs.runners, '4+8', '16+32') }} platform: ${{ inputs.platform }} @@ -338,6 +367,7 @@ jobs: name: "Event driven tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Event driven e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -352,6 +382,7 @@ jobs: name: "Java SDK e2e tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Java SDK e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -366,6 +397,7 @@ jobs: name: "Go SDK e2e tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Go SDK e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -380,6 +412,7 @@ jobs: name: "TypeScript SDK e2e tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "TypeScript SDK e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -394,6 +427,7 @@ jobs: name: "YAML Dag e2e tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "YAML Dag e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -408,6 +442,7 @@ jobs: name: "Language SDK XCom object storage backend tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Language SDK XCom object storage backend e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -423,6 +458,7 @@ jobs: name: "OpenLineage e2e tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "OpenLineage e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -444,6 +480,7 @@ jobs: compat: ${{ fromJSON(inputs.providers-compatibility-tests-matrix) }} uses: ./.github/workflows/openlineage-e2e-compat-tests.yml with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" @@ -455,6 +492,7 @@ jobs: name: "Chromium UI e2e tests with PROD image" uses: ./.github/workflows/ui-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Chromium UI e2e tests" runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} @@ -467,6 +505,7 @@ jobs: name: "Firefox UI e2e tests with PROD image" uses: ./.github/workflows/ui-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Firefox UI e2e tests" runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} @@ -479,6 +518,7 @@ jobs: name: "WebKit UI e2e tests with PROD image" uses: ./.github/workflows/ui-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "WebKit UI e2e tests" runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} @@ -506,6 +546,11 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/agent-framework-tests.yml b/.github/workflows/agent-framework-tests.yml index 2f95c34cdfe9a..51098b48e0a15 100644 --- a/.github/workflows/agent-framework-tests.yml +++ b/.github/workflows/agent-framework-tests.yml @@ -20,6 +20,13 @@ name: Agent framework tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -75,6 +82,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/airflow-distributions-tests.yml b/.github/workflows/airflow-distributions-tests.yml index d3ff1fb4af872..99a6374d03a82 100644 --- a/.github/workflows/airflow-distributions-tests.yml +++ b/.github/workflows/airflow-distributions-tests.yml @@ -20,6 +20,13 @@ name: Non-core Distribution tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string # Static inputs defined to choose which distribution to test to run runners: description: "The array of labels (in json form) determining runners." @@ -95,6 +102,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ matrix.python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/airflow-e2e-tests.yml b/.github/workflows/airflow-e2e-tests.yml index 77f3aa5b81c96..6299cca56a15d 100644 --- a/.github/workflows/airflow-e2e-tests.yml +++ b/.github/workflows/airflow-e2e-tests.yml @@ -59,6 +59,13 @@ on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string workflow-name: description: "Name of the test" type: string @@ -112,6 +119,11 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index ac58cbf35b085..14e68490b11e9 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -20,6 +20,13 @@ name: Basic tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string newsfragment-pr-number: description: "Pull request the newsfragments are checked against; empty to accept any number" required: false @@ -116,6 +123,11 @@ jobs: # Need to fetch all history for selective checks tests fetch-depth: 0 persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Run unit tests" @@ -134,6 +146,11 @@ jobs: with: fetch-depth: 0 persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Install SVN" @@ -163,6 +180,11 @@ jobs: with: fetch-depth: 1 persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: "Install uv" @@ -204,6 +226,11 @@ jobs: with: fetch-depth: 1 persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: "Install uv" @@ -234,6 +261,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: Setup pnpm @@ -270,6 +302,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Check translation completeness" @@ -306,6 +343,11 @@ jobs: ref: ${{ github.sha }} fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Fetch the last tested push commit" if: inputs.push-base-commit != '' shell: bash @@ -322,8 +364,11 @@ jobs: run: | set -euo pipefail prek_args=(--show-diff-on-failure --color always) + # A fork run checks out its merge with the base branch, whose first parent is the base, so + # the pull request's own changes are checked rather than only its last commit's. + tested="$(git rev-parse HEAD)" if [[ -z "${PUSH_BASE_COMMIT}" ]]; then - prek "${prek_args[@]}" --from-ref "${GITHUB_SHA}^" --to-ref "${GITHUB_SHA}" + prek "${prek_args[@]}" --from-ref "${tested}^" --to-ref "${tested}" elif git cat-file -e "${PUSH_BASE_COMMIT}^{commit}"; then mapfile -t files < <(git diff --name-only --no-renames --diff-filter=d \ "${PUSH_BASE_COMMIT}" "${GITHUB_SHA}") @@ -363,6 +408,11 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} test-airflow-release-commands: timeout-minutes: 80 @@ -383,6 +433,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Cleanup dist files" @@ -431,6 +486,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: "Install uv" diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index f16449144bf1f..76a57dc477b62 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -423,6 +423,9 @@ jobs: use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }} upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} is-uv-lock-update: ${{ steps.selective-checks.outputs.is-uv-lock-update }} + # The base branch commit and merge base that every job of a fork run merges the pushed + # commit with, so that it tests what a pull_request run of project CI tests; empty elsewhere. + merge-with: ${{ steps.merge-base.outputs.merge-with }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -435,6 +438,7 @@ jobs: fetch-depth: 2 persist-credentials: false - name: "Check clean integration with upstream target branch" + id: merge-base if: github.event_name == 'push' && github.repository != 'apache/airflow' shell: bash run: | @@ -467,6 +471,14 @@ jobs: exit 1 elif [ "${conflict}" -gt 1 ]; then echo "::warning::Git merge-tree exited with error code ${conflict} (not a merge conflict). Skipping conflict check." + else + target_sha=$(git rev-parse "refs/remotes/upstream/${target_branch}") + { + echo "merge-with=${target_sha} ${merge_base}" + echo "target-branch=${target_branch}" + echo "target-sha=${target_sha}" + echo "merge-base=${merge_base}" + } >> "${GITHUB_OUTPUT}" fi else echo "::warning::Could not find merge-base with upstream/${target_branch} after deepening up to 1000 commits. Skipping conflict check." @@ -474,6 +486,13 @@ jobs: else echo "::warning::Could not fetch upstream/${target_branch} from https://github.com/apache/airflow.git. Skipping conflict check." fi + # The fork CI monitor reads from this step's name which merge the run tested. + - name: >- + Test merged with apache/airflow ${{ steps.merge-base.outputs.target-branch }}@${{ + steps.merge-base.outputs.target-sha }}, merge base ${{ steps.merge-base.outputs.merge-base }} + if: steps.merge-base.outputs.merge-with != '' + shell: bash + run: echo "The jobs of this run test the pushed commit merged with apache/airflow." - name: "Fetch the last tested push commit" # Selective checks of a push diff against this commit. A two-commit diff only needs the # commit itself, not the history in between; if it cannot be fetched, all tests run. @@ -574,6 +593,7 @@ jobs: needs: [build-info, check-run-needed] uses: ./.github/workflows/basic-tests.yml with: + merge-with: ${{ needs.build-info.outputs.merge-with }} newsfragment-pr-number: ${{ needs.build-info.outputs.newsfragment-pr-number }} runners: ${{ needs.build-info.outputs.runner-type }} run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}} @@ -604,6 +624,7 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} push-image: "false" @@ -629,6 +650,7 @@ jobs: packages: write id-token: write with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} branch: ${{ needs.build-info.outputs.default-branch }} @@ -644,6 +666,7 @@ jobs: uses: ./.github/workflows/generate-constraints.yml if: needs.build-info.outputs.generate-constraints == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} @@ -665,6 +688,7 @@ jobs: id-token: write contents: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} newsfragment-pr-number: ${{ needs.build-info.outputs.newsfragment-pr-number }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} @@ -710,6 +734,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -752,6 +781,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -787,6 +821,7 @@ jobs: needs.build-info.outputs.latest-versions-only != 'true' && needs.build-info.outputs.run-unit-tests == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} canary-run: ${{ needs.build-info.outputs.canary-run }} @@ -812,6 +847,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} @@ -831,6 +867,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -864,6 +901,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" @@ -893,6 +931,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" @@ -922,6 +961,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" @@ -952,6 +992,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" @@ -981,6 +1022,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -1012,6 +1054,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -1043,6 +1086,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -1073,6 +1117,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -1108,6 +1153,7 @@ jobs: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' || needs.build-info.outputs.full-tests-needed == 'true') with: + merge-with: ${{ needs.build-info.outputs.merge-with }} default-branch: ${{ needs.build-info.outputs.default-branch }} runners: ${{ needs.build-info.outputs.runner-type }} in-memory-runners: ${{ needs.build-info.outputs.in-memory-runner-type }} @@ -1138,6 +1184,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }} @@ -1162,6 +1209,7 @@ jobs: if: > needs.build-info.outputs.run-unit-tests == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" @@ -1192,6 +1240,7 @@ jobs: packages: read if: needs.build-info.outputs.run-unit-tests == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" @@ -1220,6 +1269,7 @@ jobs: permissions: contents: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1237,6 +1287,7 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} build-type: "Regular" @@ -1260,6 +1311,7 @@ jobs: needs: [build-info, build-prod-images, generate-constraints] uses: ./.github/workflows/additional-prod-image-tests.yml with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} default-branch: ${{ needs.build-info.outputs.default-branch }} @@ -1299,6 +1351,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} @@ -1322,6 +1375,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '8+16', '32+64') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1354,6 +1408,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary # keep this in sync with go.mod in go-sdk/ @@ -1417,6 +1476,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: Setup pnpm @@ -1459,6 +1523,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: Setup Java @@ -1489,6 +1558,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary # Serializes the same Dags with Airflow, from airflow-core's sources, to compare them. @@ -1545,6 +1619,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Build Java SDK Javadoc" @@ -1577,6 +1656,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Build TypeScript SDK API reference" @@ -1598,6 +1682,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '4+8', '16+32') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1619,6 +1704,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '8+16', '32+64') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1635,6 +1721,7 @@ jobs: permissions: contents: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1685,6 +1772,7 @@ jobs: - tests-with-lowest-direct-resolution-providers uses: ./.github/workflows/finalize-tests.yml with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions: ${{ needs.build-info.outputs.python-versions }} diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 3aadb07674e29..98d8f07c50a5c 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -299,6 +299,9 @@ jobs: use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }} upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} is-uv-lock-update: ${{ steps.selective-checks.outputs.is-uv-lock-update }} + # The base branch commit and merge base that every job of a fork run merges the pushed + # commit with, so that it tests what a pull_request run of project CI tests; empty elsewhere. + merge-with: ${{ steps.merge-base.outputs.merge-with }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -311,6 +314,7 @@ jobs: fetch-depth: 2 persist-credentials: false - name: "Check clean integration with upstream target branch" + id: merge-base if: github.event_name == 'push' && github.repository != 'apache/airflow' shell: bash run: | @@ -343,6 +347,14 @@ jobs: exit 1 elif [ "${conflict}" -gt 1 ]; then echo "::warning::Git merge-tree exited with error code ${conflict} (not a merge conflict). Skipping conflict check." + else + target_sha=$(git rev-parse "refs/remotes/upstream/${target_branch}") + { + echo "merge-with=${target_sha} ${merge_base}" + echo "target-branch=${target_branch}" + echo "target-sha=${target_sha}" + echo "merge-base=${merge_base}" + } >> "${GITHUB_OUTPUT}" fi else echo "::warning::Could not find merge-base with upstream/${target_branch} after deepening up to 1000 commits. Skipping conflict check." @@ -350,6 +362,13 @@ jobs: else echo "::warning::Could not fetch upstream/${target_branch} from https://github.com/apache/airflow.git. Skipping conflict check." fi + # The fork CI monitor reads from this step's name which merge the run tested. + - name: >- + Test merged with apache/airflow ${{ steps.merge-base.outputs.target-branch }}@${{ + steps.merge-base.outputs.target-sha }}, merge base ${{ steps.merge-base.outputs.merge-base }} + if: steps.merge-base.outputs.merge-with != '' + shell: bash + run: echo "The jobs of this run test the pushed commit merged with apache/airflow." - name: "Fetch the last tested push commit" # Selective checks of a push diff against this commit. A two-commit diff only needs the # commit itself, not the history in between; if it cannot be fetched, all tests run. @@ -450,6 +469,7 @@ jobs: needs: [build-info, check-run-needed] uses: ./.github/workflows/basic-tests.yml with: + merge-with: ${{ needs.build-info.outputs.merge-with }} newsfragment-pr-number: ${{ needs.build-info.outputs.newsfragment-pr-number }} runners: ${{ needs.build-info.outputs.runner-type }} run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}} @@ -480,6 +500,7 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} push-image: "false" @@ -505,6 +526,7 @@ jobs: packages: write id-token: write with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} branch: ${{ needs.build-info.outputs.default-branch }} @@ -520,6 +542,7 @@ jobs: uses: ./.github/workflows/generate-constraints.yml if: needs.build-info.outputs.generate-constraints == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} @@ -541,6 +564,7 @@ jobs: id-token: write contents: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} newsfragment-pr-number: ${{ needs.build-info.outputs.newsfragment-pr-number }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} @@ -586,6 +610,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -628,6 +657,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -663,6 +697,7 @@ jobs: needs.build-info.outputs.latest-versions-only != 'true' && needs.build-info.outputs.run-unit-tests == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} canary-run: ${{ needs.build-info.outputs.canary-run }} @@ -688,6 +723,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} @@ -707,6 +743,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -740,6 +777,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" @@ -769,6 +807,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" @@ -798,6 +837,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" @@ -828,6 +868,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" @@ -857,6 +898,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -888,6 +930,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -919,6 +962,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -949,6 +993,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -984,6 +1029,7 @@ jobs: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' || needs.build-info.outputs.full-tests-needed == 'true') with: + merge-with: ${{ needs.build-info.outputs.merge-with }} default-branch: ${{ needs.build-info.outputs.default-branch }} runners: ${{ needs.build-info.outputs.runner-type }} in-memory-runners: ${{ needs.build-info.outputs.in-memory-runner-type }} @@ -1014,6 +1060,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }} @@ -1038,6 +1085,7 @@ jobs: if: > needs.build-info.outputs.run-unit-tests == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" @@ -1068,6 +1116,7 @@ jobs: packages: read if: needs.build-info.outputs.run-unit-tests == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" @@ -1096,6 +1145,7 @@ jobs: permissions: contents: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1113,6 +1163,7 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} build-type: "Regular" @@ -1136,6 +1187,7 @@ jobs: needs: [build-info, build-prod-images, generate-constraints] uses: ./.github/workflows/additional-prod-image-tests.yml with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} default-branch: ${{ needs.build-info.outputs.default-branch }} @@ -1175,6 +1227,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} @@ -1198,6 +1251,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '8+16', '32+64') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1230,6 +1284,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary # keep this in sync with go.mod in go-sdk/ @@ -1293,6 +1352,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: Setup pnpm @@ -1335,6 +1399,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: Setup Java @@ -1365,6 +1434,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary # Serializes the same Dags with Airflow, from airflow-core's sources, to compare them. @@ -1421,6 +1495,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Build Java SDK Javadoc" @@ -1453,6 +1532,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: needs.build-info.outputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Build TypeScript SDK API reference" @@ -1474,6 +1558,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '4+8', '16+32') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1495,6 +1580,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '8+16', '32+64') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1511,6 +1597,7 @@ jobs: permissions: contents: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1561,6 +1648,7 @@ jobs: - tests-with-lowest-direct-resolution-providers uses: ./.github/workflows/finalize-tests.yml with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions: ${{ needs.build-info.outputs.python-versions }} diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 78d676715c4b3..86bf2d593df02 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -20,6 +20,13 @@ name: Build CI images on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -166,6 +173,11 @@ jobs: sparse-checkout: | .github/actions scripts/ci + - name: "Merge with the base branch" + if: inputs.checkout-ref == '' && inputs.merge-with != '' + uses: ./workflow-version/.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Free up disk space" shell: bash run: ./scripts/tools/free_up_disk_space.sh diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index dcadcd6dbfd5f..ef6adf242e395 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -20,6 +20,13 @@ name: CI Image Checks on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string newsfragment-pr-number: description: "Pull request the newsfragments are checked against; empty to accept any number" required: false @@ -142,6 +149,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -214,6 +226,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -375,6 +392,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Download docs prepared as artifacts" @@ -510,6 +532,11 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/dependency-upgrade-summary.yml b/.github/workflows/dependency-upgrade-summary.yml index 626f594940c25..497fe0ef26c04 100644 --- a/.github/workflows/dependency-upgrade-summary.yml +++ b/.github/workflows/dependency-upgrade-summary.yml @@ -23,6 +23,13 @@ name: Dependency upgrade summary on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -64,6 +71,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ matrix.python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 52926b1028dc8..6ace614f400c2 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -20,6 +20,13 @@ name: Finalize tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -103,6 +110,11 @@ jobs: with: # Needed to perform push action persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: "Set constraints branch name" @@ -135,6 +147,7 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: + merge-with: ${{ inputs.merge-with }} runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} cache-type: "Regular AMD" diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index b388af7196f5d..a15d2930ea71f 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -20,6 +20,13 @@ name: Generate constraints on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -116,6 +123,11 @@ jobs: with: ref: ${{ inputs.checkout-ref }} persist-credentials: false + - name: "Merge with the base branch" + if: inputs.checkout-ref == '' && inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ matrix.python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 1f505b8a45345..5c5c1dbd1df42 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -20,6 +20,13 @@ name: Helm tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -81,6 +88,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -112,6 +124,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: Setup git for tagging diff --git a/.github/workflows/integration-system-tests.yml b/.github/workflows/integration-system-tests.yml index e46173b9c709d..0e250cbafebe4 100644 --- a/.github/workflows/integration-system-tests.yml +++ b/.github/workflows/integration-system-tests.yml @@ -20,6 +20,13 @@ name: Integration and system tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining public runners." required: true @@ -103,6 +110,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -156,6 +168,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -201,6 +218,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index 90ce5a9ffee06..0ae63d236136d 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -20,6 +20,13 @@ name: K8s tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -94,6 +101,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} # env.PYTHON_MAJOR_MINOR_VERSION, env.KUBERNETES_VERSION are set in the previous # step id: prepare-versions - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" @@ -174,6 +186,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} # env.PYTHON_MAJOR_MINOR_VERSION, env.KUBERNETES_VERSION are set in the previous # step id: prepare-versions - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" diff --git a/.github/workflows/kustomize-overlays-tests.yml b/.github/workflows/kustomize-overlays-tests.yml index 5f9be6f52fb98..6814b44f9ce8f 100644 --- a/.github/workflows/kustomize-overlays-tests.yml +++ b/.github/workflows/kustomize-overlays-tests.yml @@ -20,6 +20,13 @@ name: Kustomize overlays smoke tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -73,6 +80,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/openlineage-e2e-compat-tests.yml b/.github/workflows/openlineage-e2e-compat-tests.yml index 4b9cb6d440867..f2a6f7ecd5b28 100644 --- a/.github/workflows/openlineage-e2e-compat-tests.yml +++ b/.github/workflows/openlineage-e2e-compat-tests.yml @@ -24,6 +24,13 @@ permissions: on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -73,6 +80,11 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + if: ${{ !startsWith(inputs.airflow-version, '2.') && inputs.merge-with != '' }} + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" if: ${{ !startsWith(inputs.airflow-version, '2.') }} uses: ./.github/actions/prepare_breeze_and_image diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index a7c898ff7f15a..d88e15b8fc5a1 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -20,6 +20,13 @@ name: Build PROD images on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -136,6 +143,7 @@ jobs: permissions: contents: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} default-python-version: ${{ inputs.default-python-version }} @@ -182,6 +190,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Free up disk space" shell: bash run: ./scripts/tools/free_up_disk_space.sh diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml index 5b3d072a49a72..3afa932970fc1 100644 --- a/.github/workflows/prod-image-extra-checks.yml +++ b/.github/workflows/prod-image-extra-checks.yml @@ -20,6 +20,13 @@ name: PROD images extra checks on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -62,6 +69,7 @@ jobs: pip-image: uses: ./.github/workflows/prod-image-build.yml with: + merge-with: ${{ inputs.merge-with }} runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} build-type: "pip" diff --git a/.github/workflows/prod-packages-build.yml b/.github/workflows/prod-packages-build.yml index c80bfd816db88..e8a13968feca9 100644 --- a/.github/workflows/prod-packages-build.yml +++ b/.github/workflows/prod-packages-build.yml @@ -23,6 +23,13 @@ name: Build PROD packages on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -66,6 +73,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Make /mnt writeable" run: ./scripts/ci/make_mnt_writeable.sh if: inputs.upload-package-artifact == 'true' diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index 8fdb06f95ce6f..63d14e5ee7d36 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -20,6 +20,13 @@ name: Push image cache on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -117,6 +124,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Free up disk space" run: ./scripts/tools/free_up_disk_space.sh - name: "Install Breeze" @@ -187,6 +199,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Free up disk space" run: ./scripts/tools/free_up_disk_space.sh - name: "Install Breeze" diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index c2564f71462d3..80d700933b131 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -20,6 +20,13 @@ name: Unit tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining public AMD runners." required: true @@ -199,6 +206,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Make /mnt writeable" run: ./scripts/ci/make_mnt_writeable.sh - name: "Move docker to /mnt" diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index 1b47ac5259a47..7dfbf784c9d19 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -20,6 +20,13 @@ name: Special tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -102,6 +109,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} platform: ${{ inputs.platform }} downgrade-sqlalchemy: "true" @@ -127,6 +135,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} downgrade-sqlalchemy: "true" @@ -152,6 +161,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} platform: ${{ inputs.platform }} upgrade-sqlalchemy: "true" @@ -177,6 +187,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} upgrade-sqlalchemy: "true" @@ -202,6 +213,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} platform: ${{ inputs.platform }} upgrade-boto: "true" @@ -228,6 +240,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} upgrade-boto: "true" @@ -254,6 +267,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} platform: ${{ inputs.platform }} downgrade-pendulum: "true" @@ -280,6 +294,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} downgrade-pendulum: "true" @@ -306,6 +321,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.runners, '4+8', '16+32') }} platform: ${{ inputs.platform }} test-name: "Postgres" @@ -331,6 +347,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.runners, '4+8', '16+32') }} platform: ${{ inputs.platform }} test-name: "Postgres" @@ -356,6 +373,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} test-name: "SystemTest" diff --git a/.github/workflows/test-providers.yml b/.github/workflows/test-providers.yml index 19780189cb307..3d3b907b852fb 100644 --- a/.github/workflows/test-providers.yml +++ b/.github/workflows/test-providers.yml @@ -20,6 +20,13 @@ name: Provider tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining public AMD runners." required: true @@ -96,6 +103,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -207,6 +219,11 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ matrix.compat.python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/ui-e2e-tests.yml b/.github/workflows/ui-e2e-tests.yml index 96e68f83e1209..a295ca1857c01 100644 --- a/.github/workflows/ui-e2e-tests.yml +++ b/.github/workflows/ui-e2e-tests.yml @@ -55,6 +55,13 @@ on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string workflow-name: description: "Name of the test" type: string @@ -107,6 +114,11 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + if: inputs.merge-with != '' + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5aa9c046bd0ac..e1bc54aa5614c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1740,6 +1740,16 @@ repos: pass_filenames: true require_serial: true files: ^\.github/workflows/.*\.yml$ + - id: check-fork-ci-tests-merge + name: Check fork CI jobs test the pushed commit merged with the base branch + env: + PYTHONPYCACHEPREFIX: .build/pycache + description: Every job of a fork run that checks out the pushed commit merges it with the base branch + entry: ./scripts/ci/prek/check_fork_ci_tests_merge.py + language: python + pass_filenames: false + require_serial: true + files: ^\.github/workflows/.*\.yml$|^scripts/ci/prek/check_fork_ci_tests_merge\.py$ - id: check-docstring-param-types name: Check that docstrings do not specify param types env: diff --git a/contributing-docs/18_contribution_workflow.rst b/contributing-docs/18_contribution_workflow.rst index 8c2e23990f604..14c0c8b4b5069 100644 --- a/contributing-docs/18_contribution_workflow.rst +++ b/contributing-docs/18_contribution_workflow.rst @@ -197,11 +197,11 @@ does not run for 30 days is closed with an explanation; you can reopen it after files that decide which CI jobs run and where, such as ``.github/``, the breeze sources or ``scripts/ci/prek/``, a green fork run is not enough: a maintainer reviews the change and runs project CI for it. -Fork CI tests your branch as it is, while project CI tests it merged with the target branch. The -``CI passed`` check that merging requires is therefore reported only when your branch is fewer than -200 commits behind the target branch and the target branch has not changed any file your pull request -changes since your branch's base. Otherwise the pull request comment lists those files and asks you -to rebase on ``upstream/main`` (or the target branch) and push, which runs fork CI again. +Like project CI, fork CI tests your branch merged with the target branch, as the target branch was +when the run started; a push that conflicts with it fails at once and asks you to rebase. The +``CI passed`` check that merging requires is reported only when that commit of the target branch is +fewer than 200 commits behind the target branch. Otherwise the pull request comment asks you to +rebase on ``upstream/main`` (or the target branch) and push, which runs fork CI again. **Joining the pilot.** Open a pull request that adds your GitHub login to the ``[pilot]`` table in ``.github/fork-ci-gate.toml``, with a join instant after the expected merge, for example the next day diff --git a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md index a1d932971a596..450d7ea54f607 100644 --- a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md +++ b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md @@ -464,11 +464,13 @@ upstream yields no changed files, so such a run performs basic checks only. would be the contributor's own previous tip after they merge upstream `main`, so a release manager's bump from `main` would count as theirs. A file missing at the merge base is new; any other read failure fails the run rather than letting the check pass unseen. -- **Early conflict check**: before the test matrix runs, `ci-amd.yml` verifies that the pushed - commit merges cleanly with upstream's target branch using `git merge-tree`. It fetches the - upstream target branch and deepens history in increments of 200 commits up to 1,000 commits if +- **Early conflict check and merge base**: before the test matrix runs, `build-info` verifies that + the pushed commit merges cleanly with upstream's target branch using `git merge-tree`. It fetches + the upstream target branch and deepens history in increments of 200 commits up to 1,000 commits if needed to resolve the merge base; if a merge conflict is detected, the workflow fails early with - clear rebase instructions rather than spending runner hours on conflicting code. + clear rebase instructions rather than spending runner hours on conflicting code. Otherwise it + outputs the target branch commit and the merge base as `merge-with`, which every later job uses to + test the pushed commit merged with that commit (see "Testing the merged result"). - **Labels** come from the open `apache/airflow` pull request whose head is `:`, since a push event carries none. `full tests needed`, `allow provider dependency bump`, `skip common compat check` and the other labels work as on @@ -480,6 +482,42 @@ upstream yields no changed files, so such a run performs basic checks only. lands. - Canary detection is unchanged and already false outside `apache/airflow`. +### Testing the merged result + +A `pull_request` run of project CI tests GitHub's merge commit of the pull request into its base +branch. A fork run is a `push` run of the pushed commit, so every job that checks out the pull +request's sources turns that checkout into the same kind of merge before it builds or tests +anything, with the `merge-with-base` action (`.github/actions/merge-with-base`): + +- `build-info` fixes the merge for the whole run: the target branch commit it fetched for the + conflict check and the merge base with the pushed commit, output as `merge-with`. Every job + therefore merges with the same commit, even when the target branch moves during the run. +- Each job fetches those two commits from `apache/airflow` at depth 1 and builds the merge with + `git merge-tree --write-tree --merge-base`, which needs the three commits and none of the history + between them, so the shallow checkouts the jobs already use are enough. `git commit-tree` turns the + tree into a commit with the target branch commit as its first parent and the pushed commit as its + second, as in GitHub's `refs/pull//merge`, so `^..` is what the pull request + changes; the basic static checks diff exactly that. The author, committer and dates are fixed, + so every job makes the same merge commit, which is then checked out in place of the pushed commit. +- The merge needs no write access: nothing is pushed, and fork runs keep a read-only token. +- `merge-with` is passed to every reusable workflow a fork run calls, and is empty in every run in + `apache/airflow`. Each merge step runs only when it is set: the action is read from the checkout, + and a checkout of an older ref, such as a release a constraints run builds from, or a checkout a + matrix entry skips, has no such action. A workflow whose checkout takes another ref + (`checkout-ref`, such as a target branch to build an image from) merges only when it checks out + the pushed commit; `ci-image-build.yml` reads the action from its sparse checkout of the + workflow's own version, as it reads its other local actions, so no code of that ref runs as one. +- The `check-fork-ci-tests-merge` prek hook keeps it that way: in `ci-amd.yml`, `ci-arm.yml` and every + workflow they call, each call passes `merge-with`, and each job that checks out the pushed commit + into the workspace merges after its last such checkout, in a step whose `if:` checks `merge-with`. + `gate`, `build-info`, `notify-slack` and + `summarize-warnings` are exempt: they read the pushed commit or only report on the run. +- `build-info` names the merge in a step, "Test merged with apache/airflow + ``@``, merge base ``", from which the reconciler learns what the run + tested (see "Tested base"). When the conflict check cannot fetch the target branch or find the + merge base, `merge-with` is empty, the jobs test the pushed commit as it is, and the step is + skipped. + `breeze ci selective-check` gets the compare result through a new `--changed-files-from-api` path taken automatically when `is_fork_push` holds; the `git diff-tree` path stays for every other event. The `GITHUB_TOKEN` of the fork run is sufficient: the compare reads public data. @@ -1151,14 +1189,33 @@ also run in a gated run started, none failed, and at least one was cancelled. ### Tested base -A fork run tests the pushed commit as it is, while project CI's `pull_request` run tests GitHub's -merge commit of the pull request into its base branch. A fork run on an old base could otherwise go -green and merge although the merged result was never tested. So `CI passed` is only reported for a -fork run whose base is close enough to the base branch that the merge changes nothing the pull -request touches. +A fork run tests the pushed commit merged with a commit of the base branch ("Testing the merged +result"), as project CI's `pull_request` run tests GitHub's merge commit. The base branch moves on +after the run, so `CI passed` is only reported for a run whose merge is recent enough. + +When a head commit turns green, the reconciler reads the merge the run tested from the step of its +`Build info` job that names it, and checks it against GitHub, once per head commit: + +- the named branch must be the pull request's base branch; +- `GET /repos/apache/airflow/compare/...:` must give the named merge + base, so the merge is the one GitHub would compute for the head commit; +- `GET /repos/apache/airflow/compare/...` must say the base branch is + `ahead` of the tested commit or `identical` to it, so the tested commit is on the base branch, and + its `ahead_by` is how far behind the merge is (`behind`). + +The result is stored in the marker as `"base": {"sha", "behind", "overlap": [], "merged": true}`. +The run's report is trusted as far as its green result is: both come from the workflow of the head +commit, which a pull request changing CI-defining files is not trusted with in any case (see +"Trusting a green fork run"). + +A run whose named merge does not check out, for example a pull request to a stable branch, whose +fork run merges with the test branch that shares its defaults, gets no `CI passed`: its jobs tested +that merge, so neither it nor the head commit as it is says what the merged result would do. As for +a compare that fails, the pull request is listed for Slack, and a maintainer can run project CI for +it with `use project ci`. -When a head commit turns green, the reconciler compares it with the base branch, once per head -commit, and stores the result in the marker as `"base": {"sha", "behind", "overlap"}`: +A run that names no merge, because the conflict check could not resolve one, tested the head commit +as it is and is compared as such: - `GET /repos/apache/airflow/compare/...:` gives the merge base, the commit of the base branch the fork run tested on top of (`sha`), the commits the base branch has since @@ -1168,27 +1225,26 @@ commit, and stores the result in the marker as `"base": {"sha", "behind", "overl counting under both paths. `CI passed` is reported only when the base is current: fewer than `MAX_BEHIND_COMMITS` (200, about -four days of `main`) commits behind and no overlap. Otherwise the state stays `green` (the pull -request can be reviewed, and its status, labels and draft state are those of `green`), the state -comment lists the overlapping files or says how far behind the base is and gives the rebase -commands, and `CI passed` is posted as `pending` with a description saying to rebase. A rebase and -push starts a new fork run, which is compared again when it turns green. Every green state comment -names the base commit the run tested and how far behind it was, and the `CI passed` description -names it as well. - -The compare API lists at most 300 files. A pull request with more is listed with -`GET /repos/apache/airflow/pulls//files` instead (up to 3000); a base branch that changed 300 or -more files since the merge base cannot be checked and counts as overlapping, which a rebase -resolves. When the compare fails, `CI passed` is not reported, the failure is reported to Slack and -the next run compares again. A marker written before this check, with no `base`, is compared again -on the next run, so a pull request that already had `CI passed` loses it if its base turns out not -to be current. A retarget compares again, as it decides green trust again. - -This narrows the gap to project CI without closing it. A base change in other files can still break -the pull request, within at most 200 commits, which matches what project CI leaves open after its -run: a `pull_request` run is not repeated when the base branch moves, so the merged result at merge -time is not the one it tested either. Testing the merge commit in the fork itself was considered and -rejected; see "Known limitations". +four days of `main`) commits behind, and, for a run that did not test the merge, no overlap. +Otherwise the state stays `green` (the pull request can be reviewed, and its status, labels and +draft state are those of `green`), the state comment says how far behind the base is or lists the +overlapping files and gives the rebase commands, and `CI passed` is posted as `pending` with a +description saying to rebase. A rebase and push starts a new fork run, which is compared again when +it turns green. Every green state comment names the base commit the run tested, whether merged +with it or on top of it, and how far behind it was, and the `CI passed` description names it as +well. + +The compare API lists at most 300 files. For a run that did not test the merge, a pull request with +more is listed with `GET /repos/apache/airflow/pulls//files` instead (up to 3000); a base branch +that changed 300 or more files since the merge base cannot be checked and counts as overlapping, +which a rebase resolves. When the compare fails, `CI passed` is not reported, the failure is +reported to Slack and the next run compares again. A marker written before this check, with no +`base`, is compared again on the next run, so a pull request that already had `CI passed` loses it +if its base turns out not to be current. A retarget compares again, as it decides green trust again. + +A merged run leaves the same gap as project CI: a `pull_request` run is not repeated when the base +branch moves, so the merged result at merge time is not the one it tested either. The 200-commit +limit bounds that gap, which project CI does not. ### Unknown failures and attention items go to Slack @@ -1443,17 +1499,13 @@ as a failed check rather than a crash. - An external pull request that legitimately changes workflow files, was released with `use project ci` and then had that label removed is closed on the next run, because both signals hold. Keeping `use project ci` on such a pull request is the intended handling. -- Fork CI tests the head commit, not the pull request merged into its base branch; "Tested base" - only bounds the difference. Testing the merged result in the fork was rejected because: - every job of the fork run checks out the pushed commit (about 60 `actions/checkout` steps across - `ci-amd.yml` and its reusable workflows, plus the image builds and caches keyed by the commit), - and each would have to check out one merge commit fixed for the whole run; the first push of a - branch happens before its pull request exists, so there is no `refs/pull//merge` to test yet; - GitHub computes that ref asynchronously after each push and replaces it whenever the base branch - moves, so a run would have to wait for it and could find its commit gone mid-run; and the - reconciler would have to trust the run's own report of which merge it tested. Merging the base - branch into the head in every job instead needs the shared history of both, which shallow - checkouts do not have. +- A fork run merges with the target branch that breeze's default branch at the pushed commit names + (`main`, or the release branch it was cut from), since a push does not know which pull request it + is for. A pull request to another base branch gets no `CI passed` from fork CI ("Tested base") and + needs project CI. A run whose conflict check cannot fetch the target branch, or find the merge base + within 1,000 commits, tests the head commit as it is. +- Every job of a fork run fetches the target branch commit and the merge base (two depth-1 fetches, + some seconds each) to build the merge, about 55 jobs for a full-matrix run. ## Testing diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py b/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py index 7378ad7f368db..daaae9783efea 100644 --- a/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py +++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py @@ -55,9 +55,9 @@ CI_RUN_STARTED_CONTEXT = "CI run started" CI_RUN_STARTED_DESCRIPTION = "CI runs in the contributor's fork - details open the run" CI_RUN_STARTED_PROJECT_DESCRIPTION = "Project CI runs in apache/airflow - details open the run" -# A fork run tests the head commit as it is, not merged with the base branch as project CI does. From -# this many commits behind, `CI passed` is withheld until the branch is rebased, as it is when the -# base branch changed any file the pull request changes. +# From this many commits behind the base branch commit a fork run tested with, `CI passed` is withheld +# until the branch is rebased. A run that tested the head commit as it is, not merged with the base +# branch, also gets none when the base branch has since changed a file the pull request changes. MAX_BEHIND_COMMITS = 200 # Overlapping files named in the state comment; the marker keeps them all. OVERLAP_LISTED = 10 @@ -97,20 +97,24 @@ class BaseCheck: """ The base a green fork run tested, compared with the base branch when the run turned green. - :param sha: the merge base of the head commit and the base branch, the base the run tested. + :param sha: the base branch commit the run tested the head commit merged with; for a run that + tested the head commit as it is, the merge base of the head commit and the base branch. :param behind: commits on the base branch since ``sha``. :param overlap: paths that both the pull request and the base branch since ``sha`` change; - None when GitHub listed too many changed files on either side to tell. + None when GitHub listed too many changed files on either side to tell. Not checked for a + merged run, which tested the base branch's changes until ``sha`` with the pull request. + :param merged: whether the run tested the head commit merged with ``sha``, as project CI does. """ sha: str behind: int overlap: tuple[str, ...] | None = () + merged: bool = False @property def is_current(self) -> bool: """Whether the run tested close enough to the merged result for `CI passed`.""" - return self.behind < MAX_BEHIND_COMMITS and self.overlap == () + return self.behind < MAX_BEHIND_COMMITS and (self.merged or self.overlap == ()) @dataclass(frozen=True) @@ -132,11 +136,14 @@ class Marker: def render_marker(marker: Marker) -> str: data: dict[str, object] = {"state": marker.state, "sha": marker.sha, "since": marker.since.isoformat()} if marker.base is not None: - data["base"] = { + base: dict[str, object] = { "sha": marker.base.sha, "behind": marker.base.behind, "overlap": None if marker.base.overlap is None else list(marker.base.overlap), } + if marker.base.merged: + base["merged"] = True + data["base"] = base if marker.jobs is not None: data["jobs"] = marker.jobs if marker.failed_run is not None: @@ -165,6 +172,7 @@ def parse_marker(body: str) -> Marker | None: sha=base["sha"], behind=base["behind"], overlap=None if base["overlap"] is None else tuple(base["overlap"]), + merged=base.get("merged") is True, ), jobs=data.get("jobs"), failed_run=data.get("failed_run"), @@ -290,6 +298,20 @@ def strip_failed_job_list(body: str) -> str: def tested_base_text(base: BaseCheck, *, base_ref: str) -> str: + if base.merged: + text = ( + f"Fork CI tested this branch merged with `{base_ref}` at " + f"[`{base.sha[:10]}`](https://github.com/apache/airflow/commit/{base.sha}), as project CI " + f"would, {base.behind} commits behind `{base_ref}` when the run turned green." + ) + if base.is_current: + return text + return ( + f"{text}\n\n**Rebase needed.** That is too far behind to say much about the merged result. " + f"The `CI passed` check is not reported until fork CI is green for a branch rebased on " + f"`{base_ref}`:\n\n" + f"```bash\ngit fetch upstream\ngit rebase upstream/{base_ref}\ngit push --force-with-lease\n```" + ) text = ( f"Fork CI tested this branch on top of `{base_ref}` at " f"[`{base.sha[:10]}`](https://github.com/apache/airflow/commit/{base.sha}), " @@ -320,6 +342,10 @@ def tested_base_text(base: BaseCheck, *, base_ref: str) -> str: def ci_passed_description(base: BaseCheck, *, base_ref: str) -> str: + if base.merged: + return ( + f"{_FORK_CI_PASSED_PREFIX}; merged with {base_ref}@{base.sha[:10]}, {base.behind} commits behind" + ) return ( f"{_FORK_CI_PASSED_PREFIX}; base {base_ref}@{base.sha[:10]} was {base.behind} commits behind, " "no overlap" diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py index 61a3036621943..4bd11898f3fc3 100644 --- a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py +++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py @@ -36,7 +36,7 @@ from datetime import UTC, datetime, timedelta from enum import Enum from pathlib import Path -from typing import Any +from typing import Any, NamedTuple from urllib.parse import quote import yaml @@ -128,6 +128,13 @@ ) GATED_RUN_JOB_NAMES = frozenset[str]().union(*GATED_RUN_JOBS) CHECK_RUN_NEEDED_JOB_NAME = "Check if the run is needed" +BUILD_INFO_JOB_NAME = "Build info" +# The step of build-info that names the base branch commit and merge base every job of a fork run +# merges the head commit with (ci-amd.yml). +TESTED_MERGE_STEP = re.compile( + r"^Test merged with apache/airflow (?P\S+)@(?P[0-9a-f]{40}), " + r"merge base (?P[0-9a-f]{40})$" +) PROJECT_CI_NOT_RUN_JOB_NAME = "CI passed (project CI not run)" CI_PASSED_CANCELLED_JOB_NAME = "CI passed (run cancelled)" BOT_LOGINS = frozenset( @@ -1895,31 +1902,78 @@ def find_failed_jobs(jobs: Iterable[dict[str, Any]]) -> list[FailedJob]: ReadJobs = dict[int, list[dict[str, Any]] | None] -def read_failed_jobs( +def read_run_jobs( client: GitHubClient, pr: PullRequestSnapshot, fork: ForkObservation, read_jobs: ReadJobs, -) -> list[FailedJob] | None: +) -> list[dict[str, Any]] | None: """ - ``find_failed_jobs`` of the pull request's fork run; None when its jobs cannot be read. + The jobs of the pull request's fork run; None when they cannot be read. :param read_jobs: the jobs already read in this reconciler run; a read made here is recorded too. """ if fork.run_id is None or not pr.head_repo: - return [] - if fork.run_id in read_jobs: - jobs = read_jobs[fork.run_id] - else: + return None + if fork.run_id not in read_jobs: try: - jobs = client.get_run_jobs(pr.head_repo, fork.run_id) + read_jobs[fork.run_id] = client.get_run_jobs(pr.head_repo, fork.run_id) except GitHubError as e: console_print(f"[warning]#{pr.number}: could not read the jobs of {fork.run_url}: {e}[/]") - jobs = None - read_jobs[fork.run_id] = jobs + read_jobs[fork.run_id] = None + return read_jobs[fork.run_id] + + +def read_failed_jobs( + client: GitHubClient, + pr: PullRequestSnapshot, + fork: ForkObservation, + read_jobs: ReadJobs, +) -> list[FailedJob] | None: + """``find_failed_jobs`` of the pull request's fork run; None when its jobs cannot be read.""" + if fork.run_id is None or not pr.head_repo: + return [] + jobs = read_run_jobs(client, pr, fork, read_jobs) return None if jobs is None else find_failed_jobs(jobs) +class ForkRunMerge(NamedTuple): + """The base branch commit a fork run merged the head commit with, as its build-info job names it.""" + + base_ref: str + base_sha: str + merge_base: str + + +def find_tested_merge(jobs: Iterable[dict[str, Any]]) -> ForkRunMerge | None: + """The merge a fork run tested, from the name of a step of its `Build info` job; None if it named none.""" + for job in jobs: + if job.get("name") != BUILD_INFO_JOB_NAME: + continue + for step in job.get("steps") or []: + match = TESTED_MERGE_STEP.match(step.get("name") or "") + if match and step.get("conclusion") == "success": + return ForkRunMerge(match["base_ref"], match["base_sha"], match["merge_base"]) + return None + + +def read_tested_merge( + client: GitHubClient, + pr: PullRequestSnapshot, + fork: ForkObservation, + read_jobs: ReadJobs, +) -> ForkRunMerge | None: + """ + ``find_tested_merge`` of the pull request's fork run. + + :raises GitHubError: when its jobs cannot be read, which says nothing about whether it merged. + """ + jobs = read_run_jobs(client, pr, fork, read_jobs) + if jobs is None: + raise GitHubError(f"could not read the jobs of {fork.run_url}") + return find_tested_merge(jobs) + + class JobStatusReporter: """Reports each job of a pull request's fork run as a commit status, within a per-run budget.""" @@ -2559,17 +2613,75 @@ def pull_request_changes_ci_files(client: GitHubClient, repo: str, number: int, return changes_ci_definition(changed_paths(files), file_count=len(files)) -def compare_with_base(client: GitHubClient, repo: str, pr: PullRequestSnapshot) -> BaseCheck | None: +def compare_tested_merge( + client: GitHubClient, repo: str, pr: PullRequestSnapshot, tested: ForkRunMerge +) -> BaseCheck | None: + """ + Check the merge a fork run reports it tested against GitHub; None when it does not hold. + + The run's own report is trusted as far as its green result is: both come from the workflow of + the head commit, which a pull request changing CI-defining files does not get trusted with. The + reported commit must still be on the pull request's base branch and have the reported merge base + with the head commit, so that the merge is the one project CI would test. + """ + if tested.base_ref != pr.base_ref: + console_print( + f"[warning]#{pr.number}: the fork run merged with {tested.base_ref}, not {pr.base_ref}[/]" + ) + return None + owner = (pr.head_repo or "").split("/", 1)[0] + base_ref = quote(pr.base_ref, safe="") + try: + head_side = client.rest_get( + f"repos/{repo}/compare/{tested.base_sha}...{owner}:{pr.head_sha}?per_page=1" + ) + base_side = client.rest_get(f"repos/{repo}/compare/{tested.base_sha}...{base_ref}?per_page=1") + merge_base = head_side["merge_base_commit"]["sha"] + status = base_side["status"] + behind = base_side["ahead_by"] + except (GitHubError, KeyError, TypeError) as e: + console_print(f"[warning]#{pr.number}: could not check the merge the fork run tested: {e}[/]") + return None + if merge_base != tested.merge_base or status not in ("ahead", "identical"): + console_print( + f"[warning]#{pr.number}: the fork run's merge with {tested.base_sha} does not match GitHub " + f"(merge base {merge_base}, {pr.base_ref} {status})[/]" + ) + return None + return BaseCheck(tested.base_sha, behind, (), merged=True) + + +def compare_with_base( + client: GitHubClient, + repo: str, + pr: PullRequestSnapshot, + *, + find_tested_merge: Callable[[], ForkRunMerge | None] = lambda: None, +) -> BaseCheck | None: """ Compare the base a fork run tested with the base branch; None when not readable. - The fork run tests the head commit, so its base is the merge base with the base branch. Two + A fork run that reports the merge it tested is checked with ``compare_tested_merge``, and gets + None when the merge does not hold. Otherwise it tested the head commit, so its base is the merge + base with the base branch. Two three-dot compares from it give the files the pull request changes and the files the base branch changed since; a file in both means the merged result differs from what the fork run tested in a file the pull request changes. Compares list at most 300 files (on their first page, which ``per_page`` does not limit): a pull request with more is listed with the files endpoint instead, while a base branch that changed more cannot be checked until the branch is rebased. + + :param find_tested_merge: the merge the fork run reports it tested, None when it reports none; + raises GitHubError when the report cannot be read. """ + try: + tested = find_tested_merge() + except GitHubError as e: + console_print(f"[warning]#{pr.number}: could not read which merge the fork run tested: {e}[/]") + return None + if tested is not None: + # The jobs tested that merge, not the head commit as it is, so the head commit's own base + # says nothing about what was tested. + return compare_tested_merge(client, repo, pr, tested) owner = (pr.head_repo or "").split("/", 1)[0] base_ref = quote(pr.base_ref, safe="") try: @@ -2956,7 +3068,13 @@ def is_gated_base(base_ref: str) -> bool: changes_ci_files=functools.partial( pull_request_changes_ci_files, client, repo, pr.number, pr.head_sha ), - compare_base=functools.partial(compare_with_base, client, repo, pr), + compare_base=functools.partial( + compare_with_base, + client, + repo, + pr, + find_tested_merge=functools.partial(read_tested_merge, client, pr, fork, read_jobs), + ), jobs_reported=jobs_reported, head_shared_across_bases=pr.head_sha in shared_heads, find_gate_change=functools.partial(gate_change_finder.find, pr), diff --git a/dev/breeze/src/airflow_breeze/utils/reproduce_ci.py b/dev/breeze/src/airflow_breeze/utils/reproduce_ci.py index 87dd17f484ba2..3edc5b55aa00f 100644 --- a/dev/breeze/src/airflow_breeze/utils/reproduce_ci.py +++ b/dev/breeze/src/airflow_breeze/utils/reproduce_ci.py @@ -20,6 +20,7 @@ import os import shlex +import subprocess from dataclasses import dataclass import click @@ -134,9 +135,37 @@ def build_reproduction_command_from_context( return ReproductionCommand(argv=argv, comment=comment) +def read_head_parents() -> list[str]: + """The parents of the checked-out commit; empty when git cannot tell.""" + result = subprocess.run( + ["git", "rev-list", "--parents", "-n", "1", "HEAD"], capture_output=True, text=True, check=False + ) + return result.stdout.split()[1:] if result.returncode == 0 else [] + + def build_checkout_reproduction_commands(github_repository: str) -> list[ReproductionCommand]: """Build git commands needed to reproduce the current CI checkout locally.""" current_commit_sha = os.environ.get("GITHUB_SHA") or os.environ.get("COMMIT_SHA") or commit_sha() + # A fork CI run tests the pushed commit merged with a base branch commit (merge-with-base action); + # the merge commit is made in the job and never pushed, so it is made again locally. + parents = read_head_parents() if os.environ.get("GITHUB_SHA") else [] + if len(parents) == 2 and parents[1] == current_commit_sha: + base_sha = parents[0] + return [ + ReproductionCommand( + argv=["git", "fetch", f"https://github.com/{github_repository}.git", current_commit_sha], + comment="Fetch the pushed commit", + ), + ReproductionCommand( + argv=["git", "fetch", f"https://github.com/{APACHE_AIRFLOW_GITHUB_REPOSITORY}.git", base_sha], + comment="Fetch the base branch commit fork CI merged it with", + ), + ReproductionCommand(argv=["git", "checkout", current_commit_sha]), + ReproductionCommand( + argv=["git", "merge", "--no-edit", base_sha], + comment="Merge them as fork CI did", + ), + ] github_ref = os.environ.get("GITHUB_REF", "") github_ref_parts = github_ref.split("/") if len(github_ref_parts) == 4 and github_ref_parts[:2] == ["refs", "pull"]: diff --git a/dev/breeze/tests/test_fork_ci_messages.py b/dev/breeze/tests/test_fork_ci_messages.py index d945dfe35766c..283199209ffa7 100644 --- a/dev/breeze/tests/test_fork_ci_messages.py +++ b/dev/breeze/tests/test_fork_ci_messages.py @@ -69,6 +69,7 @@ pytest.param(MARKER, id="without-base"), pytest.param(replace(MARKER, base=BaseCheck("def456", 12, ("a.py",))), id="base"), pytest.param(replace(MARKER, base=BaseCheck("def456", 12, None)), id="base-unchecked"), + pytest.param(replace(MARKER, base=BaseCheck("def456", 12, merged=True)), id="base-merged"), pytest.param(replace(MARKER, jobs="123:SUCCESS"), id="jobs"), pytest.param(replace(MARKER, failed_run="123:2"), id="failed-run"), pytest.param(replace(MARKER, gate_checked=True), id="gate-checked"), @@ -83,6 +84,10 @@ def test_marker_without_base_keeps_its_rendering(): assert "jobs" not in render_marker(MARKER) +def test_marker_of_an_unmerged_base_keeps_its_rendering(): + assert "merged" not in render_marker(replace(MARKER, base=BaseCheck("def456", 12))) + + def test_marker_from_before_the_base_check_has_no_base(): marker = parse_marker( '' @@ -99,6 +104,9 @@ def test_marker_from_before_the_base_check_has_no_base(): pytest.param(BaseCheck("d", MAX_BEHIND_COMMITS), False, id="far-behind"), pytest.param(BaseCheck("d", 1, ("a.py",)), False, id="overlap"), pytest.param(BaseCheck("d", 1, None), False, id="unchecked"), + pytest.param(BaseCheck("d", MAX_BEHIND_COMMITS - 1, merged=True), True, id="merged-close"), + pytest.param(BaseCheck("d", MAX_BEHIND_COMMITS, merged=True), False, id="merged-far-behind"), + pytest.param(BaseCheck("d", 1, None, merged=True), True, id="merged-overlap-not-checked"), ], ) def test_base_check_is_current(base, current): @@ -142,6 +150,24 @@ def test_base_check_is_current(base, current): pytest.param( "green", BaseCheck("d", 4, None), ["Rebase needed", "too many files"], [], id="unchecked" ), + pytest.param( + "green", + BaseCheck("d" * 40, 3, merged=True), + [ + "merged with `v3-1-test` at [`dddddddddd`](https://github.com/apache/airflow/commit/" + + "d" * 40, + "as project CI would, 3 commits", + ], + ["Rebase needed", "as it is"], + id="merged", + ), + pytest.param( + "green", + BaseCheck("d", MAX_BEHIND_COMMITS, merged=True), + ["merged with", "Rebase needed", "too far behind", "git rebase upstream/v3-1-test"], + [], + id="merged-far-behind", + ), pytest.param("failed", BaseCheck("d", MAX_BEHIND_COMMITS), [], ["tested this branch"], id="failed"), pytest.param("green", None, [], ["tested this branch"], id="not-compared"), ], @@ -230,6 +256,11 @@ def test_strip_failed_job_list_leaves_the_comment_without_the_list(): [ pytest.param("Fork CI passed for this commit", True, id="before-the-base-check"), pytest.param(ci_passed_description(BaseCheck("a" * 40, 3), base_ref="main"), True, id="current-base"), + pytest.param( + ci_passed_description(BaseCheck("a" * 40, 3, merged=True), base_ref="main"), + True, + id="merged-base", + ), pytest.param( rebase_needed_description(BaseCheck("a" * 40, 300), base_ref="main"), True, id="rebase-needed" ), @@ -304,6 +335,7 @@ def test_status_descriptions_fit_github_limit(): *_STATUS_DESCRIPTIONS.values(), CI_PASSED_REVOKED_DESCRIPTION, ci_passed_description(BaseCheck("d" * 40, 199), base_ref=base_ref), + ci_passed_description(BaseCheck("d" * 40, 199, merged=True), base_ref=base_ref), *( rebase_needed_description(BaseCheck("d" * 40, 9999, overlap), base_ref=base_ref) for overlap in ((), ("a",) * 300, None) @@ -352,3 +384,9 @@ def test_ci_modification_note_mentions_fork_execution_only_when_gated(gated): assert (GATED_NOTE_SENTENCE in comment) is gated assert comment.endswith(FOOTER) + + +def test_ci_passed_description_names_the_merged_base(): + description = ci_passed_description(BaseCheck("a" * 40, 3, merged=True), base_ref="main") + + assert description == "Fork CI passed; merged with main@aaaaaaaaaa, 3 commits behind" diff --git a/dev/breeze/tests/test_fork_ci_reconcile.py b/dev/breeze/tests/test_fork_ci_reconcile.py index 4e2ddd2bf9038..fd9fd8978845d 100644 --- a/dev/breeze/tests/test_fork_ci_reconcile.py +++ b/dev/breeze/tests/test_fork_ci_reconcile.py @@ -62,6 +62,7 @@ BypassFinder, DraftChange, ForkObservation, + ForkRunMerge, ForkState, GateChangeFinder, JobStatusReporter, @@ -90,6 +91,7 @@ find_project_ci_passed_url, find_project_ci_run_url, find_state_comment, + find_tested_merge, is_cancelled_gate_run, is_gated_skip_run, is_own_pilot_change, @@ -105,6 +107,7 @@ read_job_status_failure_runs, read_project_ci_passed_url, read_project_ci_run_url, + read_tested_merge, report_project_ci_passed, report_project_ci_runs_started, run_reconcile, @@ -2957,6 +2960,137 @@ def test_compare_with_base(client, responses, expected, expected_calls): assert [call.args[0] for call in client.rest_get.call_args_list] == expected_calls +TESTED_BASE = "b" * 40 +TESTED_MERGE_BASE = "e" * 40 +TESTED = ForkRunMerge("providers-fab/v1-5", TESTED_BASE, TESTED_MERGE_BASE) +TESTED_HEAD_COMPARE = f"repos/{REPO}/compare/{TESTED_BASE}...someone:abc123?per_page=1" +TESTED_BASE_COMPARE = f"repos/{REPO}/compare/{TESTED_BASE}...providers-fab%2Fv1-5?per_page=1" + + +def _tested_base(status="ahead", ahead_by=7): + return {"status": status, "ahead_by": ahead_by} + + +@pytest.mark.parametrize( + ("tested", "responses", "expected"), + [ + pytest.param( + TESTED, + [_compare(0, [], merge_base=TESTED_MERGE_BASE), _tested_base()], + BaseCheck(TESTED_BASE, 7, (), merged=True), + id="merged", + ), + pytest.param( + TESTED, + [_compare(0, [], merge_base=TESTED_MERGE_BASE), _tested_base("identical", 0)], + BaseCheck(TESTED_BASE, 0, (), merged=True), + id="merged-with-the-tip", + ), + pytest.param( + TESTED._replace(base_ref="main"), + [], + None, + id="merged-with-another-branch", + ), + pytest.param( + TESTED, + [_compare(0, [], merge_base="c" * 40), _tested_base()], + None, + id="other-merge-base", + ), + pytest.param( + TESTED, + [_compare(0, [], merge_base=TESTED_MERGE_BASE), _tested_base("diverged")], + None, + id="not-on-the-base-branch", + ), + pytest.param( + TESTED, + [GitHubError("gh api failed: HTTP 500")], + None, + id="merge-not-checkable", + ), + ], +) +def test_compare_with_base_checks_the_merge_the_fork_run_tested(client, tested, responses, expected): + client.rest_get.side_effect = responses + + check = compare_with_base( + client, REPO, _pr(base_ref="providers-fab/v1-5"), find_tested_merge=lambda: tested + ) + + assert check == expected + if expected is not None: + assert [call.args[0] for call in client.rest_get.call_args_list] == [ + TESTED_HEAD_COMPARE, + TESTED_BASE_COMPARE, + ] + + +def _build_info(*steps): + return { + "name": "Build info", + "steps": [{"name": name, "conclusion": conclusion} for name, conclusion in steps], + } + + +MERGE_STEP = ( + f"Test merged with apache/airflow providers-fab/v1-5@{TESTED_BASE}, merge base {TESTED_MERGE_BASE}" +) + + +@pytest.mark.parametrize( + ("jobs", "expected"), + [ + pytest.param([_build_info(("Checkout", "success"), (MERGE_STEP, "success"))], TESTED, id="named"), + pytest.param([_build_info((MERGE_STEP, "skipped"))], None, id="step-skipped"), + pytest.param( + [_build_info(("Test merged with apache/airflow main@, merge base ", "skipped"))], + None, + id="no-merge", + ), + pytest.param( + [{"name": "Static checks", "steps": [{"name": MERGE_STEP, "conclusion": "success"}]}], + None, + id="other-job", + ), + pytest.param([{"name": "Build info"}], None, id="steps-not-listed"), + ], +) +def test_find_tested_merge(jobs, expected): + assert find_tested_merge(jobs) == expected + + +def test_read_tested_merge_reuses_the_jobs_read_in_this_run(client): + read_jobs = {1: [_build_info((MERGE_STEP, "success"))]} + + assert read_tested_merge(client, _pr(), COMPLETED_RUN, read_jobs) == TESTED + client.get_run_jobs.assert_not_called() + + +def test_read_tested_merge_reads_the_jobs_once(client): + client.get_run_jobs.return_value = [_build_info((MERGE_STEP, "success"))] + read_jobs: dict = {} + + assert read_tested_merge(client, _pr(), COMPLETED_RUN, read_jobs) == TESTED + assert read_jobs == {1: client.get_run_jobs.return_value} + + +def test_read_tested_merge_without_readable_jobs(client): + client.get_run_jobs.side_effect = GitHubError("gh api failed: HTTP 502") + + with pytest.raises(GitHubError): + read_tested_merge(client, _pr(), COMPLETED_RUN, {}) + + +def test_compare_with_base_waits_for_a_readable_merge_report(client): + def unreadable(): + raise GitHubError("could not read the jobs") + + assert compare_with_base(client, REPO, _pr(), find_tested_merge=unreadable) is None + client.rest_get.assert_not_called() + + def _fork_job(name, status="completed", conclusion="success", url=None): return { "name": name, @@ -5038,3 +5172,17 @@ def test_monitor_interval_is_the_monitor_workflow_cadence(): crons = [entry["cron"] for entry in workflow[True]["schedule"]] assert crons == [f"*/{MONITOR_INTERVAL // timedelta(minutes=1)} * * * *"] + + +@mock.patch("airflow_breeze.utils.fork_ci_reconcile.JobStatusReporter", autospec=True) +@mock.patch("airflow_breeze.utils.fork_ci_reconcile.classify", autospec=True) +def test_run_reconcile_compares_the_merge_the_fork_run_tested(mock_classify, mock_reporter, client, tmp_path): + mock_classify.return_value = Plan(state=ForkState.RUNNING) + mock_reporter.return_value.report.return_value = None + mock_reporter.return_value.deferred = [] + mock_reporter.return_value.delayed = [] + + _reconcile_with_reporter(client, tmp_path) + + compare_base = mock_classify.call_args.kwargs["compare_base"] + assert compare_base.keywords["find_tested_merge"].func is read_tested_merge diff --git a/dev/breeze/tests/test_reproduce_ci.py b/dev/breeze/tests/test_reproduce_ci.py index dd6490d3ae4a4..d17819f27ee4b 100644 --- a/dev/breeze/tests/test_reproduce_ci.py +++ b/dev/breeze/tests/test_reproduce_ci.py @@ -106,7 +106,8 @@ def test_build_checkout_reproduction_commands_fetches_pull_request_ref(pr_ref_ki assert commands[1].argv == ["git", "checkout", "merge-sha"] -def test_build_checkout_reproduction_commands_plain_sha(monkeypatch): +@mock.patch("airflow_breeze.utils.reproduce_ci.read_head_parents", autospec=True, return_value=["abc123"]) +def test_build_checkout_reproduction_commands_plain_sha(mock_parents, monkeypatch): monkeypatch.delenv("GITHUB_REF", raising=False) monkeypatch.setenv("GITHUB_SHA", "def456") @@ -116,6 +117,35 @@ def test_build_checkout_reproduction_commands_plain_sha(monkeypatch): assert commands[0].argv == ["git", "checkout", "def456"] +@pytest.mark.parametrize( + ("parents", "merged"), + [ + pytest.param(["base789", "def456"], True, id="fork-ci-merge"), + pytest.param(["def456", "other"], False, id="other-merge"), + pytest.param([], False, id="parents-unknown"), + ], +) +@mock.patch("airflow_breeze.utils.reproduce_ci.read_head_parents", autospec=True) +def test_build_checkout_reproduction_commands_recreates_the_fork_ci_merge( + mock_parents, parents, merged, monkeypatch +): + mock_parents.return_value = parents + monkeypatch.setenv("GITHUB_REF", "refs/heads/my-change") + monkeypatch.setenv("GITHUB_SHA", "def456") + + commands = build_checkout_reproduction_commands("someone/airflow") + + if merged: + assert [command.argv for command in commands] == [ + ["git", "fetch", "https://github.com/someone/airflow.git", "def456"], + ["git", "fetch", "https://github.com/apache/airflow.git", "base789"], + ["git", "checkout", "def456"], + ["git", "merge", "--no-edit", "base789"], + ] + else: + assert [command.argv for command in commands] == [["git", "checkout", "def456"]] + + @mock.patch("airflow_breeze.utils.reproduce_ci.get_console", autospec=True) def test_print_local_reproduction_renders_copyable_commands(mock_get_console, monkeypatch): monkeypatch.setenv("CI", "true") diff --git a/scripts/ci/prek/check_fork_ci_tests_merge.py b/scripts/ci/prek/check_fork_ci_tests_merge.py new file mode 100755 index 0000000000000..baa7652a43897 --- /dev/null +++ b/scripts/ci/prek/check_fork_ci_tests_merge.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.11,<3.12" +# dependencies = [ +# "pyyaml>=6.0.3", +# "rich>=13.6.0", +# ] +# /// +""" +Check that every job of a fork CI run tests the pushed commit merged with the base branch. + +A fork run checks out the pushed commit; the ``merge-with-base`` action then replaces it with its +merge into the base branch, which is what a pull_request run of project CI tests (AIP-120). The +check covers ``ci-amd.yml``, ``ci-arm.yml`` and every workflow they call, directly or not: + +- every call of a reusable workflow passes ``merge-with``, and every reusable workflow takes it; +- in every job that checks out the pushed commit into the workspace root, the merge step follows + the last such checkout, with an ``if:`` that runs it only when ``merge-with`` is set. + +A checkout of another ref (a constraints branch, the gate configuration) or into a sub-directory is +not the pull request's sources and is not counted. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path +from typing import Any + +import yaml +from rich.console import Console + +console = Console(color_system="standard", width=200) + +WORKFLOWS_DIR = Path(__file__).resolve().parents[3] / ".github" / "workflows" +TOP_WORKFLOWS = ("ci-amd.yml", "ci-arm.yml") +MERGE_ACTION = "./.github/actions/merge-with-base" +# The same action, read from a sparse checkout of the workflow's own version where a job checks out +# another ref into the workspace, so that no code of that ref runs as an action. +WORKFLOW_VERSION_MERGE_ACTION = "./workflow-version/.github/actions/merge-with-base" +REUSABLE_WORKFLOW = re.compile(r"^\./\.github/workflows/([\w.-]+\.yml)$") +# Refs a checkout of the pushed commit is written with. +PUSHED_COMMIT_REFS = frozenset({"", "${{ github.sha }}", "${{ inputs.checkout-ref }}"}) +# Jobs that read the pushed commit itself rather than test it. +EXEMPT_JOBS = { + **{(top, "gate"): "reads the gate configuration" for top in TOP_WORKFLOWS}, + **{ + (top, "build-info"): "selects tests from the pushed commit's changes and resolves the merge base" + for top in TOP_WORKFLOWS + }, + **{(top, "notify-slack"): "only reports on the run" for top in TOP_WORKFLOWS}, + **{(top, "summarize-warnings"): "only reports on the run" for top in TOP_WORKFLOWS}, +} + + +def load_workflow(name: str, workflows_dir: Path) -> dict[str, Any]: + return yaml.safe_load((workflows_dir / name).read_text()) + + +def find_called_workflows(workflow: dict[str, Any]) -> set[str]: + called = set() + for job in (workflow.get("jobs") or {}).values(): + match = REUSABLE_WORKFLOW.match(str(job.get("uses", ""))) + if match: + called.add(match.group(1)) + return called + + +def find_fork_run_workflows(workflows_dir: Path) -> dict[str, dict[str, Any]]: + """The workflows a fork run executes: the top ones and every workflow they call.""" + found: dict[str, dict[str, Any]] = {} + pending = list(TOP_WORKFLOWS) + while pending: + name = pending.pop() + if name in found: + continue + found[name] = load_workflow(name, workflows_dir) + pending.extend(find_called_workflows(found[name])) + return found + + +def is_pushed_commit_checkout(step: dict[str, Any]) -> bool: + if not str(step.get("uses", "")).startswith("actions/checkout@"): + return False + options = step.get("with") or {} + return options.get("path", ".") in (".", "") and str(options.get("ref", "")) in PUSHED_COMMIT_REFS + + +def check_job(workflow_name: str, job_id: str, job: dict[str, Any]) -> list[str]: + if REUSABLE_WORKFLOW.match(str(job.get("uses", ""))): + if "merge-with" not in (job.get("with") or {}): + return [f"job [yellow]{job_id}[/] calls {job['uses']} without passing `merge-with`"] + return [] + if (workflow_name, job_id) in EXEMPT_JOBS: + return [] + steps = job.get("steps") or [] + checkouts = [index for index, step in enumerate(steps) if is_pushed_commit_checkout(step)] + if not checkouts: + return [] + merges = [ + index + for index, step in enumerate(steps) + if step.get("uses") in (MERGE_ACTION, WORKFLOW_VERSION_MERGE_ACTION) + ] + if not any(index > checkouts[-1] for index in merges): + return [ + f"job [yellow]{job_id}[/] checks out the pushed commit without a `{MERGE_ACTION}` step " + "after its last such checkout" + ] + # The action is read from the checkout, which an older ref or a skipped checkout lacks, so it may + # only be loaded when there is something to merge. + return [ + f"job [yellow]{job_id}[/] runs `{MERGE_ACTION}` without an `if:` that checks `merge-with`" + for index in merges + if "merge-with" not in str(steps[index].get("if", "")) + ] + + +def check_workflow(name: str, workflow: dict[Any, Any]) -> list[str]: + errors = [] + if name not in TOP_WORKFLOWS: + # PyYAML reads the bare `on:` key as boolean True. + triggers = workflow.get("on", workflow.get(True)) or {} + call = triggers.get("workflow_call") if isinstance(triggers, dict) else None + if "merge-with" not in ((call or {}).get("inputs") or {}): + errors.append("does not take a `merge-with` input") + for job_id, job in (workflow.get("jobs") or {}).items(): + errors.extend(check_job(name, job_id, job)) + return errors + + +def main(workflows_dir: Path = WORKFLOWS_DIR) -> int: + errors = [ + f"[red].github/workflows/{name}[/]: {error}" + for name, workflow in sorted(find_fork_run_workflows(workflows_dir).items()) + for error in check_workflow(name, workflow) + ] + if not errors: + return 0 + for error in errors: + console.print(error) + console.print( + "\nFork CI tests the pushed commit merged with the base branch, so every job of a fork run must " + f"merge it after checking it out: add a `uses: {MERGE_ACTION}` step taking `merge-with`, pass " + "`merge-with` to called workflows, or add the job to EXEMPT_JOBS in " + "scripts/ci/prek/check_fork_ci_tests_merge.py with the reason it does not test the sources." + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/ci/prek/test_check_fork_ci_tests_merge.py b/scripts/tests/ci/prek/test_check_fork_ci_tests_merge.py new file mode 100644 index 0000000000000..24123775e6164 --- /dev/null +++ b/scripts/tests/ci/prek/test_check_fork_ci_tests_merge.py @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest +import yaml +from check_fork_ci_tests_merge import MERGE_ACTION, check_job, check_workflow, main + +CHECKOUT = {"uses": "actions/checkout@v7"} +MERGE = { + "if": "inputs.merge-with != ''", + "uses": MERGE_ACTION, + "with": {"merge-with": "${{ inputs.merge-with }}"}, +} + + +@pytest.mark.parametrize( + ("steps", "valid"), + [ + pytest.param([CHECKOUT, MERGE], True, id="merge-after-checkout"), + pytest.param([CHECKOUT], False, id="no-merge"), + pytest.param( + [CHECKOUT, {key: value for key, value in MERGE.items() if key != "if"}], + False, + id="merge-unguarded", + ), + pytest.param([MERGE, CHECKOUT], False, id="merge-before-checkout"), + pytest.param( + [CHECKOUT, MERGE, {"uses": "actions/checkout@v7", "with": {"ref": "${{ github.sha }}"}}], + False, + id="checkout-again-after-merge", + ), + pytest.param( + [{"uses": "actions/checkout@v7", "with": {"ref": "${{ inputs.checkout-ref }}"}}, MERGE], + True, + id="checkout-ref-input", + ), + pytest.param( + [CHECKOUT, MERGE, {"uses": "actions/checkout@v7", "with": {"path": "constraints"}}], + True, + id="checkout-into-sub-directory", + ), + pytest.param( + [{"uses": "actions/checkout@v7", "with": {"ref": "constraints-main"}}], + True, + id="checkout-of-another-ref", + ), + pytest.param([{"run": "echo"}], True, id="no-checkout"), + pytest.param( + [ + CHECKOUT, + {"uses": "actions/checkout@v7", "with": {"path": "workflow-version"}}, + {**MERGE, "uses": "./workflow-version/.github/actions/merge-with-base"}, + ], + True, + id="merge-action-from-the-workflow-version", + ), + ], +) +def test_check_job_steps(steps, valid): + assert (check_job("tests.yml", "job", {"steps": steps}) == []) is valid + + +@pytest.mark.parametrize( + ("job", "valid"), + [ + pytest.param({"uses": "./.github/workflows/x.yml", "with": {"merge-with": "a b"}}, True, id="passes"), + pytest.param({"uses": "./.github/workflows/x.yml", "with": {}}, False, id="missing"), + pytest.param({"uses": "./.github/workflows/x.yml"}, False, id="no-with"), + ], +) +def test_check_job_calls(job, valid): + assert (check_job("tests.yml", "job", job) == []) is valid + + +@pytest.mark.parametrize("job_id", ["gate", "build-info", "notify-slack", "summarize-warnings"]) +def test_exempt_jobs_of_the_top_workflows(job_id): + assert check_job("ci-amd.yml", job_id, {"steps": [CHECKOUT]}) == [] + assert check_job("other.yml", job_id, {"steps": [CHECKOUT]}) != [] + + +@pytest.mark.parametrize( + ("inputs", "valid"), + [ + pytest.param({"merge-with": {"type": "string"}}, True, id="takes-it"), + pytest.param({"other": {"type": "string"}}, False, id="lacks-it"), + ], +) +def test_reusable_workflows_take_merge_with(inputs, valid): + workflow = {True: {"workflow_call": {"inputs": inputs}}, "jobs": {}} + assert (check_workflow("reusable.yml", workflow) == []) is valid + + +def _write(path, workflow): + path.write_text(yaml.safe_dump(workflow)) + + +def test_main_follows_called_workflows(tmp_path): + call = {"uses": "./.github/workflows/called.yml", "with": {"merge-with": "a b"}} + for top in ("ci-amd.yml", "ci-arm.yml"): + _write(tmp_path / top, {"on": {"push": {}}, "jobs": {"tests": call}}) + _write( + tmp_path / "called.yml", + {"on": {"workflow_call": {"inputs": {"merge-with": {}}}}, "jobs": {"j": {"steps": [CHECKOUT]}}}, + ) + _write(tmp_path / "unrelated.yml", {"on": {"push": {}}, "jobs": {"j": {"steps": [CHECKOUT]}}}) + + assert main(tmp_path) == 1 + + _write( + tmp_path / "called.yml", + { + "on": {"workflow_call": {"inputs": {"merge-with": {}}}}, + "jobs": {"j": {"steps": [CHECKOUT, MERGE]}}, + }, + ) + + assert main(tmp_path) == 0