Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions .github/actions/merge-with-base/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: 'Merge with the base branch'
description: >-
Replace the checked-out pushed commit of a fork CI run with its merge into the base branch, the
commit a pull_request run of project CI tests (AIP-120). Callers run it only when merge-with is
set: the action is read from the checkout, which an older ref or a skipped checkout lacks.
inputs:
merge-with:
description: >-
The base branch commit and the merge base with the pushed commit, space-separated, as
build-info resolved them for the whole run; empty outside fork runs.
required: true
outputs:
merge-sha:
description: "The merge commit now checked out; empty when nothing was merged."
value: ${{ steps.merge.outputs.merge-sha }}
runs:
using: "composite"
steps:
- name: "Merge the pushed commit with the base branch"
id: merge
if: inputs.merge-with != ''
shell: bash
env:
MERGE_WITH: ${{ inputs.merge-with }}
# The merge base is given, so the three commits are all that is fetched: no history between
# them is needed. Identity and dates are fixed, so every job of a run makes the same commit.
# The base branch is the first parent, as in GitHub's refs/pull/<n>/merge, so `<merge>^`
# is the base and `<merge>^..<merge>` is what the pull request changes.
run: |
set -euo pipefail
read -r base_sha merge_base <<< "${MERGE_WITH}"
head_sha="$(git rev-parse HEAD)"
for attempt in 1 2 3; do
if git fetch --no-tags --depth=1 https://github.com/apache/airflow.git \
"${base_sha}" "${merge_base}"; then
break
fi
if [[ "${attempt}" -eq 3 ]]; then
echo "::error::Could not fetch ${base_sha} and ${merge_base} from apache/airflow"
exit 1
fi
sleep $((attempt * 5))
done
if ! tree="$(git merge-tree --write-tree --merge-base "${merge_base}" \
"${base_sha}" "${head_sha}")"; then
echo "::error::The pushed commit conflicts with apache/airflow ${base_sha}; rebase it."
exit 1
fi
commit_date="$(git show -s --format=%cI "${head_sha}")"
merge_sha="$(
GIT_AUTHOR_NAME="Fork CI" GIT_AUTHOR_EMAIL="dev@airflow.apache.org" \
GIT_COMMITTER_NAME="Fork CI" GIT_COMMITTER_EMAIL="dev@airflow.apache.org" \
GIT_AUTHOR_DATE="${commit_date}" GIT_COMMITTER_DATE="${commit_date}" \
git commit-tree "${tree}" -p "${base_sha}" -p "${head_sha}" \
-m "Merge ${head_sha} into apache/airflow ${base_sha}"
)"
git checkout --quiet --detach "${merge_sha}"
echo "merge-sha=${merge_sha}" >> "${GITHUB_OUTPUT}"
echo "Testing ${merge_sha}: ${head_sha} merged into apache/airflow ${base_sha}"
12 changes: 12 additions & 0 deletions .github/workflows/additional-ci-image-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@ name: Additional CI image checks
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
merge-with:
description: >-
The base branch commit and merge base to merge the pushed commit with, space-separated;
empty outside fork CI runs.
required: false
default: ""
type: string
runners:
description: "The array of labels (in json form) determining runners."
required: true
Expand Down Expand Up @@ -81,6 +88,11 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "Merge with the base branch"
if: inputs.merge-with != ''
uses: ./.github/actions/merge-with-base
with:
merge-with: ${{ inputs.merge-with }}
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: "Check that image builds quickly"
Expand Down
45 changes: 45 additions & 0 deletions .github/workflows/additional-prod-image-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@ name: Additional PROD image tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
merge-with:
description: >-
The base branch commit and merge base to merge the pushed commit with, space-separated;
empty outside fork CI runs.
required: false
default: ""
type: string
runners:
description: "The array of labels (in json form) determining runners."
required: true
Expand Down Expand Up @@ -137,6 +144,7 @@ jobs:
name: PROD image extra checks (main)
uses: ./.github/workflows/prod-image-extra-checks.yml
with:
merge-with: ${{ inputs.merge-with }}
runners: ${{ inputs.runners }}
platform: ${{ inputs.platform }}
python-versions: "[ '${{ inputs.default-python-version }}' ]"
Expand All @@ -154,6 +162,7 @@ jobs:
name: PROD image extra checks (release)
uses: ./.github/workflows/prod-image-extra-checks.yml
with:
merge-with: ${{ inputs.merge-with }}
runners: ${{ inputs.runners }}
platform: ${{ inputs.platform }}
python-versions: "[ '${{ inputs.default-python-version }}' ]"
Expand Down Expand Up @@ -186,6 +195,11 @@ jobs:
with:
fetch-depth: 2
persist-credentials: false
- name: "Merge with the base branch"
if: inputs.merge-with != ''
uses: ./.github/actions/merge-with-base
with:
merge-with: ${{ inputs.merge-with }}
- name: "Prepare breeze & PROD image: ${{ inputs.default-python-version }}"
uses: ./.github/actions/prepare_breeze_and_image
with:
Expand Down Expand Up @@ -225,6 +239,11 @@ jobs:
with:
fetch-depth: 2
persist-credentials: false
- name: "Merge with the base branch"
if: inputs.merge-with != ''
uses: ./.github/actions/merge-with-base
with:
merge-with: ${{ inputs.merge-with }}
- name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
uses: ./.github/actions/prepare_breeze_and_image
with:
Expand Down Expand Up @@ -256,6 +275,11 @@ jobs:
with:
fetch-depth: 2
persist-credentials: false
- name: "Merge with the base branch"
if: inputs.merge-with != ''
uses: ./.github/actions/merge-with-base
with:
merge-with: ${{ inputs.merge-with }}
- name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
uses: ./.github/actions/prepare_breeze_and_image
with:
Expand All @@ -274,6 +298,7 @@ jobs:
if: inputs.run-generic-prod-image-tests == 'true'
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
workflow-name: "Regular e2e test"
runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
Expand All @@ -284,6 +309,7 @@ jobs:
name: "Remote logging tests with PROD image"
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
workflow-name: "Remote logging e2e test"
runners: ${{ format(inputs.runners, '4+8', '16+32') }}
platform: ${{ inputs.platform }}
Expand All @@ -298,6 +324,7 @@ jobs:
name: "Elasticsearch remote logging tests with PROD image"
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
workflow-name: "Elasticsearch remote logging e2e test"
runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
Expand All @@ -312,6 +339,7 @@ jobs:
name: "OpenSearch remote logging tests with PROD image"
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
workflow-name: "OpenSearch remote logging e2e test"
runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
Expand All @@ -327,6 +355,7 @@ jobs:
if: inputs.run-generic-prod-image-tests == 'true'
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
workflow-name: "XCom object storage backend e2e test"
runners: ${{ format(inputs.runners, '4+8', '16+32') }}
platform: ${{ inputs.platform }}
Expand All @@ -338,6 +367,7 @@ jobs:
name: "Event driven tests with PROD image"
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
workflow-name: "Event driven e2e test"
runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
Expand All @@ -352,6 +382,7 @@ jobs:
name: "Java SDK e2e tests with PROD image"
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
workflow-name: "Java SDK e2e test"
runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
Expand All @@ -366,6 +397,7 @@ jobs:
name: "Go SDK e2e tests with PROD image"
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
workflow-name: "Go SDK e2e test"
runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
Expand All @@ -380,6 +412,7 @@ jobs:
name: "TypeScript SDK e2e tests with PROD image"
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
workflow-name: "TypeScript SDK e2e test"
runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
Expand All @@ -394,6 +427,7 @@ jobs:
name: "YAML Dag e2e tests with PROD image"
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
workflow-name: "YAML Dag e2e test"
runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
Expand All @@ -408,6 +442,7 @@ jobs:
name: "Language SDK XCom object storage backend tests with PROD image"
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
workflow-name: "Language SDK XCom object storage backend e2e test"
runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
Expand All @@ -423,6 +458,7 @@ jobs:
name: "OpenLineage e2e tests with PROD image"
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
workflow-name: "OpenLineage e2e test"
runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
Expand All @@ -444,6 +480,7 @@ jobs:
compat: ${{ fromJSON(inputs.providers-compatibility-tests-matrix) }}
uses: ./.github/workflows/openlineage-e2e-compat-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
Expand All @@ -455,6 +492,7 @@ jobs:
name: "Chromium UI e2e tests with PROD image"
uses: ./.github/workflows/ui-e2e-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
workflow-name: "Chromium UI e2e tests"
runners: ${{ inputs.runners }}
platform: ${{ inputs.platform }}
Expand All @@ -467,6 +505,7 @@ jobs:
name: "Firefox UI e2e tests with PROD image"
uses: ./.github/workflows/ui-e2e-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
workflow-name: "Firefox UI e2e tests"
runners: ${{ inputs.runners }}
platform: ${{ inputs.platform }}
Expand All @@ -479,6 +518,7 @@ jobs:
name: "WebKit UI e2e tests with PROD image"
uses: ./.github/workflows/ui-e2e-tests.yml
with:
merge-with: ${{ inputs.merge-with }}
workflow-name: "WebKit UI e2e tests"
runners: ${{ inputs.runners }}
platform: ${{ inputs.platform }}
Expand Down Expand Up @@ -506,6 +546,11 @@ jobs:
with:
fetch-depth: 2
persist-credentials: false
- name: "Merge with the base branch"
if: inputs.merge-with != ''
uses: ./.github/actions/merge-with-base
with:
merge-with: ${{ inputs.merge-with }}
- name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
uses: ./.github/actions/prepare_breeze_and_image
with:
Expand Down
12 changes: 12 additions & 0 deletions .github/workflows/agent-framework-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@ name: Agent framework tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
merge-with:
description: >-
The base branch commit and merge base to merge the pushed commit with, space-separated;
empty outside fork CI runs.
required: false
default: ""
type: string
runners:
description: "The array of labels (in json form) determining runners."
required: true
Expand Down Expand Up @@ -75,6 +82,11 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "Merge with the base branch"
if: inputs.merge-with != ''
uses: ./.github/actions/merge-with-base
with:
merge-with: ${{ inputs.merge-with }}
- name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}"
uses: ./.github/actions/prepare_breeze_and_image
with:
Expand Down
12 changes: 12 additions & 0 deletions .github/workflows/airflow-distributions-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@ name: Non-core Distribution tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
merge-with:
description: >-
The base branch commit and merge base to merge the pushed commit with, space-separated;
empty outside fork CI runs.
required: false
default: ""
type: string
# Static inputs defined to choose which distribution to test to run
runners:
description: "The array of labels (in json form) determining runners."
Expand Down Expand Up @@ -95,6 +102,11 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "Merge with the base branch"
if: inputs.merge-with != ''
uses: ./.github/actions/merge-with-base
with:
merge-with: ${{ inputs.merge-with }}
- name: "Prepare breeze & CI image: ${{ matrix.python-version }}"
uses: ./.github/actions/prepare_breeze_and_image
with:
Expand Down
12 changes: 12 additions & 0 deletions .github/workflows/airflow-e2e-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,13 @@ on: # yamllint disable-line rule:truthy

workflow_call:
inputs:
merge-with:
description: >-
The base branch commit and merge base to merge the pushed commit with, space-separated;
empty outside fork CI runs.
required: false
default: ""
type: string
workflow-name:
description: "Name of the test"
type: string
Expand Down Expand Up @@ -112,6 +119,11 @@ jobs:
with:
fetch-depth: 2
persist-credentials: false
- name: "Merge with the base branch"
if: inputs.merge-with != ''
uses: ./.github/actions/merge-with-base
with:
merge-with: ${{ inputs.merge-with }}
- name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
uses: ./.github/actions/prepare_breeze_and_image
with:
Expand Down
Loading
Loading