mirror of
https://github.com/apache/airflow.git
synced 2026-03-26 15:28:46 +00:00
94e189618a
* Fix all build-system/requires including transitive dependencies Add a feature to fix all build-system/requires to make build reproducibility works. The build-system requires are managed automatically by the `upgrade-important-versions` prek hook. This should not only provide reproducibility, but also it should prevent cases where transient dependency upgrade might break building sdist (for example as it happened with virtualenv 21 breaking hatchling with https://github.com/pypa/hatch/issues/2193 * Fix mypy 'Cannot infer type of lambda' error in upgrade script Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
468 lines
19 KiB
YAML
468 lines
19 KiB
YAML
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
---
|
|
name: Basic tests
|
|
on: # yamllint disable-line rule:truthy
|
|
workflow_call:
|
|
inputs:
|
|
runners:
|
|
description: "The array of labels (in json form) determining runners."
|
|
required: true
|
|
type: string
|
|
run-ui-tests:
|
|
description: "Whether to run UI tests (true/false)"
|
|
required: true
|
|
type: string
|
|
run-www-tests:
|
|
description: "Whether to run WWW tests (true/false)"
|
|
required: true
|
|
type: string
|
|
run-api-codegen:
|
|
description: "Whether to run API codegen (true/false)"
|
|
required: true
|
|
type: string
|
|
run-breeze-integration-tests:
|
|
description: "Whether to run breeze integration tests (true/false)"
|
|
required: true
|
|
type: string
|
|
basic-checks-only:
|
|
description: "Whether to run only basic checks (true/false)"
|
|
required: true
|
|
type: string
|
|
skip-prek-hooks:
|
|
description: "Whether to skip prek hooks (true/false)"
|
|
required: true
|
|
type: string
|
|
default-python-version:
|
|
description: "Which version of python should be used by default"
|
|
required: true
|
|
type: string
|
|
shared-distributions-as-json:
|
|
description: "Json array of shared distributions to run tests for"
|
|
required: true
|
|
type: string
|
|
canary-run:
|
|
description: "Whether to run canary tests (true/false)"
|
|
required: true
|
|
type: string
|
|
latest-versions-only:
|
|
description: "Whether to run only latest version checks (true/false)"
|
|
required: true
|
|
type: string
|
|
use-uv:
|
|
description: "Whether to use uv in the image"
|
|
required: true
|
|
type: string
|
|
uv-version:
|
|
description: 'uv version to use'
|
|
default: "0.10.6" # Keep this comment to allow automatic replacement of uv version
|
|
type: string
|
|
platform:
|
|
description: 'Platform for the build - linux/amd64 or linux/arm64'
|
|
required: true
|
|
type: string
|
|
permissions:
|
|
contents: read
|
|
jobs:
|
|
run-breeze-tests:
|
|
timeout-minutes: 10
|
|
name: Breeze unit tests
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
# Need to fetch all history for selective checks tests
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
- name: "Run unit tests"
|
|
run: uv tool run --from apache-airflow-breeze pytest -n auto --color=yes
|
|
working-directory: ./dev/breeze/
|
|
run-breeze-integration-tests:
|
|
timeout-minutes: 120
|
|
name: Breeze integration tests
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
if: inputs.run-breeze-integration-tests == 'true'
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
- name: "Install SVN"
|
|
run: sudo apt-get update && sudo apt-get install -y subversion
|
|
- name: "Install Java (for Apache RAT)"
|
|
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
|
with:
|
|
distribution: 'temurin'
|
|
java-version: '17'
|
|
- name: "Install hatch"
|
|
run: uv tool install hatch
|
|
- name: "Run integration tests"
|
|
run: uv tool run --from apache-airflow-breeze pytest -v --color=yes -m integration_tests
|
|
working-directory: ./dev/breeze/
|
|
tests-shared-distributions:
|
|
timeout-minutes: 10
|
|
name: Shared ${{ matrix.shared-distribution }} tests
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
shared-distribution: ${{ fromJSON(inputs.shared-distributions-as-json) }}
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
steps:
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
fetch-depth: 1
|
|
persist-credentials: false
|
|
- name: "Install uv"
|
|
run: curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh
|
|
env:
|
|
UV_VERSION: ${{ inputs.uv-version }}
|
|
- name: "Run shared ${{ matrix.shared-distribution }} tests"
|
|
run: uv run --group dev pytest --color=yes -n auto
|
|
working-directory: shared/${{ matrix.shared-distribution }}
|
|
tests-ui:
|
|
timeout-minutes: 15
|
|
name: React UI tests
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
if: inputs.run-ui-tests == 'true'
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: Setup pnpm
|
|
uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 # v4.0.0
|
|
with:
|
|
version: 9
|
|
run_install: false
|
|
- name: "Setup node"
|
|
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
|
with:
|
|
node-version: 21
|
|
cache: 'pnpm'
|
|
cache-dependency-path: 'airflow-core/src/airflow/**/pnpm-lock.yaml'
|
|
- name: "Restore eslint cache (ui)"
|
|
uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
|
|
with:
|
|
path: airflow-core/src/airflow/ui/node_modules/
|
|
# yamllint disable-line rule:line-length
|
|
key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow-core/src/airflow/ui/**/pnpm-lock.yaml') }}
|
|
id: restore-eslint-cache-ui
|
|
- run: cd airflow-core/src/airflow/ui && pnpm install --frozen-lockfile
|
|
- run: cd airflow-core/src/airflow/ui && pnpm test
|
|
env:
|
|
FORCE_COLOR: 2
|
|
- name: "Save eslint cache (ui)"
|
|
uses: apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
|
|
with:
|
|
path: airflow-core/src/airflow/ui/node_modules/
|
|
key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
|
|
if-no-files-found: 'error'
|
|
retention-days: '2'
|
|
if: steps.restore-eslint-cache-ui.outputs.stash-hit != 'true'
|
|
- name: "Restore eslint cache (simple auth manager UI)"
|
|
uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
|
|
with:
|
|
path: airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/node_modules/
|
|
key: >
|
|
cache-simple-am-ui-node-modules-v1-
|
|
${{ runner.os }}-${{ hashFiles('airflow/api_fastapi/auth/managers/simple/ui/**/pnpm-lock.yaml') }}
|
|
id: restore-eslint-cache-simple-am-ui
|
|
- run: cd airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui && pnpm install --frozen-lockfile
|
|
- run: cd airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui && pnpm test
|
|
env:
|
|
FORCE_COLOR: 2
|
|
- name: "Save eslint cache (ui)"
|
|
uses: apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
|
|
with:
|
|
path: airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/node_modules/
|
|
key: >
|
|
cache-simple-am-ui-node-modules-v1-
|
|
${{ runner.os }}-${{ hashFiles('airflow/api_fastapi/auth/managers/simple/ui/**/pnpm-lock.yaml') }}
|
|
if-no-files-found: 'error'
|
|
retention-days: '2'
|
|
if: steps.restore-eslint-cache-simple-am-ui.outputs.stash-hit != 'true'
|
|
|
|
check-translation-completness:
|
|
timeout-minutes: 15
|
|
name: "Check translation completeness"
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
steps:
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
- name: "Check translation completeness"
|
|
run: breeze ui check-translation-completeness || true
|
|
|
|
# Those checks are run if no image needs to be built for checks. This is for simple changes that
|
|
# Do not touch any of the python code or any of the important files that might require building
|
|
# The CI Docker image and they can be run entirely using the prek virtual environments on host
|
|
static-checks-basic-checks-only:
|
|
timeout-minutes: 30
|
|
name: "Static checks: basic checks only"
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
if: inputs.basic-checks-only == 'true'
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
id: breeze
|
|
- name: "Install prek"
|
|
uses: ./.github/actions/install-prek
|
|
id: prek
|
|
with:
|
|
python-version: ${{ steps.breeze.outputs.host-python-version }}
|
|
platform: ${{ inputs.platform }}
|
|
save-cache: true
|
|
- name: Fetch incoming commit ${{ github.sha }} with its parent
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
ref: ${{ github.sha }}
|
|
fetch-depth: 2
|
|
persist-credentials: false
|
|
- name: "Static checks: basic checks only"
|
|
run: >
|
|
prek --show-diff-on-failure --color always
|
|
--from-ref "${{ github.sha }}"^ --to-ref "${{ github.sha }}"
|
|
env:
|
|
VERBOSE: "false"
|
|
SKIP_BREEZE_PREK_HOOKS: "true"
|
|
SKIP: ${{ inputs.skip-prek-hooks }}
|
|
COLUMNS: "202"
|
|
|
|
test-git-clone-on-windows:
|
|
timeout-minutes: 5
|
|
name: "Test git clone on Windows"
|
|
runs-on: ["windows-2025"]
|
|
steps:
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
fetch-depth: 2
|
|
persist-credentials: false
|
|
|
|
upgrade-check:
|
|
timeout-minutes: 45
|
|
name: "Upgrade checks"
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
env:
|
|
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
|
|
if: inputs.canary-run == 'true'
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
id: breeze
|
|
- name: "Install prek"
|
|
uses: ./.github/actions/install-prek
|
|
id: prek
|
|
with:
|
|
python-version: ${{ steps.breeze.outputs.host-python-version }}
|
|
platform: ${{ inputs.platform }}
|
|
save-cache: false
|
|
- name: "Autoupdate all prek hooks"
|
|
run: prek autoupdate --cooldown-days 4 --freeze
|
|
- name: "Check if there are any changes in prek hooks"
|
|
run: |
|
|
if ! git diff --exit-code; then
|
|
echo -e "\n\033[0;31mThere are changes in prek hooks after upgrade check.\033[0m"
|
|
echo -e "\n\033[0;33mHow to fix:\033[0m Run \`breeze ci upgrade\` locally to fix it!.\n"
|
|
exit 1
|
|
fi
|
|
- name: "Run automated upgrade for chart dependencies"
|
|
run: >
|
|
prek
|
|
--all-files --show-diff-on-failure --color always --verbose
|
|
--hook-stage manual
|
|
update-chart-dependencies
|
|
if: always()
|
|
# For UV we are not failing the upgrade installers check if it is updated because
|
|
# it is upgraded very frequently, so we want to manually upgrade it rather than
|
|
# get notified about it - until it stabilizes in 1.* version
|
|
- name: "Run automated upgrade for uv, prek (not failing - just informational)"
|
|
run: >
|
|
prek
|
|
--all-files --show-diff-on-failure --color always --verbose
|
|
--hook-stage manual upgrade-important-versions || true
|
|
if: always()
|
|
env:
|
|
UPGRADE_FLIT: "false"
|
|
UPGRADE_GITPYTHON: "false"
|
|
UPGRADE_GOLANG: "false"
|
|
UPGRADE_HATCH: "false"
|
|
UPGRADE_HATCHLING: "false"
|
|
UPGRADE_MYPY: "false"
|
|
UPGRADE_NODE_LTS: "false"
|
|
UPGRADE_PIP: "false"
|
|
UPGRADE_PYTHON: "false"
|
|
UPGRADE_PYYAML: "false"
|
|
UPGRADE_RICH: "false"
|
|
UPGRADE_RUFF: "false"
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
- name: "Run automated upgrade for important versions minus uv (failing if needed)"
|
|
run: |
|
|
if ! prek \
|
|
--all-files --show-diff-on-failure --color always --verbose \
|
|
--hook-stage manual upgrade-important-versions; then
|
|
echo -e "\n\033[0;31mThere are changes in prek hooks after upgrade check.\033[0m"
|
|
echo -e "\n\033[0;33mHow to fix:\033[0m Run \`breeze ci upgrade\` locally to fix it!.\n"
|
|
exit 1
|
|
fi
|
|
if: always()
|
|
env:
|
|
UPGRADE_UV: "false"
|
|
UPGRADE_PREK: "false"
|
|
UPGRADE_MPROCS: "false"
|
|
UPGRADE_PROTOC: "false"
|
|
UPGRADE_OPENAPI_GENERATOR: "false"
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
test-airflow-release-commands:
|
|
timeout-minutes: 80
|
|
name: "Test Airflow release commands"
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
env:
|
|
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
|
|
GITHUB_REPOSITORY: ${{ github.repository }}
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
GITHUB_USERNAME: ${{ github.actor }}
|
|
VERBOSE: "true"
|
|
if: inputs.canary-run == 'true'
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
- name: "Cleanup dist files"
|
|
run: rm -fv ./dist/*
|
|
- name: Setup git for tagging
|
|
run: |
|
|
git config --global user.email "bot@airflow.apache.org"
|
|
git config --global user.name "Your friendly bot"
|
|
- name: Install twine
|
|
run: pip install twine
|
|
- name: "Check Airflow create minor branch command"
|
|
run: breeze release-management create-minor-branch --version-branch 3-1 --answer yes --dry-run
|
|
- name: "Check Airflow RC process command"
|
|
run: >
|
|
breeze release-management start-rc-process --version 3.1.0rc1 --previous-version 3.0.0
|
|
--task-sdk-version 1.0.0rc1 --sync-branch v3-1-test --answer yes --dry-run
|
|
- name: "Check Airflow release process command"
|
|
run: >
|
|
breeze release-management start-release --version 3.1.7
|
|
--answer yes --dry-run
|
|
- name: "Test providers metadata generation"
|
|
run: |
|
|
git remote add apache https://github.com/apache/airflow.git
|
|
git fetch apache --tags
|
|
breeze release-management generate-providers-metadata --refresh-constraints-and-airflow-releases
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
- name: "Fetch all git tags for origin"
|
|
run: git fetch --tags >/dev/null 2>&1 || true
|
|
- name: "Test airflow core issue generation automatically"
|
|
run: |
|
|
breeze release-management generate-issue-content-core \
|
|
--limit-pr-count 2 --previous-release 3.0.1 --current-release 3.0.2 --verbose
|
|
|
|
|
|
test-airflow-standalone:
|
|
timeout-minutes: 30
|
|
name: "Test Airflow standalone commands"
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
env:
|
|
AIRFLOW_HOME: ~/airflow
|
|
FORCE_COLOR: 1
|
|
steps:
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Install uv"
|
|
run: curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh
|
|
env:
|
|
UV_VERSION: ${{ inputs.uv-version }}
|
|
- name: "Set up Airflow home directory"
|
|
run: |
|
|
echo "Setting AIRFLOW_HOME to $AIRFLOW_HOME"
|
|
mkdir -p $AIRFLOW_HOME
|
|
- name: "Install Airflow from current repo (simulating user installation)"
|
|
run: |
|
|
uv venv
|
|
set -x
|
|
uv pip install -e ./airflow-core
|
|
- name: "Test airflow standalone command"
|
|
run: |
|
|
uv run --no-sync airflow standalone 2>&1 | tee airflow_startup.log &
|
|
AIRFLOW_PID=$!
|
|
|
|
# Wait for ready message till timeout (10 minutes)
|
|
for i in {1..600}; do
|
|
if ! kill -0 $AIRFLOW_PID 2>/dev/null; then
|
|
wait $AIRFLOW_PID
|
|
EXIT_CODE=$?
|
|
echo "FAILED: Airflow standalone exited with code $EXIT_CODE"
|
|
exit $EXIT_CODE
|
|
fi
|
|
|
|
if grep -q "Airflow is ready" airflow_startup.log; then
|
|
echo "SUCCESS: Airflow standalone is ready!"
|
|
kill $AIRFLOW_PID
|
|
exit 0
|
|
fi
|
|
|
|
sleep 1
|
|
done
|
|
|
|
echo "FAILED: Airflow standalone did not become ready in time"
|
|
kill $AIRFLOW_PID 2>/dev/null || true
|
|
exit 1
|