mirror of
https://github.com/apache/airflow.git
synced 2026-03-26 15:28:46 +00:00
492 lines
19 KiB
YAML
492 lines
19 KiB
YAML
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
---
|
|
name: Basic tests
|
|
on: # yamllint disable-line rule:truthy
|
|
workflow_call:
|
|
inputs:
|
|
runners:
|
|
description: "The array of labels (in json form) determining runners."
|
|
required: true
|
|
type: string
|
|
run-ui-tests:
|
|
description: "Whether to run UI tests (true/false)"
|
|
required: true
|
|
type: string
|
|
run-www-tests:
|
|
description: "Whether to run WWW tests (true/false)"
|
|
required: true
|
|
type: string
|
|
run-api-codegen:
|
|
description: "Whether to run API codegen (true/false)"
|
|
required: true
|
|
type: string
|
|
run-breeze-integration-tests:
|
|
description: "Whether to run breeze integration tests (true/false)"
|
|
required: true
|
|
type: string
|
|
run-scripts-tests:
|
|
description: "Whether to run scripts tests (true/false)"
|
|
required: true
|
|
type: string
|
|
basic-checks-only:
|
|
description: "Whether to run only basic checks (true/false)"
|
|
required: true
|
|
type: string
|
|
skip-prek-hooks:
|
|
description: "Whether to skip prek hooks (true/false)"
|
|
required: true
|
|
type: string
|
|
default-python-version:
|
|
description: "Which version of python should be used by default"
|
|
required: true
|
|
type: string
|
|
shared-distributions-as-json:
|
|
description: "Json array of shared distributions to run tests for"
|
|
required: true
|
|
type: string
|
|
canary-run:
|
|
description: "Whether to run canary tests (true/false)"
|
|
required: true
|
|
type: string
|
|
latest-versions-only:
|
|
description: "Whether to run only latest version checks (true/false)"
|
|
required: true
|
|
type: string
|
|
use-uv:
|
|
description: "Whether to use uv in the image"
|
|
required: true
|
|
type: string
|
|
uv-version:
|
|
description: 'uv version to use'
|
|
default: "0.11.1" # Keep this comment to allow automatic replacement of uv version
|
|
type: string
|
|
platform:
|
|
description: 'Platform for the build - linux/amd64 or linux/arm64'
|
|
required: true
|
|
type: string
|
|
permissions:
|
|
contents: read
|
|
jobs:
|
|
run-breeze-tests:
|
|
timeout-minutes: 10
|
|
name: Breeze unit tests
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
# Need to fetch all history for selective checks tests
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
- name: "Run unit tests"
|
|
run: uv tool run --from apache-airflow-breeze pytest -n auto --color=yes
|
|
working-directory: ./dev/breeze/
|
|
run-breeze-integration-tests:
|
|
timeout-minutes: 120
|
|
name: Breeze integration tests
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
if: inputs.run-breeze-integration-tests == 'true'
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
- name: "Install SVN"
|
|
run: sudo apt-get update && sudo apt-get install -y subversion
|
|
- name: "Install Java (for Apache RAT)"
|
|
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
|
|
with:
|
|
distribution: 'temurin'
|
|
java-version: '17'
|
|
- name: "Install hatch"
|
|
run: uv tool install hatch
|
|
- name: "Run integration tests"
|
|
run: uv tool run --from apache-airflow-breeze pytest -v --color=yes -m integration_tests
|
|
working-directory: ./dev/breeze/
|
|
tests-shared-distributions:
|
|
timeout-minutes: 10
|
|
name: Shared ${{ matrix.shared-distribution }} tests
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
shared-distribution: ${{ fromJSON(inputs.shared-distributions-as-json) }}
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
steps:
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 1
|
|
persist-credentials: false
|
|
- name: "Install uv"
|
|
run: pip install "uv==${UV_VERSION}"
|
|
env:
|
|
UV_VERSION: ${{ inputs.uv-version }}
|
|
- name: "Run shared ${{ matrix.shared-distribution }} tests"
|
|
run: uv run --group dev pytest --color=yes -n auto
|
|
working-directory: shared/${{ matrix.shared-distribution }}
|
|
tests-scripts:
|
|
timeout-minutes: 10
|
|
name: Scripts tests
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
if: inputs.run-scripts-tests == 'true'
|
|
steps:
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 1
|
|
persist-credentials: false
|
|
- name: "Install uv"
|
|
run: pip install "uv==${UV_VERSION}"
|
|
env:
|
|
UV_VERSION: ${{ inputs.uv-version }}
|
|
- name: "Run scripts tests"
|
|
run: uv run --project . pytest --color=yes -n auto
|
|
working-directory: ./scripts/
|
|
|
|
tests-ui:
|
|
timeout-minutes: 15
|
|
name: React UI tests
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
if: inputs.run-ui-tests == 'true'
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: Setup pnpm
|
|
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
|
|
with:
|
|
version: 9
|
|
run_install: false
|
|
- name: "Setup node"
|
|
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
|
|
with:
|
|
node-version: 24
|
|
cache: 'pnpm'
|
|
cache-dependency-path: 'airflow-core/src/airflow/**/pnpm-lock.yaml'
|
|
- name: "Restore eslint cache (ui)"
|
|
uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
|
|
with:
|
|
path: airflow-core/src/airflow/ui/node_modules/
|
|
# yamllint disable-line rule:line-length
|
|
key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow-core/src/airflow/ui/**/pnpm-lock.yaml') }}
|
|
id: restore-eslint-cache-ui
|
|
- run: cd airflow-core/src/airflow/ui && pnpm install --frozen-lockfile
|
|
- run: cd airflow-core/src/airflow/ui && pnpm test
|
|
env:
|
|
FORCE_COLOR: 2
|
|
- name: "Save eslint cache (ui)"
|
|
uses: apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
|
|
with:
|
|
path: airflow-core/src/airflow/ui/node_modules/
|
|
key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
|
|
if-no-files-found: 'error'
|
|
retention-days: '2'
|
|
if: steps.restore-eslint-cache-ui.outputs.stash-hit != 'true'
|
|
- name: "Restore eslint cache (simple auth manager UI)"
|
|
uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
|
|
with:
|
|
path: airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/node_modules/
|
|
key: >
|
|
cache-simple-am-ui-node-modules-v1-
|
|
${{ runner.os }}-${{ hashFiles('airflow/api_fastapi/auth/managers/simple/ui/**/pnpm-lock.yaml') }}
|
|
id: restore-eslint-cache-simple-am-ui
|
|
- run: cd airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui && pnpm install --frozen-lockfile
|
|
- run: cd airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui && pnpm test
|
|
env:
|
|
FORCE_COLOR: 2
|
|
- name: "Save eslint cache (ui)"
|
|
uses: apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
|
|
with:
|
|
path: airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/node_modules/
|
|
key: >
|
|
cache-simple-am-ui-node-modules-v1-
|
|
${{ runner.os }}-${{ hashFiles('airflow/api_fastapi/auth/managers/simple/ui/**/pnpm-lock.yaml') }}
|
|
if-no-files-found: 'error'
|
|
retention-days: '2'
|
|
if: steps.restore-eslint-cache-simple-am-ui.outputs.stash-hit != 'true'
|
|
|
|
check-translation-completness:
|
|
timeout-minutes: 15
|
|
name: "Check translation completeness"
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
steps:
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
- name: "Check translation completeness"
|
|
run: breeze ui check-translation-completeness || true
|
|
|
|
# Those checks are run if no image needs to be built for checks. This is for simple changes that
|
|
# Do not touch any of the python code or any of the important files that might require building
|
|
# The CI Docker image and they can be run entirely using the prek virtual environments on host
|
|
static-checks-basic-checks-only:
|
|
timeout-minutes: 30
|
|
name: "Static checks: basic checks only"
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
if: inputs.basic-checks-only == 'true'
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
id: breeze
|
|
- name: "Install prek"
|
|
uses: ./.github/actions/install-prek
|
|
id: prek
|
|
with:
|
|
python-version: ${{ steps.breeze.outputs.host-python-version }}
|
|
platform: ${{ inputs.platform }}
|
|
save-cache: true
|
|
- name: Fetch incoming commit ${{ github.sha }} with its parent
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
ref: ${{ github.sha }}
|
|
fetch-depth: 2
|
|
persist-credentials: false
|
|
- name: "Static checks: basic checks only"
|
|
run: >
|
|
prek --show-diff-on-failure --color always
|
|
--from-ref "${{ github.sha }}"^ --to-ref "${{ github.sha }}"
|
|
env:
|
|
VERBOSE: "false"
|
|
SKIP_BREEZE_PREK_HOOKS: "true"
|
|
SKIP: ${{ inputs.skip-prek-hooks }}
|
|
COLUMNS: "202"
|
|
|
|
test-git-clone-on-windows:
|
|
timeout-minutes: 5
|
|
name: "Test git clone on Windows"
|
|
runs-on: ["windows-2025"]
|
|
steps:
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 2
|
|
persist-credentials: false
|
|
|
|
upgrade-check:
|
|
timeout-minutes: 45
|
|
name: "Upgrade checks"
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
env:
|
|
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
|
|
if: inputs.canary-run == 'true'
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
id: breeze
|
|
- name: "Install prek"
|
|
uses: ./.github/actions/install-prek
|
|
id: prek
|
|
with:
|
|
python-version: ${{ steps.breeze.outputs.host-python-version }}
|
|
platform: ${{ inputs.platform }}
|
|
save-cache: false
|
|
- name: "Autoupdate all prek hooks"
|
|
run: prek autoupdate --cooldown-days 4 --freeze
|
|
- name: "Check if there are any changes in prek hooks"
|
|
run: |
|
|
if ! git diff --exit-code; then
|
|
echo -e "\n\033[0;31mThere are changes in prek hooks after upgrade check.\033[0m"
|
|
echo -e "\n\033[0;33mHow to fix:\033[0m Run \`breeze ci upgrade\` locally to fix it!.\n"
|
|
exit 1
|
|
fi
|
|
- name: "Run automated upgrade for chart dependencies"
|
|
run: >
|
|
prek
|
|
--all-files --show-diff-on-failure --color always --verbose
|
|
--stage manual
|
|
update-chart-dependencies
|
|
if: always()
|
|
# For UV we are not failing the upgrade installers check if it is updated because
|
|
# it is upgraded very frequently, so we want to manually upgrade it rather than
|
|
# get notified about it - until it stabilizes in 1.* version
|
|
- name: "Run automated upgrade for uv, prek (not failing - just informational)"
|
|
run: >
|
|
prek
|
|
--all-files --show-diff-on-failure --color always --verbose
|
|
--stage manual upgrade-important-versions || true
|
|
if: always()
|
|
env:
|
|
UPGRADE_FLIT: "false"
|
|
UPGRADE_GITPYTHON: "false"
|
|
UPGRADE_GOLANG: "false"
|
|
UPGRADE_HATCH: "false"
|
|
UPGRADE_HATCHLING: "false"
|
|
UPGRADE_MYPY: "false"
|
|
UPGRADE_NODE_LTS: "false"
|
|
UPGRADE_PIP: "false"
|
|
UPGRADE_PYTHON: "false"
|
|
UPGRADE_PYYAML: "false"
|
|
UPGRADE_RICH: "false"
|
|
UPGRADE_RUFF: "false"
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
- name: "Run automated upgrade for important versions minus uv (failing if needed)"
|
|
run: |
|
|
if ! prek \
|
|
--all-files --show-diff-on-failure --color always --verbose \
|
|
--stage manual upgrade-important-versions; then
|
|
echo -e "\n\033[0;31mThere are changes in prek hooks after upgrade check.\033[0m"
|
|
echo -e "\n\033[0;33mHow to fix:\033[0m Run \`breeze ci upgrade\` locally to fix it!.\n"
|
|
exit 1
|
|
fi
|
|
if: always()
|
|
env:
|
|
UPGRADE_UV: "false"
|
|
UPGRADE_PREK: "false"
|
|
UPGRADE_MPROCS: "false"
|
|
UPGRADE_PROTOC: "false"
|
|
UPGRADE_OPENAPI_GENERATOR: "false"
|
|
UPGRADE_COOLDOWN_DAYS: "4"
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
test-airflow-release-commands:
|
|
timeout-minutes: 80
|
|
name: "Test Airflow release commands"
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
env:
|
|
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
|
|
GITHUB_REPOSITORY: ${{ github.repository }}
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
GITHUB_USERNAME: ${{ github.actor }}
|
|
VERBOSE: "true"
|
|
if: inputs.canary-run == 'true'
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
- name: "Cleanup dist files"
|
|
run: rm -fv ./dist/*
|
|
- name: Setup git for tagging
|
|
run: |
|
|
git config --global user.email "bot@airflow.apache.org"
|
|
git config --global user.name "Your friendly bot"
|
|
- name: Install twine
|
|
run: pip install twine
|
|
- name: "Check Airflow create minor branch command"
|
|
run: breeze release-management create-minor-branch --version-branch 3-1 --answer yes --dry-run
|
|
- name: "Check Airflow RC process command"
|
|
run: >
|
|
breeze release-management start-rc-process --version 3.1.0rc1 --previous-version 3.0.0
|
|
--task-sdk-version 1.0.0rc1 --sync-branch v3-1-test --answer yes --dry-run
|
|
- name: "Check Airflow release process command"
|
|
run: >
|
|
breeze release-management start-release --version 3.1.7
|
|
--answer yes --dry-run
|
|
- name: "Test providers metadata generation"
|
|
run: |
|
|
git remote add apache https://github.com/apache/airflow.git
|
|
git fetch apache --tags
|
|
breeze release-management generate-providers-metadata --refresh-constraints-and-airflow-releases
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
- name: "Fetch all git tags for origin"
|
|
run: git fetch --tags >/dev/null 2>&1 || true
|
|
- name: "Test airflow core issue generation automatically"
|
|
run: |
|
|
breeze release-management generate-issue-content-core \
|
|
--limit-pr-count 2 --previous-release 3.0.1 --current-release 3.0.2 --verbose
|
|
|
|
|
|
test-airflow-standalone:
|
|
timeout-minutes: 30
|
|
name: "Test Airflow standalone commands"
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
env:
|
|
AIRFLOW_HOME: ~/airflow
|
|
FORCE_COLOR: 1
|
|
steps:
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Install uv"
|
|
run: pip install "uv==${UV_VERSION}"
|
|
env:
|
|
UV_VERSION: ${{ inputs.uv-version }}
|
|
- name: "Set up Airflow home directory"
|
|
run: |
|
|
echo "Setting AIRFLOW_HOME to $AIRFLOW_HOME"
|
|
mkdir -p $AIRFLOW_HOME
|
|
- name: "Install Airflow from current repo (simulating user installation)"
|
|
run: |
|
|
uv venv
|
|
set -x
|
|
uv pip install -e ./airflow-core
|
|
- name: "Test airflow standalone command"
|
|
run: |
|
|
uv run --no-sync airflow standalone 2>&1 | tee airflow_startup.log &
|
|
AIRFLOW_PID=$!
|
|
|
|
# Wait for ready message till timeout (10 minutes)
|
|
for i in {1..600}; do
|
|
if ! kill -0 $AIRFLOW_PID 2>/dev/null; then
|
|
wait $AIRFLOW_PID
|
|
EXIT_CODE=$?
|
|
echo "FAILED: Airflow standalone exited with code $EXIT_CODE"
|
|
exit $EXIT_CODE
|
|
fi
|
|
|
|
if grep -q "Airflow is ready" airflow_startup.log; then
|
|
echo "SUCCESS: Airflow standalone is ready!"
|
|
kill $AIRFLOW_PID
|
|
exit 0
|
|
fi
|
|
|
|
sleep 1
|
|
done
|
|
|
|
echo "FAILED: Airflow standalone did not become ready in time"
|
|
kill $AIRFLOW_PID 2>/dev/null || true
|
|
exit 1
|