mirror of
https://github.com/apache/airflow.git
synced 2026-03-27 19:31:27 +00:00
3aa24487f0
The 0.0.28 prek has an important fix - avoids craching when you run `git commit -a` https://github.com/j178/prek/issues/385 - we bump the minimum_version to 0.0.28 as well as upgrading the CI and Breeze version to 0.0.28 - also, we change the strategy of failing upgrade check in CI - we do not fail the upgrade check when there is a new upgrade of `prek` - similar like with `uv` we expect new versions to be released quickly over the `teething` period.
370 lines
15 KiB
YAML
370 lines
15 KiB
YAML
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
---
|
|
name: Basic tests
|
|
on: # yamllint disable-line rule:truthy
|
|
workflow_call:
|
|
inputs:
|
|
runners:
|
|
description: "The array of labels (in json form) determining runners."
|
|
required: true
|
|
type: string
|
|
run-ui-tests:
|
|
description: "Whether to run UI tests (true/false)"
|
|
required: true
|
|
type: string
|
|
run-www-tests:
|
|
description: "Whether to run WWW tests (true/false)"
|
|
required: true
|
|
type: string
|
|
needs-api-codegen:
|
|
description: "Whether to run API codegen (true/false)"
|
|
required: true
|
|
type: string
|
|
basic-checks-only:
|
|
description: "Whether to run only basic checks (true/false)"
|
|
required: true
|
|
type: string
|
|
skip-prek-hooks:
|
|
description: "Whether to skip prek hooks (true/false)"
|
|
required: true
|
|
type: string
|
|
default-python-version:
|
|
description: "Which version of python should be used by default"
|
|
required: true
|
|
type: string
|
|
canary-run:
|
|
description: "Whether to run canary tests (true/false)"
|
|
required: true
|
|
type: string
|
|
latest-versions-only:
|
|
description: "Whether to run only latest version checks (true/false)"
|
|
required: true
|
|
type: string
|
|
use-uv:
|
|
description: "Whether to use uv in the image"
|
|
required: true
|
|
type: string
|
|
permissions:
|
|
contents: read
|
|
jobs:
|
|
run-breeze-tests:
|
|
timeout-minutes: 10
|
|
name: Breeze unit tests
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
# Need to fetch all history for selective checks tests
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
- name: "Prepare and cleanup runner"
|
|
run: ./scripts/ci/prepare_and_cleanup_runner.sh
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
- run: uv tool run --from apache-airflow-breeze pytest -n auto --color=yes
|
|
working-directory: ./dev/breeze/
|
|
tests-ui:
|
|
timeout-minutes: 15
|
|
name: React UI tests
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
if: inputs.run-ui-tests == 'true'
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Prepare and cleanup runner"
|
|
run: ./scripts/ci/prepare_and_cleanup_runner.sh
|
|
- name: Setup pnpm
|
|
uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 # v4.0.0
|
|
with:
|
|
version: 9
|
|
run_install: false
|
|
- name: "Setup node"
|
|
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
|
with:
|
|
node-version: 21
|
|
cache: 'pnpm'
|
|
cache-dependency-path: 'airflow-core/src/airflow/**/pnpm-lock.yaml'
|
|
- name: "Restore eslint cache (ui)"
|
|
uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
|
|
with:
|
|
path: airflow-core/src/airflow/ui/node_modules/
|
|
# yamllint disable-line rule:line-length
|
|
key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow-core/src/airflow/ui/**/pnpm-lock.yaml') }}
|
|
id: restore-eslint-cache-ui
|
|
- run: cd airflow-core/src/airflow/ui && pnpm install --frozen-lockfile
|
|
- run: cd airflow-core/src/airflow/ui && pnpm test
|
|
env:
|
|
FORCE_COLOR: 2
|
|
- name: "Save eslint cache (ui)"
|
|
uses: apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
|
|
with:
|
|
path: airflow-core/src/airflow/ui/node_modules/
|
|
key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
|
|
if-no-files-found: 'error'
|
|
retention-days: '2'
|
|
if: steps.restore-eslint-cache-ui.outputs.stash-hit != 'true'
|
|
- name: "Restore eslint cache (simple auth manager UI)"
|
|
uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
|
|
with:
|
|
path: airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/node_modules/
|
|
key: >
|
|
cache-simple-am-ui-node-modules-v1-
|
|
${{ runner.os }}-${{ hashFiles('airflow/api_fastapi/auth/managers/simple/ui/**/pnpm-lock.yaml') }}
|
|
id: restore-eslint-cache-simple-am-ui
|
|
- run: cd airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui && pnpm install --frozen-lockfile
|
|
- run: cd airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui && pnpm test
|
|
env:
|
|
FORCE_COLOR: 2
|
|
- name: "Save eslint cache (ui)"
|
|
uses: apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
|
|
with:
|
|
path: airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/node_modules/
|
|
key: >
|
|
cache-simple-am-ui-node-modules-v1-
|
|
${{ runner.os }}-${{ hashFiles('airflow/api_fastapi/auth/managers/simple/ui/**/pnpm-lock.yaml') }}
|
|
if-no-files-found: 'error'
|
|
retention-days: '2'
|
|
if: steps.restore-eslint-cache-simple-am-ui.outputs.stash-hit != 'true'
|
|
|
|
install-prek:
|
|
timeout-minutes: 5
|
|
name: "Install prek for cache"
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
env:
|
|
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
|
|
SKIP: ${{ inputs.skip-prek-hooks }}
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
id: breeze
|
|
- name: "Install prek"
|
|
uses: ./.github/actions/install-prek
|
|
id: prek
|
|
with:
|
|
python-version: ${{steps.breeze.outputs.host-python-version}}
|
|
skip-prek-hooks: ${{ inputs.skip-prek-hooks }}
|
|
|
|
# Those checks are run if no image needs to be built for checks. This is for simple changes that
|
|
# Do not touch any of the python code or any of the important files that might require building
|
|
# The CI Docker image and they can be run entirely using the prek virtual environments on host
|
|
static-checks-basic-checks-only:
|
|
timeout-minutes: 30
|
|
name: "Static checks: basic checks only"
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
needs: install-prek
|
|
if: inputs.basic-checks-only == 'true'
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Prepare and cleanup runner"
|
|
run: ./scripts/ci/prepare_and_cleanup_runner.sh
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
id: breeze
|
|
- name: "Install prek"
|
|
uses: ./.github/actions/install-prek
|
|
id: prek
|
|
with:
|
|
python-version: ${{ steps.breeze.outputs.host-python-version }}
|
|
skip-prek-hooks: ${{ inputs.skip-prek-hooks }}
|
|
- name: Fetch incoming commit ${{ github.sha }} with its parent
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
ref: ${{ github.sha }}
|
|
fetch-depth: 2
|
|
persist-credentials: false
|
|
- name: "Static checks: basic checks only"
|
|
run: >
|
|
prek --show-diff-on-failure --color always
|
|
--from-ref "${{ github.sha }}" --to-ref "${{ github.sha }}"
|
|
env:
|
|
VERBOSE: "false"
|
|
SKIP_BREEZE_PREK_HOOKS: "true"
|
|
SKIP: ${{ inputs.skip-prek-hooks }}
|
|
COLUMNS: "202"
|
|
|
|
test-git-clone-on-windows:
|
|
timeout-minutes: 5
|
|
name: "Test git clone on Windows"
|
|
runs-on: ["windows-latest"]
|
|
steps:
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
fetch-depth: 2
|
|
persist-credentials: false
|
|
|
|
upgrade-check:
|
|
timeout-minutes: 45
|
|
name: "Upgrade checks"
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
needs: install-prek
|
|
env:
|
|
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
|
|
if: inputs.canary-run == 'true'
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Prepare and cleanup runner"
|
|
run: ./scripts/ci/prepare_and_cleanup_runner.sh
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
id: breeze
|
|
- name: "Install prek"
|
|
uses: ./.github/actions/install-prek
|
|
id: prek
|
|
with:
|
|
python-version: ${{steps.breeze.outputs.host-python-version}}
|
|
skip-prek-hooks: ${{ inputs.skip-prek-hooks }}
|
|
# TODO(potiuk): enable this once prek supports automatic upgrade of hooks
|
|
# after https://github.com/j178/prek/issues/35 is implemented§
|
|
# - name: "Autoupdate all prek hooks"
|
|
# run: prek autoupdate --freeze
|
|
# - name: "Autoupdate Lucas-C hooks to bleeding edge"
|
|
# run: prek autoupdate --bleeding-edge --freeze --repo https://github.com/Lucas-C/pre-commit-hooks
|
|
# - name: "Check if there are any changes in pre-commit hooks"
|
|
# run: git diff --exit-code
|
|
- name: "Run automated upgrade for black"
|
|
run: >
|
|
prek
|
|
--all-files --show-diff-on-failure --color always --verbose
|
|
--hook-stage manual
|
|
update-black-version
|
|
if: always()
|
|
- name: "Run automated upgrade for chart dependencies"
|
|
run: >
|
|
prek
|
|
--all-files --show-diff-on-failure --color always --verbose
|
|
--hook-stage manual
|
|
update-chart-dependencies
|
|
if: always()
|
|
# For UV we are not failing the upgrade installers check if it is updated because
|
|
# it is upgraded very frequently, so we want to manually upgrade it rather than
|
|
# get notified about it - until it stabilizes in 1.* version
|
|
- name: "Run automated upgrade for uv (not failing - just informational)"
|
|
run: >
|
|
prek
|
|
--all-files --show-diff-on-failure --color always --verbose
|
|
--hook-stage manual upgrade-important-versions || true
|
|
if: always()
|
|
env:
|
|
UPGRADE_UV: "true"
|
|
UPGRADE_PYTHON: "false"
|
|
UPGRADE_GOLANG: "true"
|
|
UPGRADE_PIP: "false"
|
|
UPGRADE_PREK: "true"
|
|
UPGRADE_NODE_LTS: "false"
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
- name: "Run automated upgrade for important versions minus uv(failing if needed)"
|
|
run: >
|
|
prek
|
|
--all-files --show-diff-on-failure --color always --verbose
|
|
--hook-stage manual upgrade-important-versions
|
|
if: always()
|
|
env:
|
|
UPGRADE_UV: "false"
|
|
UPGRADE_PYTHON: "true"
|
|
UPGRADE_GOLANG: "false"
|
|
UPGRADE_PIP: "true"
|
|
UPGRADE_PREK: "false"
|
|
UPGRADE_NODE_LTS: "true"
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
test-airflow-release-commands:
|
|
timeout-minutes: 80
|
|
name: "Test Airflow release commands"
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
env:
|
|
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
|
|
GITHUB_REPOSITORY: ${{ github.repository }}
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
GITHUB_USERNAME: ${{ github.actor }}
|
|
VERBOSE: "true"
|
|
if: inputs.canary-run == 'true'
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Prepare and cleanup runner"
|
|
run: ./scripts/ci/prepare_and_cleanup_runner.sh
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
- name: "Cleanup dist files"
|
|
run: rm -fv ./dist/*
|
|
- name: Setup git for tagging
|
|
run: |
|
|
git config --global user.email "bot@airflow.apache.org"
|
|
git config --global user.name "Your friendly bot"
|
|
- name: Install twine
|
|
run: pip install twine
|
|
- name: "Check Airflow create minor branch command"
|
|
run: >
|
|
breeze release-management create-minor-branch
|
|
--version-branch 3-1 --answer yes --dry-run
|
|
- name: "Check Airflow RC process command"
|
|
run: >
|
|
breeze release-management start-rc-process
|
|
--version 3.1.0rc1 --previous-version 3.0.0 --answer yes --dry-run
|
|
- name: "Check Airflow release process command"
|
|
run: >
|
|
breeze release-management start-release --release-candidate 3.1.0rc1
|
|
--previous-release 3.0.0 --answer yes --dry-run
|
|
- name: "Test providers metadata generation"
|
|
run: |
|
|
git remote add apache https://github.com/apache/airflow.git
|
|
git fetch apache --tags
|
|
breeze release-management generate-providers-metadata --refresh-constraints-and-airflow-releases
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
- name: "Fetch all git tags for origin"
|
|
run: git fetch --tags >/dev/null 2>&1 || true
|
|
- name: "Test airflow core issue generation automatically"
|
|
run: |
|
|
breeze release-management generate-issue-content-core \
|
|
--limit-pr-count 2 --previous-release 3.0.1 --current-release 3.0.2 --verbose
|