airflow/.github/workflows/test-providers.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: Provider tests
on:  # yamllint disable-line rule:truthy
  workflow_call:
    inputs:
      runners:
        description: "The array of labels (in json form) determining public AMD runners."
        required: true
        type: string
      platform:
        description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
        required: true
        type: string
      canary-run:
        description: "Whether this is a canary run"
        required: true
        type: string
      default-python-version:
        description: "Which version of python should be used by default"
        required: true
        type: string
      upgrade-to-newer-dependencies:
        description: "Whether to upgrade to newer dependencies"
        required: true
        type: string
      selected-providers-list-as-string:
        description: "List of affected providers as string"
        required: false
        type: string
      providers-compatibility-tests-matrix:
        description: >
          JSON-formatted array of providers compatibility tests in the form of array of dicts
          (airflow-version, python-versions, remove-providers, run-unit-tests)
        required: true
        type: string
      providers-test-types-list-as-strings-in-json:
        description: "List of parallel provider test types as string"
        required: true
        type: string
      skip-providers-tests:
        description: "Whether to skip provider tests (true/false)"
        required: true
        type: string
      python-versions:
        description: "JSON-formatted array of Python versions to build images from"
        required: true
        type: string
      use-uv:
        description: "Whether to use uv"
        required: true
        type: string
permissions:
  contents: read
jobs:
  prepare-install-verify-provider-distributions:
    timeout-minutes: 80
    name: "Providers ${{ matrix.package-format }} tests"
    runs-on: ${{ fromJSON(inputs.runners) }}
    strategy:
      fail-fast: false
      matrix:
        package-format: ["wheel", "sdist"]
    env:
      GITHUB_REPOSITORY: ${{ github.repository }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GITHUB_USERNAME: ${{ github.actor }}
      INCLUDE_NOT_READY_PROVIDERS: "true"
      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
      VERBOSE: "true"
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          persist-credentials: false
      - name: "Free up disk space"
        shell: bash
        run: ./scripts/tools/free_up_disk_space.sh
      - name: "Install prek"
        uses: ./.github/actions/install-prek
        id: prek
        with:
          python-version: ${{ inputs.default-python-version }}
          platform: ${{ inputs.platform }}
          save-cache: false
      - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}"
        uses: ./.github/actions/prepare_breeze_and_image
        with:
          platform: ${{ inputs.platform }}
          python: "${{ inputs.default-python-version }}"
          use-uv: ${{ inputs.use-uv }}
          make-mnt-writeable-and-cleanup: true
      - name: "Cleanup dist files"
        run: rm -fv ./dist/*
      - name: "Set current date as RELEASE_DATE variable"
        id: date
        run: echo "RELEASE_DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
      - name: "Prepare provider documentation"
        run: >
          breeze release-management prepare-provider-documentation --include-not-ready-providers
          --non-interactive --answer yes
        if: matrix.package-format == 'wheel'
      - name: "Prepare provider distributions: ${{ matrix.package-format }}"
        run: >
          breeze release-management prepare-provider-distributions --include-not-ready-providers
          --skip-tag-check --distribution-format ${{ matrix.package-format }}
      - name: "Prepare airflow package: ${{ matrix.package-format }}"
        run: >
          breeze release-management prepare-airflow-distributions
          --distribution-format ${{ matrix.package-format }}
      - name: "Prepare task-sdk package: ${{ matrix.package-format }}"
        run: >
          breeze release-management prepare-task-sdk-distributions
          --distribution-format ${{ matrix.package-format }}
      - name: "Verify ${{ matrix.package-format }} packages with twine"
        run: |
          uv tool uninstall twine || true
          uv tool install twine && twine check dist/*
      - name: "Test providers issue generation automatically"
        run: >
          breeze release-management generate-issue-content-providers
          --only-available-in-dist --disable-progress
        if: matrix.package-format == 'wheel'
      - name: "Generate source constraints from CI image"
        shell: bash
        run: >
          breeze release-management generate-constraints
          --airflow-constraints-mode constraints-source-providers --answer yes
      - name: "Install and verify wheel provider distributions"
        env:
          DISTRIBUTION_FORMAT: ${{ matrix.package-format }}
          # yamllint disable rule:line-length
          INSTALL_AIRFLOW_WITH_CONSTRAINTS: "${{ inputs.upgrade-to-newer-dependencies == 'true' && 'false' || 'true' }}"
        run: >
          breeze release-management verify-provider-distributions
          --use-distributions-from-dist
          --distribution-format "${DISTRIBUTION_FORMAT}"
          --use-airflow-version "${DISTRIBUTION_FORMAT}"
          --airflow-constraints-reference default
          --providers-constraints-location
          /files/constraints-${PYTHON_MAJOR_MINOR_VERSION}/constraints-source-providers-${PYTHON_MAJOR_MINOR_VERSION}.txt
        if: matrix.package-format == 'wheel'
      - name: "Install all sdist provider distributions and airflow"
        env:
          DISTRIBUTION_FORMAT: ${{ matrix.package-format }}
        run: >
          breeze release-management install-provider-distributions
          --use-distributions-from-dist
          --distribution-format "${DISTRIBUTION_FORMAT}"
          --use-airflow-version ${DISTRIBUTION_FORMAT}
          --airflow-constraints-reference default
          --providers-constraints-location
          /files/constraints-${PYTHON_MAJOR_MINOR_VERSION}/constraints-source-providers-${PYTHON_MAJOR_MINOR_VERSION}.txt
          --run-in-parallel
        if: matrix.package-format == 'sdist'

  # All matrix parameters are passed as JSON string in the input variable providers-compatibility-tests-matrix
  providers-compatibility-tests-matrix:
    timeout-minutes: 80
    # yamllint disable rule:line-length
    name: Compat ${{ matrix.compat.airflow-version }}:P${{ matrix.compat.python-version }}:${{ matrix.compat.test-types.description }}
    runs-on: ${{ fromJSON(inputs.runners) }}
    strategy:
      fail-fast: false
      matrix:
        compat: ${{fromJSON(inputs.providers-compatibility-tests-matrix)}}
        test-types: ${{ fromJSON(inputs.providers-test-types-list-as-strings-in-json) }}
    env:
      GITHUB_REPOSITORY: ${{ github.repository }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GITHUB_USERNAME: ${{ github.actor }}
      INCLUDE_NOT_READY_PROVIDERS: "true"
      PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.compat.python-version }}"
      VERBOSE: "true"
      CLEAN_AIRFLOW_INSTALLATION: "true"
    if: inputs.skip-providers-tests != 'true'
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          persist-credentials: false
      - name: "Free up disk space"
        shell: bash
        run: ./scripts/tools/free_up_disk_space.sh
      - name: "Install prek"
        uses: ./.github/actions/install-prek
        id: prek
        with:
          python-version: ${{ matrix.compat.python-version }}
          platform: ${{ inputs.platform }}
          save-cache: false
      - name: "Prepare breeze & CI image: ${{ matrix.compat.python-version }}"
        uses: ./.github/actions/prepare_breeze_and_image
        with:
          platform: ${{ inputs.platform }}
          python: ${{ matrix.compat.python-version }}
          use-uv: ${{ inputs.use-uv }}
          make-mnt-writeable-and-cleanup: true
      - name: "Cleanup dist files"
        run: rm -fv ./dist/*
      - name: "Prepare provider distributions: wheel"
        run: >
          breeze release-management prepare-provider-distributions --include-not-ready-providers
          --distribution-format wheel --skip-tag-check
      # yamllint disable rule:line-length
      - name: Remove incompatible Airflow ${{ matrix.compat.airflow-version }}:Python ${{ matrix.compat.python-version }} provider distributions
        env:
          REMOVE_PROVIDERS: ${{ matrix.compat.remove-providers }}
        run: |
          for provider in ${REMOVE_PROVIDERS}; do
            echo "Removing incompatible provider: ${provider}"
            rm -vf dist/apache_airflow_providers_${provider/./_}-*
          done
        if: matrix.compat.remove-providers != ''
      - name: "Download airflow package: wheel"
        run: |
          pip download "apache-airflow==${{ matrix.compat.airflow-version }}" -d dist --no-deps
      - name: >
          Install and verify all provider distributions and airflow on
          Airflow ${{ matrix.compat.airflow-version }}:Python ${{ matrix.compat.python-version }}
        # We do not need to run import check if we run tests, the tests should cover all the import checks
        # automatically
        if: matrix.compat.run-unit-tests != 'true'
        env:
          AIRFLOW_VERSION: "${{ matrix.compat.airflow-version }}"
        run: >
          breeze release-management verify-provider-distributions
          --use-distributions-from-dist
          --distribution-format wheel
          --use-airflow-version wheel
          --airflow-constraints-reference constraints-${AIRFLOW_VERSION}
          --providers-skip-constraints
          --install-airflow-with-constraints
      - name: Check amount of disk space available
        run: df -H
        shell: bash
      - name: >
          Run provider unit tests on Airflow ${{ matrix.compat.airflow-version }}:Python
          ${{ matrix.compat.python-version }}:${{ matrix.test-types.description }}
        if: matrix.compat.run-unit-tests == 'true'
        env:
          PROVIDERS_TEST_TYPES: "${{ matrix.test-types.test_types }}"
          AIRFLOW_VERSION: "${{ matrix.compat.airflow-version }}"
          REMOVE_PROVIDERS: "${{ matrix.compat.remove-providers }}"
        run: >
          breeze testing providers-tests --run-in-parallel
          --parallel-test-types "${PROVIDERS_TEST_TYPES}"
          --use-distributions-from-dist
          --distribution-format wheel
          --use-airflow-version "${AIRFLOW_VERSION}"
          --airflow-constraints-reference constraints-${AIRFLOW_VERSION}
          --install-airflow-with-constraints
          --providers-skip-constraints
          --skip-providers "${REMOVE_PROVIDERS}"