airflow/.github/workflows/run-unit-tests.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: Unit tests
on:  # yamllint disable-line rule:truthy
  workflow_call:
    inputs:
      runners:
        description: "The array of labels (in json form) determining public AMD runners."
        required: true
        type: string
      platform:
        description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
        required: true
        type: string
      test-group:
        description: "Test group to run: ('core', 'providers')"
        required: true
        type: string
      test-types-as-strings-in-json:
        description: "The list of list of test types to run (types in item are separated by spaces) as json"
        required: true
        type: string
      backend:
        description: "The backend to run the tests on"
        required: true
        type: string
      test-scope:
        description: "The scope of the test to run: ('DB', 'Non-DB', 'All')"
        required: true
        type: string
      test-name:
        description: "The name of the test to run"
        required: true
        type: string
      test-name-separator:
        description: "The separator to use after the test name"
        required: false
        default: ":"
        type: string
      python-versions:
        description: "The list of python versions (stringified JSON array) to run the tests on."
        required: true
        type: string
      backend-versions:
        description: "The list of backend versions (stringified JSON array) to run the tests on."
        required: true
        type: string
      excluded-providers-as-string:
        description: "Excluded providers (per Python version) as json string"
        required: true
        type: string
      excludes:
        description: "Excluded combos (stringified JSON array of python-version/backend-version dicts)"
        required: true
        type: string
      run-migration-tests:
        description: "Whether to run migration tests or not (true/false)"
        required: false
        default: "false"
        type: string
      run-coverage:
        description: "Whether to run coverage or not (true/false)"
        required: true
        type: string
      debug-resources:
        description: "Whether to debug resources or not (true/false)"
        required: true
        type: string
      include-success-outputs:
        description: "Whether to include success outputs or not (true/false)"
        required: false
        default: "false"
        type: string
      downgrade-sqlalchemy:
        description: "Whether to downgrade SQLAlchemy or not (true/false)"
        required: false
        default: "false"
        type: string
      upgrade-sqlalchemy:
        description: "Whether to upgrade SQLAlchemy or not (true/false)"
        required: false
        default: "false"
        type: string
      upgrade-boto:
        description: "Whether to upgrade boto or not (true/false)"
        required: false
        default: "false"
        type: string
      downgrade-pendulum:
        description: "Whether to downgrade pendulum or not (true/false)"
        required: false
        default: "false"
        type: string
      force-lowest-dependencies:
        description: "Whether to force lowest dependencies for the tests or not (true/false)"
        required: false
        default: "false"
        type: string
      monitor-delay-time-in-seconds:
        description: "How much time to wait between printing parallel monitor summary"
        required: false
        default: 20
        type: number
      skip-providers-tests:
        description: "Whether to skip providers tests or not (true/false)"
        required: true
        type: string
      use-uv:
        description: "Whether to use uv"
        required: true
        type: string
      default-branch:
        description: "The default branch of the repository"
        required: true
        type: string
permissions:
  contents: read
jobs:
  tests:
    timeout-minutes: 65
    # yamllint disable rule:line-length
    name: "\
      ${{ inputs.test-scope == 'All' && '' || inputs.test-scope == 'Quarantined' && 'Qrnt' || inputs.test-scope }}\
      ${{ inputs.test-scope == 'All' && '' || '-' }}\
      ${{ inputs.test-group == 'providers' && 'prov' || inputs.test-group}}:\
      ${{ inputs.test-name }}${{ inputs.test-name-separator }}${{ matrix.backend-version }}:\
      ${{ matrix.python-version}}:${{ matrix.test-types.description }}"
    runs-on: ${{ fromJSON(inputs.runners) }}
    strategy:
      fail-fast: false
      max-parallel: 20
      matrix:
        python-version: "${{fromJSON(inputs.python-versions)}}"
        backend-version: "${{fromJSON(inputs.backend-versions)}}"
        test-types: ${{ fromJSON(inputs.test-types-as-strings-in-json) }}
        exclude: "${{fromJSON(inputs.excludes)}}"
    env:
      BACKEND: "${{ inputs.backend }}"
      BACKEND_VERSION: "${{ matrix.backend-version }}"
      DB_RESET: "true"
      DEBUG_RESOURCES: "${{ inputs.debug-resources }}"
      DOWNGRADE_SQLALCHEMY: "${{ inputs.downgrade-sqlalchemy }}"
      DOWNGRADE_PENDULUM: "${{ inputs.downgrade-pendulum }}"
      ENABLE_COVERAGE: "${{ inputs.run-coverage }}"
      EXCLUDED_PROVIDERS: "${{ inputs.excluded-providers-as-string }}"
      FORCE_LOWEST_DEPENDENCIES: "${{ inputs.force-lowest-dependencies }}"
      GITHUB_REPOSITORY: ${{ github.repository }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GITHUB_USERNAME: ${{ github.actor }}
      INCLUDE_SUCCESS_OUTPUTS: ${{ inputs.include-success-outputs }}
      PLATFORM: "${{ inputs.platform }}"
      # yamllint disable rule:line-length
      JOB_ID: "${{ inputs.test-group }}-${{ matrix.test-types.description }}-${{ inputs.test-scope }}-${{ inputs.test-name }}-${{inputs.backend}}-${{ matrix.backend-version }}-${{ matrix.python-version }}"
      MOUNT_SOURCES: "skip"
      # yamllint disable rule:line-length
      PARALLEL_TEST_TYPES: ${{ matrix.test-types.test_types }}
      PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}"
      UPGRADE_BOTO: "${{ inputs.upgrade-boto }}"
      UPGRADE_SQLALCHEMY: "${{ inputs.upgrade-sqlalchemy }}"
      AIRFLOW_MONITOR_DELAY_TIME_IN_SECONDS: "${{inputs.monitor-delay-time-in-seconds}}"
      VERBOSE: "true"
      DEFAULT_BRANCH: "${{ inputs.default-branch }}"
      TOTAL_TEST_TIMEOUT: "3600"  # 60 minutes in seconds
    if: inputs.test-group == 'core' || inputs.skip-providers-tests != 'true'
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          persist-credentials: false
      - name: "Make /mnt writeable"
        run: ./scripts/ci/make_mnt_writeable.sh
      - name: "Move docker to /mnt"
        run: ./scripts/ci/move_docker_to_mnt.sh
      - name: "Prepare breeze & CI image: ${{ matrix.python-version }}"
        uses: ./.github/actions/prepare_breeze_and_image
        with:
          platform: ${{ inputs.platform }}
          python: ${{ matrix.python-version }}
          use-uv: ${{ inputs.use-uv }}
          # We do not want to clean up /mnt here - it's been already done before preparing image
          make-mnt-writeable-and-cleanup: false
      - name: >
          Migration Tests: ${{ matrix.python-version }}:${{ env.PARALLEL_TEST_TYPES }}
        uses: ./.github/actions/migration_tests
        with:
          python-version: ${{ matrix.python-version }}
        if: inputs.run-migration-tests == 'true' && inputs.test-group == 'core'
      - name: >
          ${{ inputs.test-group }}:${{ inputs.test-scope }} Tests ${{ inputs.test-name }} ${{ matrix.backend-version }}
          Py${{ matrix.python-version }}:${{ env.PARALLEL_TEST_TYPES }}
        env:
          TEST_GROUP: "${{ inputs.test-group }}"
          TEST_SCOPE: "${{ inputs.test-scope }}"
        run: ./scripts/ci/testing/run_unit_tests.sh "${TEST_GROUP}" "${TEST_SCOPE}"
      - name: "Post Tests success"
        uses: ./.github/actions/post_tests_success
        with:
          codecov-token: ${{ secrets.CODECOV_TOKEN }}
          python-version: ${{ matrix.python-version }}
        if: success()
      - name: "Post Tests failure"
        uses: ./.github/actions/post_tests_failure
        if: failure() || cancelled()