airflow/.github/workflows/push-image-cache.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: Push image cache
on:  # yamllint disable-line rule:truthy
  workflow_call:
    inputs:
      runners:
        description: "The array of labels (in json form) determining runners."
        required: true
        type: string
      cache-type:
        description: "Type of cache to push (Early / Regular)."
        required: true
        type: string
      include-prod-images:
        description: "Whether to build PROD image cache additionally to CI image cache (true/false)."
        required: true
        type: string
      push-latest-images:
        description: "Whether to also push latest images (true/false)."
        required: true
        type: string
      debian-version:
        description: "Base Debian distribution to use for the build (bookworm)"
        type: string
        default: "bookworm"
      install-mysql-client-type:
        description: "MySQL client type to use during build (mariadb/mysql)"
        type: string
        default: "mariadb"
      platform:
        description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
        required: true
        type: string
      python-versions:
        description: "JSON-formatted array of Python versions to build images from"
        required: true
        type: string
      branch:
        description: "Branch used to run the CI jobs in (main/v*_*_test)."
        required: true
        type: string
      constraints-branch:
        description: "Branch used to construct constraints URL from."
        required: true
        type: string
      use-uv:
        description: "Whether to use uv to build the image (true/false)"
        required: true
        type: string
      include-success-outputs:
        description: "Whether to include success outputs (true/false)."
        required: true
        type: string
      docker-cache:
        description: "Docker cache specification to build the image (registry, local, disabled)."
        required: true
        type: string
      disable-airflow-repo-cache:
        description: "Disable airflow repo cache read from main."
        required: true
        type: string
jobs:
  push-ci-image-cache:
    name: "Push CI ${{ inputs.cache-type }}:${{ matrix.python }} image cache "
    runs-on: ${{ fromJSON(inputs.runners) }}
    permissions:
      contents: read
      packages: write
    strategy:
      fail-fast: false
      max-parallel: 20
      matrix:
        python: ${{ fromJSON(inputs.python-versions) }}
    env:
      COMMIT_SHA: ${{ github.sha }}
      # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
      # Airflow one is going to be used
      CONSTRAINTS_GITHUB_REPOSITORY: >-
        ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
        secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
      # In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one
      DEBIAN_VERSION: ${{ inputs.debian-version }}
      DEFAULT_BRANCH: ${{ inputs.branch }}
      DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }}
      DOCKER_CACHE: ${{ inputs.docker-cache }}
      DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }}
      GITHUB_REPOSITORY: ${{ github.repository }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GITHUB_USERNAME: ${{ github.actor }}
      INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}"
      INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
      PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python }}"
      UPGRADE_TO_NEWER_DEPENDENCIES: "false"
      VERBOSE: "true"
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          persist-credentials: false
      - name: "Free up disk space"
        run: ./scripts/tools/free_up_disk_space.sh
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
      - name: Login to ghcr.io
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          ACTOR: ${{ github.actor }}
        run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin
      - name: "Push CI latest images: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} (linux/amd64 only)"
        env:
          PLATFORM: ${{ inputs.platform }}
        run: >
          breeze
          ci-image build
          --builder airflow_cache
          --platform "${PLATFORM}"
          --push
        if: inputs.push-latest-images == 'true' && inputs.platform == 'linux/amd64'
        # yamllint disable-line rule:line-length
      - name: "Push CI ${{ inputs.cache-type }} cache:${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ inputs.platform }}"
        env:
          PLATFORM: ${{ inputs.platform }}
        run: >
          breeze ci-image build
          --builder airflow_cache
          --prepare-buildx-cache
          --platform "${PLATFORM}"
          --push

  push-prod-image-cache:
    name: "Push PROD ${{ inputs.cache-type }}:${{ matrix.python }} image cache"
    runs-on: ${{ fromJSON(inputs.runners) }}
    permissions:
      contents: read
      packages: write
    strategy:
      fail-fast: false
      max-parallel: 20
      matrix:
        python: ${{ fromJSON(inputs.python-versions) }}
    env:
      COMMIT_SHA: ${{ github.sha }}
      # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
      # Airflow one is going to be used
      CONSTRAINTS_GITHUB_REPOSITORY: >-
        ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
        secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
      # In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one
      DEBIAN_VERSION: ${{ inputs.debian-version }}
      DEFAULT_BRANCH: ${{ inputs.branch }}
      DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }}
      DOCKER_CACHE: ${{ inputs.docker-cache }}
      DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }}
      GITHUB_REPOSITORY: ${{ github.repository }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GITHUB_USERNAME: ${{ github.actor }}
      INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
      PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python }}"
      UPGRADE_TO_NEWER_DEPENDENCIES: "false"
      VERBOSE: "true"
    if: inputs.include-prod-images == 'true'
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          persist-credentials: false
      - name: "Free up disk space"
        run: ./scripts/tools/free_up_disk_space.sh
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
      - name: "Cleanup dist and context file"
        run: rm -fv ./dist/* ./docker-context-files/*
      - name: "Download packages prepared as artifacts"
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
        with:
          name: prod-packages
          path: ./docker-context-files
      - name: Login to ghcr.io
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          ACTOR: ${{ github.actor }}
        run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin
        # We only push "AMD" images as it is really only needed for any kind of automated builds in CI
        # and currently there is not an easy way to make multi-platform image from two separate builds
        # and we can do it after we stopped the ARM instance as it is not needed anymore
      - name: "Push PROD latest image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} (linux/amd64 ONLY)"
        env:
          PLATFORM: ${{ inputs.platform }}
        run: >
          breeze prod-image build
          --builder airflow_cache
          --install-distributions-from-context
          --platform "${PLATFORM}"
          --airflow-constraints-mode constraints-source-providers
        if: inputs.push-latest-images == 'true' && inputs.platform == 'linux/amd64'
        # yamllint disable-line rule:line-length
      - name: "Push PROD ${{ inputs.cache-type }} cache: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} ${{ inputs.platform }}"
        env:
          PLATFORM: ${{ inputs.platform }}
        run: >
          breeze prod-image build
          --builder airflow_cache
          --prepare-buildx-cache
          --install-distributions-from-context
          --platform "${PLATFORM}"
          --airflow-constraints-mode constraints-source-providers
          --push