# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # --- name: "Release single PROD image" on: # yamllint disable-line rule:truthy workflow_call: inputs: airflowVersion: description: 'Airflow version (e.g. 3.0.1, 3.0.1rc1, 3.0.1b1)' type: string required: true platformMatrix: description: 'Platform matrix formatted as json (e.g. ["linux/amd64", "linux/arm64"])' type: string required: true pythonVersion: description: 'Python version (e.g. 3.10, 3.11)' type: string required: true skipLatest: description: "Skip tagging latest release (true/false)" type: string required: true amdRunners: description: "Amd64 runners (e.g. [\"ubuntu-22.04\", \"ubuntu-24.04\"])" type: string required: true armRunners: description: "Arm64 runners (e.g. [\"ubuntu-22.04\", \"ubuntu-24.04\"])" type: string required: true secrets: DOCKERHUB_USER: required: true DOCKERHUB_TOKEN: required: true permissions: contents: read env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} VERBOSE: true jobs: build-images: timeout-minutes: 50 # yamllint disable rule:line-length name: "Build: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }}" runs-on: ${{ (matrix.platform == 'linux/amd64') && fromJSON(inputs.amdRunners) || fromJSON(inputs.armRunners) }} strategy: fail-fast: false max-parallel: 20 matrix: platform: ${{ fromJSON(inputs.platformMatrix) }} env: AIRFLOW_VERSION: ${{ inputs.airflowVersion }} PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.pythonVersion }} PLATFORM: ${{ matrix.platform }} SKIP_LATEST: ${{ inputs.skipLatest == 'true' && '--skip-latest' || '' }} COMMIT_SHA: ${{ github.sha }} REPOSITORY: ${{ github.repository }} steps: - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze - name: Free space run: breeze ci free-space --answer yes - name: "Cleanup dist and context file" run: rm -fv ./dist/* ./docker-context-files/* - name: "Login to hub.docker.com" run: > echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }} - name: "Get env vars for metadata" shell: bash run: | echo "ARTIFACT_NAME=metadata-${PYTHON_MAJOR_MINOR_VERSION}-${PLATFORM/\//_}" >> "${GITHUB_ENV}" echo "MANIFEST_FILE_NAME=metadata-${AIRFLOW_VERSION}-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json" >> "${GITHUB_ENV}" echo "MANIFEST_SLIM_FILE_NAME=metadata-${AIRFLOW_VERSION}-slim-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json" >> "${GITHUB_ENV}" - name: Login to ghcr.io env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ACTOR: ${{ github.actor }} run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin - name: "Install buildx plugin" # yamllint disable rule:line-length run: | sudo apt-get update sudo apt-get install ca-certificates curl sudo install -m 0755 -d /etc/apt/keyrings sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc sudo chmod a+r /etc/apt/keyrings/docker.asc # Add the repository to Apt sources: echo \ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update sudo apt install docker-buildx-plugin - name: "Create airflow_cache builder" run: docker buildx create --name airflow_cache --driver docker-container - name: > Build regular images: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }} run: > breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}" --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} --python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist - name: > Verify regular image: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }} run: > breeze prod-image verify --pull --manifest-file dist/${MANIFEST_FILE_NAME} - name: > Release slim images: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }} run: > breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}" --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} --python ${PYTHON_MAJOR_MINOR_VERSION} --slim-images --metadata-folder dist - name: > Verify slim image: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }} run: > breeze prod-image verify --pull --slim-image --manifest-file dist/${MANIFEST_SLIM_FILE_NAME} - name: "List upload-able artifacts" shell: bash run: find ./dist -name '*.json' - name: "Upload metadata artifact ${{ env.ARTIFACT_NAME }}" uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: ${{ env.ARTIFACT_NAME }} path: ./dist/metadata-* retention-days: 7 if-no-files-found: error - name: "Docker logout" run: docker logout if: always() merge-images: timeout-minutes: 5 name: "Merge: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}" runs-on: ["ubuntu-22.04"] needs: [build-images] env: AIRFLOW_VERSION: ${{ inputs.airflowVersion }} PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.pythonVersion }} SKIP_LATEST: ${{ inputs.skipLatest == 'true' && '--skip-latest' || '' }} COMMIT_SHA: ${{ github.sha }} REPOSITORY: ${{ github.repository }} steps: - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze - name: Free space run: breeze ci free-space --answer yes - name: "Cleanup dist and context file" run: rm -fv ./dist/* ./docker-context-files/* - name: "Login to hub.docker.com" run: > echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }} - name: Login to ghcr.io env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ACTOR: ${{ github.actor }} run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin - name: "Download metadata artifacts" uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: ./dist pattern: metadata-${{ inputs.pythonVersion }}-* - name: "List downloaded artifacts" shell: bash run: find ./dist -name '*.json' - name: "Install buildx plugin" # yamllint disable rule:line-length run: | sudo apt-get update sudo apt-get install ca-certificates curl sudo install -m 0755 -d /etc/apt/keyrings sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc sudo chmod a+r /etc/apt/keyrings/docker.asc # Add the repository to Apt sources: echo \ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update sudo apt install docker-buildx-plugin - name: "Install regctl" # yamllint disable rule:line-length run: | mkdir -p ~/bin curl -L https://github.com/regclient/regclient/releases/latest/download/regctl-linux-amd64 >${HOME}/bin/regctl chmod 755 ${HOME}/bin/regctl echo "${HOME}/bin" >>${GITHUB_PATH} - name: "Merge regular images ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}" run: > breeze release-management merge-prod-images --dockerhub-repo "${REPOSITORY}" --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} --python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist - name: "Merge slim images ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}" run: > breeze release-management merge-prod-images --dockerhub-repo "${REPOSITORY}" --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} --python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist --slim-images - name: "Docker logout" run: docker logout if: always()