mirror of
https://github.com/apache/airflow.git
synced 2026-03-26 15:28:46 +00:00
b507852799
We decided to clean-up runner and remove all the potential garbage we get when we start runner - we also move docker to a separate /mnt partitiion available on the runners - because for unit testing especially we need as much space as possible. Also when we build images, it is best if we use clean, pristine docker engine located on the /mnt folder. All of our other jobs are ok with small disk space and sometimes that cleanup can take up to a minute (usually it is 10-20s) This change (experimentally) removes all cleanup from all jobs except the unit tests and image building to see if we can get stability and speed at the same time.
235 lines
10 KiB
YAML
235 lines
10 KiB
YAML
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
---
|
|
name: "Release single PROD image"
|
|
on: # yamllint disable-line rule:truthy
|
|
workflow_call:
|
|
inputs:
|
|
airflowVersion:
|
|
description: 'Airflow version (e.g. 3.0.1, 3.0.1rc1, 3.0.1b1)'
|
|
type: string
|
|
required: true
|
|
platformMatrix:
|
|
description: 'Platform matrix formatted as json (e.g. ["linux/amd64", "linux/arm64"])'
|
|
type: string
|
|
required: true
|
|
pythonVersion:
|
|
description: 'Python version (e.g. 3.10, 3.11)'
|
|
type: string
|
|
required: true
|
|
skipLatest:
|
|
description: "Skip tagging latest release (true/false)"
|
|
type: string
|
|
required: true
|
|
amdRunners:
|
|
description: "Amd64 runners (e.g. [\"ubuntu-22.04\", \"ubuntu-24.04\"])"
|
|
type: string
|
|
required: true
|
|
armRunners:
|
|
description: "Arm64 runners (e.g. [\"ubuntu-22.04\", \"ubuntu-24.04\"])"
|
|
type: string
|
|
required: true
|
|
secrets:
|
|
DOCKERHUB_USER:
|
|
required: true
|
|
DOCKERHUB_TOKEN:
|
|
required: true
|
|
permissions:
|
|
contents: read
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
VERBOSE: true
|
|
jobs:
|
|
build-images:
|
|
timeout-minutes: 50
|
|
# yamllint disable rule:line-length
|
|
name: "Build: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }}"
|
|
runs-on: ${{ (matrix.platform == 'linux/amd64') && fromJSON(inputs.amdRunners) || fromJSON(inputs.armRunners) }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
platform: ${{ fromJSON(inputs.platformMatrix) }}
|
|
env:
|
|
AIRFLOW_VERSION: ${{ inputs.airflowVersion }}
|
|
PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.pythonVersion }}
|
|
PLATFORM: ${{ matrix.platform }}
|
|
SKIP_LATEST: ${{ inputs.skipLatest == 'true' && '--skip-latest' || '' }}
|
|
COMMIT_SHA: ${{ github.sha }}
|
|
REPOSITORY: ${{ github.repository }}
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
- name: Free space
|
|
run: breeze ci free-space --answer yes
|
|
- name: "Cleanup dist and context file"
|
|
run: rm -fv ./dist/* ./docker-context-files/*
|
|
- name: "Login to hub.docker.com"
|
|
run: >
|
|
echo ${{ secrets.DOCKERHUB_TOKEN }} |
|
|
docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }}
|
|
- name: "Get env vars for metadata"
|
|
shell: bash
|
|
run: |
|
|
echo "ARTIFACT_NAME=metadata-${PYTHON_MAJOR_MINOR_VERSION}-${PLATFORM/\//_}" >> "${GITHUB_ENV}"
|
|
echo "MANIFEST_FILE_NAME=metadata-${AIRFLOW_VERSION}-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json" >> "${GITHUB_ENV}"
|
|
echo "MANIFEST_SLIM_FILE_NAME=metadata-${AIRFLOW_VERSION}-slim-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json" >> "${GITHUB_ENV}"
|
|
- name: Login to ghcr.io
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
ACTOR: ${{ github.actor }}
|
|
run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin
|
|
- name: "Install buildx plugin"
|
|
# yamllint disable rule:line-length
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install ca-certificates curl
|
|
sudo install -m 0755 -d /etc/apt/keyrings
|
|
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
|
|
sudo chmod a+r /etc/apt/keyrings/docker.asc
|
|
|
|
# Add the repository to Apt sources:
|
|
echo \
|
|
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
|
|
$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
|
|
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
|
|
sudo apt-get update
|
|
sudo apt install docker-buildx-plugin
|
|
- name: "Create airflow_cache builder"
|
|
run: docker buildx create --name airflow_cache --driver docker-container
|
|
- name: >
|
|
Build regular images: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }}
|
|
run: >
|
|
breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}"
|
|
--airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST}
|
|
--python ${PYTHON_MAJOR_MINOR_VERSION}
|
|
--metadata-folder dist
|
|
- name: >
|
|
Verify regular image: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }}
|
|
run: >
|
|
breeze prod-image verify --pull --manifest-file dist/${MANIFEST_FILE_NAME}
|
|
- name: >
|
|
Release slim images: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }}
|
|
run: >
|
|
breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}"
|
|
--airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST}
|
|
--python ${PYTHON_MAJOR_MINOR_VERSION} --slim-images
|
|
--metadata-folder dist
|
|
- name: >
|
|
Verify slim image: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }}
|
|
run: >
|
|
breeze prod-image verify --pull --slim-image --manifest-file dist/${MANIFEST_SLIM_FILE_NAME}
|
|
- name: "List upload-able artifacts"
|
|
shell: bash
|
|
run: find ./dist -name '*.json'
|
|
- name: "Upload metadata artifact ${{ env.ARTIFACT_NAME }}"
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
with:
|
|
name: ${{ env.ARTIFACT_NAME }}
|
|
path: ./dist/metadata-*
|
|
retention-days: 7
|
|
if-no-files-found: error
|
|
- name: "Docker logout"
|
|
run: docker logout
|
|
if: always()
|
|
|
|
merge-images:
|
|
timeout-minutes: 5
|
|
name: "Merge: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}"
|
|
runs-on: ["ubuntu-22.04"]
|
|
needs: [build-images]
|
|
env:
|
|
AIRFLOW_VERSION: ${{ inputs.airflowVersion }}
|
|
PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.pythonVersion }}
|
|
SKIP_LATEST: ${{ inputs.skipLatest == 'true' && '--skip-latest' || '' }}
|
|
COMMIT_SHA: ${{ github.sha }}
|
|
REPOSITORY: ${{ github.repository }}
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
persist-credentials: false
|
|
- name: "Install Breeze"
|
|
uses: ./.github/actions/breeze
|
|
- name: Free space
|
|
run: breeze ci free-space --answer yes
|
|
- name: "Cleanup dist and context file"
|
|
run: rm -fv ./dist/* ./docker-context-files/*
|
|
- name: "Login to hub.docker.com"
|
|
run: >
|
|
echo ${{ secrets.DOCKERHUB_TOKEN }} |
|
|
docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }}
|
|
- name: Login to ghcr.io
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
ACTOR: ${{ github.actor }}
|
|
run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin
|
|
- name: "Download metadata artifacts"
|
|
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
|
with:
|
|
path: ./dist
|
|
pattern: metadata-${{ inputs.pythonVersion }}-*
|
|
- name: "List downloaded artifacts"
|
|
shell: bash
|
|
run: find ./dist -name '*.json'
|
|
- name: "Install buildx plugin"
|
|
# yamllint disable rule:line-length
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install ca-certificates curl
|
|
sudo install -m 0755 -d /etc/apt/keyrings
|
|
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
|
|
sudo chmod a+r /etc/apt/keyrings/docker.asc
|
|
|
|
# Add the repository to Apt sources:
|
|
echo \
|
|
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
|
|
$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
|
|
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
|
|
sudo apt-get update
|
|
sudo apt install docker-buildx-plugin
|
|
- name: "Install regctl"
|
|
# yamllint disable rule:line-length
|
|
run: |
|
|
mkdir -p ~/bin
|
|
curl -L https://github.com/regclient/regclient/releases/latest/download/regctl-linux-amd64 >${HOME}/bin/regctl
|
|
chmod 755 ${HOME}/bin/regctl
|
|
echo "${HOME}/bin" >>${GITHUB_PATH}
|
|
- name: "Merge regular images ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}"
|
|
run: >
|
|
breeze release-management merge-prod-images --dockerhub-repo "${REPOSITORY}"
|
|
--airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST}
|
|
--python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist
|
|
- name: "Merge slim images ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}"
|
|
run: >
|
|
breeze release-management merge-prod-images --dockerhub-repo "${REPOSITORY}"
|
|
--airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST}
|
|
--python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist --slim-images
|
|
- name: "Docker logout"
|
|
run: docker logout
|
|
if: always()
|