2022-03-27 19:19:02 +02:00
|
|
|
# syntax=docker/dockerfile:1.4
|
2026-02-17 23:38:20 +01:00
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
|
|
|
# or more contributor license agreements. See the NOTICE file
|
|
|
|
|
# distributed with this work for additional information
|
|
|
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
|
|
|
# to you under the Apache License, Version 2.0 (the
|
|
|
|
|
# "License"); you may not use this file except in compliance
|
|
|
|
|
# with the License. You may obtain a copy of the License at
|
2019-01-12 14:06:23 +01:00
|
|
|
#
|
2026-02-17 23:38:20 +01:00
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
2019-01-12 14:06:23 +01:00
|
|
|
#
|
2026-02-17 23:38:20 +01:00
|
|
|
# Unless required by applicable law or agreed to in writing,
|
|
|
|
|
# software distributed under the License is distributed on an
|
|
|
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
|
# KIND, either express or implied. See the License for the
|
|
|
|
|
# specific language governing permissions and limitations
|
|
|
|
|
# under the License.
|
2019-04-13 15:03:02 +02:00
|
|
|
#
|
|
|
|
|
# WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT.
|
|
|
|
|
#
|
2025-11-29 03:23:17 +05:30
|
|
|
# PYTHON_LTO: Controls whether Python is built with Link-Time Optimization (LTO).
|
|
|
|
|
#
|
|
|
|
|
# Link-Time Optimization uses MD5 checksums during the compilation process to verify
|
|
|
|
|
# object files and intermediate representations. In FIPS-compliant environments, MD5
|
|
|
|
|
# is blocked as it's not an approved cryptographic algorithm (see FIPS 140-2/140-3).
|
|
|
|
|
# This can cause Python builds with LTO to fail when FIPS mode is enabled.
|
|
|
|
|
#
|
|
|
|
|
# When building FIPS-compliant images, set this to "false" to disable LTO:
|
|
|
|
|
# docker build --build-arg PYTHON_LTO="false" ...
|
|
|
|
|
#
|
|
|
|
|
# Default: "true" (LTO enabled for better performance)
|
|
|
|
|
#
|
|
|
|
|
# Related: https://github.com/apache/airflow/issues/58337
|
|
|
|
|
ARG PYTHON_LTO="true"
|
2025-07-11 21:18:10 +05:30
|
|
|
ARG BASE_IMAGE="debian:bookworm-slim"
|
2022-03-27 19:19:02 +02:00
|
|
|
|
|
|
|
|
##############################################################################################
|
|
|
|
|
# This is the script image where we keep all inlined bash scripts needed in other segments
|
2025-07-11 21:18:10 +05:30
|
|
|
# We use BASE_IMAGE to make sure that the scripts are different for different platforms.
|
2022-03-27 19:19:02 +02:00
|
|
|
##############################################################################################
|
2025-07-11 21:18:10 +05:30
|
|
|
FROM ${BASE_IMAGE} as scripts
|
2022-03-27 19:19:02 +02:00
|
|
|
|
|
|
|
|
##############################################################################################
|
|
|
|
|
# Please DO NOT modify the inlined scripts manually. The content of those files will be
|
2025-08-17 09:00:14 +02:00
|
|
|
# replaced by prek automatically from the "scripts/docker/" folder.
|
2022-03-27 19:19:02 +02:00
|
|
|
# This is done in order to avoid problems with caching and file permissions and in order to
|
|
|
|
|
# make the PROD Dockerfile standalone
|
|
|
|
|
##############################################################################################
|
|
|
|
|
|
2025-08-30 20:18:30 +05:30
|
|
|
# The content below is automatically copied from scripts/docker/install_os_dependencies.sh
|
|
|
|
|
COPY <<"EOF" /install_os_dependencies.sh
|
2023-08-21 06:12:48 +08:00
|
|
|
#!/usr/bin/env bash
|
2022-08-21 14:58:21 +02:00
|
|
|
set -euo pipefail
|
|
|
|
|
|
|
|
|
|
if [[ "$#" != 1 ]]; then
|
2025-09-01 23:16:20 +02:00
|
|
|
echo
|
2025-07-11 21:18:10 +05:30
|
|
|
echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as argument.".
|
2025-09-01 23:16:20 +02:00
|
|
|
echo
|
2022-08-21 14:58:21 +02:00
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
|
2025-08-30 20:18:30 +05:30
|
|
|
AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18}
|
2025-11-29 03:23:17 +05:30
|
|
|
PYTHON_LTO=${PYTHON_LTO:-true}
|
2025-07-11 21:18:10 +05:30
|
|
|
GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}
|
2026-03-16 21:28:37 +02:00
|
|
|
COSIGN_VERSION=${COSIGN_VERSION:-3.0.5}
|
2025-07-11 21:18:10 +05:30
|
|
|
|
2022-08-21 14:58:21 +02:00
|
|
|
if [[ "${1}" == "runtime" ]]; then
|
|
|
|
|
INSTALLATION_TYPE="RUNTIME"
|
|
|
|
|
elif [[ "${1}" == "dev" ]]; then
|
2025-07-11 21:18:10 +05:30
|
|
|
INSTALLATION_TYPE="DEV"
|
|
|
|
|
elif [[ "${1}" == "ci" ]]; then
|
|
|
|
|
INSTALLATION_TYPE="CI"
|
2022-08-21 14:58:21 +02:00
|
|
|
else
|
2025-09-01 23:16:20 +02:00
|
|
|
echo
|
2025-07-11 21:18:10 +05:30
|
|
|
echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime', 'ci' or 'dev'.".
|
2025-09-01 23:16:20 +02:00
|
|
|
echo
|
2022-08-21 14:58:21 +02:00
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
function get_dev_apt_deps() {
|
|
|
|
|
if [[ "${DEV_APT_DEPS=}" == "" ]]; then
|
2025-09-01 23:16:20 +02:00
|
|
|
DEV_APT_DEPS="\
|
|
|
|
|
apt-transport-https \
|
|
|
|
|
apt-utils \
|
|
|
|
|
build-essential \
|
|
|
|
|
dirmngr \
|
|
|
|
|
freetds-bin \
|
|
|
|
|
freetds-dev \
|
|
|
|
|
git \
|
|
|
|
|
graphviz \
|
|
|
|
|
graphviz-dev \
|
|
|
|
|
krb5-user \
|
|
|
|
|
lcov \
|
|
|
|
|
ldap-utils \
|
|
|
|
|
libbluetooth-dev \
|
|
|
|
|
libbz2-dev \
|
|
|
|
|
libc6-dev \
|
|
|
|
|
libdb-dev \
|
|
|
|
|
libev-dev \
|
|
|
|
|
libev4 \
|
|
|
|
|
libffi-dev \
|
|
|
|
|
libgdbm-compat-dev \
|
|
|
|
|
libgdbm-dev \
|
|
|
|
|
libgeos-dev \
|
|
|
|
|
libkrb5-dev \
|
|
|
|
|
libldap2-dev \
|
|
|
|
|
libleveldb-dev \
|
|
|
|
|
libleveldb1d \
|
|
|
|
|
liblzma-dev \
|
|
|
|
|
libncurses5-dev \
|
|
|
|
|
libreadline6-dev \
|
|
|
|
|
libsasl2-2 \
|
|
|
|
|
libsasl2-dev \
|
|
|
|
|
libsasl2-modules \
|
|
|
|
|
libsqlite3-dev \
|
|
|
|
|
libssl-dev \
|
|
|
|
|
libxmlsec1 \
|
|
|
|
|
libxmlsec1-dev \
|
|
|
|
|
libzstd-dev \
|
|
|
|
|
locales \
|
|
|
|
|
lsb-release \
|
|
|
|
|
lzma \
|
|
|
|
|
lzma-dev \
|
|
|
|
|
openssh-client \
|
|
|
|
|
openssl \
|
|
|
|
|
pkg-config \
|
|
|
|
|
pkgconf \
|
|
|
|
|
sasl2-bin \
|
|
|
|
|
sqlite3 \
|
|
|
|
|
sudo \
|
|
|
|
|
tk-dev \
|
|
|
|
|
unixodbc \
|
|
|
|
|
unixodbc-dev \
|
|
|
|
|
uuid-dev \
|
|
|
|
|
wget \
|
|
|
|
|
xz-utils \
|
|
|
|
|
zlib1g-dev \
|
|
|
|
|
"
|
2022-08-21 14:58:21 +02:00
|
|
|
export DEV_APT_DEPS
|
2022-03-27 19:19:02 +02:00
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
2022-08-21 14:58:21 +02:00
|
|
|
function get_runtime_apt_deps() {
|
2023-11-06 10:32:46 +01:00
|
|
|
local debian_version
|
|
|
|
|
local debian_version_apt_deps
|
|
|
|
|
# Get debian version without installing lsb_release
|
|
|
|
|
# shellcheck disable=SC1091
|
|
|
|
|
debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";)
|
|
|
|
|
echo
|
|
|
|
|
echo "DEBIAN CODENAME: ${debian_version}"
|
|
|
|
|
echo
|
2025-09-01 23:16:20 +02:00
|
|
|
debian_version_apt_deps="\
|
|
|
|
|
libffi8 \
|
|
|
|
|
libldap-2.5-0 \
|
|
|
|
|
libssl3 \
|
|
|
|
|
netcat-openbsd\
|
|
|
|
|
"
|
2023-11-06 10:32:46 +01:00
|
|
|
echo
|
|
|
|
|
echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION: ${debian_version}"
|
|
|
|
|
echo
|
2022-08-21 14:58:21 +02:00
|
|
|
if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then
|
2025-09-01 23:16:20 +02:00
|
|
|
RUNTIME_APT_DEPS="\
|
|
|
|
|
${debian_version_apt_deps} \
|
|
|
|
|
apt-transport-https \
|
|
|
|
|
apt-utils \
|
|
|
|
|
curl \
|
|
|
|
|
dumb-init \
|
|
|
|
|
freetds-bin \
|
|
|
|
|
git \
|
|
|
|
|
gnupg \
|
|
|
|
|
iputils-ping \
|
|
|
|
|
krb5-user \
|
|
|
|
|
ldap-utils \
|
|
|
|
|
libev4 \
|
|
|
|
|
libgeos-dev \
|
|
|
|
|
libsasl2-2 \
|
|
|
|
|
libsasl2-modules \
|
|
|
|
|
libxmlsec1 \
|
|
|
|
|
locales \
|
|
|
|
|
lsb-release \
|
|
|
|
|
openssh-client \
|
|
|
|
|
rsync \
|
|
|
|
|
sasl2-bin \
|
|
|
|
|
sqlite3 \
|
|
|
|
|
sudo \
|
|
|
|
|
unixodbc \
|
|
|
|
|
wget\
|
|
|
|
|
"
|
2022-08-21 14:58:21 +02:00
|
|
|
export RUNTIME_APT_DEPS
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
2022-10-31 18:25:03 +01:00
|
|
|
function install_docker_cli() {
|
2024-02-23 10:42:50 +01:00
|
|
|
apt-get update
|
|
|
|
|
apt-get install ca-certificates curl
|
|
|
|
|
install -m 0755 -d /etc/apt/keyrings
|
|
|
|
|
curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
|
|
|
|
|
chmod a+r /etc/apt/keyrings/docker.asc
|
|
|
|
|
# shellcheck disable=SC1091
|
|
|
|
|
echo \
|
|
|
|
|
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \
|
|
|
|
|
$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
|
|
|
|
|
tee /etc/apt/sources.list.d/docker.list > /dev/null
|
|
|
|
|
apt-get update
|
|
|
|
|
apt-get install -y --no-install-recommends docker-ce-cli
|
2022-10-31 18:25:03 +01:00
|
|
|
}
|
|
|
|
|
|
2022-08-21 14:58:21 +02:00
|
|
|
function install_debian_dev_dependencies() {
|
|
|
|
|
apt-get update
|
2024-02-23 10:42:50 +01:00
|
|
|
apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1
|
2025-08-30 20:18:30 +05:30
|
|
|
apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release ca-certificates
|
2022-08-21 14:58:21 +02:00
|
|
|
# shellcheck disable=SC2086
|
|
|
|
|
export ${ADDITIONAL_DEV_APT_ENV?}
|
|
|
|
|
if [[ ${DEV_APT_COMMAND} != "" ]]; then
|
|
|
|
|
bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}"
|
|
|
|
|
fi
|
|
|
|
|
if [[ ${ADDITIONAL_DEV_APT_COMMAND} != "" ]]; then
|
|
|
|
|
bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_DEV_APT_COMMAND}"
|
|
|
|
|
fi
|
|
|
|
|
apt-get update
|
2023-11-06 10:32:46 +01:00
|
|
|
local debian_version
|
|
|
|
|
local debian_version_apt_deps
|
|
|
|
|
# Get debian version without installing lsb_release
|
|
|
|
|
# shellcheck disable=SC1091
|
|
|
|
|
debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";)
|
|
|
|
|
echo
|
|
|
|
|
echo "DEBIAN CODENAME: ${debian_version}"
|
|
|
|
|
echo
|
2022-08-21 14:58:21 +02:00
|
|
|
# shellcheck disable=SC2086
|
2025-09-01 23:16:20 +02:00
|
|
|
apt-get install -y --no-install-recommends ${DEV_APT_DEPS}
|
2022-08-21 14:58:21 +02:00
|
|
|
}
|
|
|
|
|
|
2025-09-01 23:16:20 +02:00
|
|
|
function install_additional_dev_dependencies() {
|
|
|
|
|
if [[ "${ADDITIONAL_DEV_APT_DEPS=}" != "" ]]; then
|
|
|
|
|
# shellcheck disable=SC2086
|
|
|
|
|
apt-get install -y --no-install-recommends ${ADDITIONAL_DEV_APT_DEPS}
|
|
|
|
|
fi
|
|
|
|
|
}
|
2025-08-30 20:18:30 +05:30
|
|
|
|
|
|
|
|
function link_python() {
|
|
|
|
|
# link python binaries to /usr/local/bin and /usr/python/bin with and without 3 suffix
|
|
|
|
|
# Links in /usr/local/bin are needed for tools that expect python to be there
|
|
|
|
|
# Links in /usr/python/bin are needed for tools that are detecting home of python installation including
|
|
|
|
|
# lib/site-packages. The /usr/python/bin should be first in PATH in order to help with the last part.
|
|
|
|
|
for dst in pip3 python3 python3-config; do
|
|
|
|
|
src="$(echo "${dst}" | tr -d 3)"
|
|
|
|
|
echo "Linking ${dst} in /usr/local/bin and /usr/python/bin"
|
|
|
|
|
ln -sv "/usr/python/bin/${dst}" "/usr/local/bin/${dst}"
|
|
|
|
|
for dir in /usr/local/bin /usr/python/bin; do
|
|
|
|
|
if [[ ! -e "${dir}/${src}" ]]; then
|
|
|
|
|
echo "Creating ${src} - > ${dst} link in ${dir}"
|
|
|
|
|
ln -sv "${dir}/${dst}" "${dir}/${src}"
|
|
|
|
|
fi
|
|
|
|
|
done
|
|
|
|
|
done
|
2025-09-01 05:14:03 +02:00
|
|
|
for dst in /usr/python/lib/*
|
|
|
|
|
do
|
|
|
|
|
src="/usr/local/lib/$(basename "${dst}")"
|
|
|
|
|
if [[ -e "${src}" ]]; then
|
|
|
|
|
rm -rf "${src}"
|
|
|
|
|
fi
|
|
|
|
|
echo "Linking ${dst} to ${src}"
|
|
|
|
|
ln -sv "${dst}" "${src}"
|
|
|
|
|
done
|
|
|
|
|
ldconfig
|
2025-08-30 20:18:30 +05:30
|
|
|
}
|
|
|
|
|
|
2022-08-21 14:58:21 +02:00
|
|
|
function install_debian_runtime_dependencies() {
|
|
|
|
|
apt-get update
|
|
|
|
|
apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1
|
2025-08-30 20:18:30 +05:30
|
|
|
apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release ca-certificates
|
2022-08-21 14:58:21 +02:00
|
|
|
# shellcheck disable=SC2086
|
|
|
|
|
export ${ADDITIONAL_RUNTIME_APT_ENV?}
|
|
|
|
|
if [[ "${RUNTIME_APT_COMMAND}" != "" ]]; then
|
|
|
|
|
bash -o pipefail -o errexit -o nounset -o nolog -c "${RUNTIME_APT_COMMAND}"
|
|
|
|
|
fi
|
|
|
|
|
if [[ "${ADDITIONAL_RUNTIME_APT_COMMAND}" != "" ]]; then
|
|
|
|
|
bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_RUNTIME_APT_COMMAND}"
|
|
|
|
|
fi
|
|
|
|
|
apt-get update
|
|
|
|
|
# shellcheck disable=SC2086
|
|
|
|
|
apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS} ${ADDITIONAL_RUNTIME_APT_DEPS}
|
|
|
|
|
apt-get autoremove -yqq --purge
|
|
|
|
|
apt-get clean
|
2025-08-30 20:18:30 +05:30
|
|
|
link_python
|
2022-08-21 14:58:21 +02:00
|
|
|
rm -rf /var/lib/apt/lists/* /var/log/*
|
|
|
|
|
}
|
|
|
|
|
|
2026-03-16 21:28:37 +02:00
|
|
|
function install_cosign() {
|
|
|
|
|
local arch
|
|
|
|
|
arch="$(dpkg --print-architecture)"
|
|
|
|
|
declare -A cosign_sha256s=(
|
|
|
|
|
# https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign_checksums.txt
|
|
|
|
|
[amd64]="db15cc99e6e4837daabab023742aaddc3841ce57f193d11b7c3e06c8003642b2"
|
|
|
|
|
[arm64]="d098f3168ae4b3aa70b4ca78947329b953272b487727d1722cb3cb098a1a20ab"
|
|
|
|
|
)
|
|
|
|
|
local cosign_sha256="${cosign_sha256s[${arch}]}"
|
|
|
|
|
if [[ -z "${cosign_sha256}" ]]; then
|
|
|
|
|
echo "Unsupported architecture for cosign: ${arch}"
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
curl -fsSL \
|
|
|
|
|
"https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-${arch}" \
|
|
|
|
|
-o /tmp/cosign
|
|
|
|
|
echo "${cosign_sha256} /tmp/cosign" | sha256sum --check
|
|
|
|
|
chmod +x /tmp/cosign
|
|
|
|
|
}
|
|
|
|
|
|
2025-07-11 21:18:10 +05:30
|
|
|
function install_python() {
|
2025-09-01 23:16:20 +02:00
|
|
|
# If system python (3.11 in bookworm) is installed (via automatic installation of some dependencies for example), we need
|
|
|
|
|
# to fail and make sure that it is not there, because there can be strange interactions if we install
|
|
|
|
|
# newer version and system libraries are installed, because
|
|
|
|
|
# when you create a virtualenv part of the shared libraries of Python can be taken from the system
|
|
|
|
|
# Installation leading to weird errors when you want to install some modules - for example when you install ssl:
|
|
|
|
|
# /usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so: undefined symbol: _PyModule_Add
|
|
|
|
|
if dpkg -l | grep '^ii' | grep '^ii libpython' >/dev/null; then
|
|
|
|
|
echo
|
|
|
|
|
echo "ERROR! System python is installed by one of the previous steps"
|
|
|
|
|
echo
|
|
|
|
|
echo "Please make sure that no python packages are installed by default. Displaying the reason why libpython3.11 is installed:"
|
|
|
|
|
echo
|
|
|
|
|
apt-get install -yqq aptitude >/dev/null
|
|
|
|
|
aptitude why libpython3.11
|
|
|
|
|
echo
|
|
|
|
|
exit 1
|
|
|
|
|
else
|
|
|
|
|
echo
|
|
|
|
|
echo "GOOD! System python is not installed - OK"
|
|
|
|
|
echo
|
|
|
|
|
fi
|
2025-08-30 20:18:30 +05:30
|
|
|
wget -O python.tar.xz "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz"
|
2026-03-16 21:28:37 +02:00
|
|
|
local major_minor_version
|
2025-08-30 20:18:30 +05:30
|
|
|
major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}"
|
2026-03-16 21:28:37 +02:00
|
|
|
local major minor
|
|
|
|
|
major="${major_minor_version%.*}"
|
|
|
|
|
minor="${major_minor_version#*.}"
|
2025-08-30 20:18:30 +05:30
|
|
|
echo "Verifying Python ${AIRFLOW_PYTHON_VERSION} (${major_minor_version})"
|
2026-03-16 21:28:37 +02:00
|
|
|
if [[ "${major}" -gt 3 ]] || [[ "${major}" -eq 3 && "${minor}" -ge 11 ]]; then
|
|
|
|
|
# Sigstore verification for Python >= 3.11 (PEP 761)
|
|
|
|
|
declare -A sigstore_identities=(
|
|
|
|
|
# https://peps.python.org/pep-0664/#release-manager-and-crew
|
|
|
|
|
[3.11]="pablogsal@python.org"
|
|
|
|
|
# https://peps.python.org/pep-0693/#release-manager-and-crew
|
|
|
|
|
[3.12]="thomas@python.org"
|
|
|
|
|
# https://peps.python.org/pep-0719/#release-manager-and-crew
|
|
|
|
|
[3.13]="thomas@python.org"
|
|
|
|
|
# https://peps.python.org/pep-0745/#release-manager-and-crew
|
|
|
|
|
[3.14]="hugo@python.org"
|
|
|
|
|
)
|
|
|
|
|
declare -A sigstore_issuers=(
|
|
|
|
|
[3.11]="https://accounts.google.com"
|
|
|
|
|
[3.12]="https://accounts.google.com"
|
|
|
|
|
[3.13]="https://accounts.google.com"
|
|
|
|
|
[3.14]="https://github.com/login/oauth"
|
|
|
|
|
)
|
|
|
|
|
wget -O python.tar.xz.sigstore \
|
|
|
|
|
"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.sigstore"
|
|
|
|
|
install_cosign
|
|
|
|
|
local identity="${sigstore_identities[${major_minor_version}]}"
|
|
|
|
|
local issuer="${sigstore_issuers[${major_minor_version}]}"
|
|
|
|
|
/tmp/cosign verify-blob \
|
|
|
|
|
--bundle python.tar.xz.sigstore \
|
|
|
|
|
--certificate-identity "${identity}" \
|
|
|
|
|
--certificate-oidc-issuer "${issuer}" \
|
|
|
|
|
python.tar.xz
|
|
|
|
|
rm -f python.tar.xz.sigstore /tmp/cosign
|
|
|
|
|
else
|
|
|
|
|
# PGP verification for Python 3.10
|
|
|
|
|
declare -A keys=(
|
|
|
|
|
# gpg: key 64E628F8D684696D: public key "Pablo Galindo Salgado <pablogsal@gmail.com>" imported
|
|
|
|
|
# https://peps.python.org/pep-0619/#release-manager-and-crew
|
|
|
|
|
[3.10]="A035C8C19219BA821ECEA86B64E628F8D684696D"
|
|
|
|
|
)
|
|
|
|
|
wget -O python.tar.xz.asc \
|
|
|
|
|
"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.asc"
|
|
|
|
|
GNUPGHOME="$(mktemp -d)"; export GNUPGHOME
|
|
|
|
|
local gpg_key="${keys[${major_minor_version}]}"
|
|
|
|
|
echo "Using GPG key ${gpg_key}"
|
|
|
|
|
gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "${gpg_key}"
|
|
|
|
|
gpg --batch --verify python.tar.xz.asc python.tar.xz
|
|
|
|
|
gpgconf --kill all
|
|
|
|
|
rm -rf "${GNUPGHOME}" python.tar.xz.asc
|
|
|
|
|
fi
|
2025-08-30 20:18:30 +05:30
|
|
|
mkdir -p /usr/src/python
|
|
|
|
|
tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz
|
|
|
|
|
rm python.tar.xz
|
|
|
|
|
cd /usr/src/python
|
|
|
|
|
arch="$(dpkg --print-architecture)"; arch="${arch##*-}"
|
|
|
|
|
gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"
|
|
|
|
|
EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"
|
|
|
|
|
EXTRA_CFLAGS="${EXTRA_CFLAGS:-} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer";
|
|
|
|
|
LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"
|
|
|
|
|
LDFLAGS="${LDFLAGS:--Wl},--strip-all"
|
2025-11-29 03:23:17 +05:30
|
|
|
# Link-Time Optimization (LTO) uses MD5 checksums for object file verification during
|
|
|
|
|
# compilation. In FIPS mode, MD5 is blocked as a non-approved algorithm, causing builds
|
|
|
|
|
# to fail. The PYTHON_LTO variable allows disabling LTO for FIPS-compliant builds.
|
|
|
|
|
# See: https://github.com/apache/airflow/issues/58337
|
|
|
|
|
local lto_option=""
|
|
|
|
|
if [[ "${PYTHON_LTO:-true}" == "true" ]]; then
|
|
|
|
|
lto_option="--with-lto"
|
|
|
|
|
fi
|
2026-03-20 23:15:00 +01:00
|
|
|
local build_log
|
|
|
|
|
build_log=$(mktemp)
|
|
|
|
|
echo "Building Python ${AIRFLOW_PYTHON_VERSION} from source..."
|
|
|
|
|
if ! (
|
|
|
|
|
./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip --build="$gnuArch" \
|
|
|
|
|
--enable-loadable-sqlite-extensions --enable-option-checking=fatal \
|
|
|
|
|
--enable-shared ${lto_option} && \
|
|
|
|
|
make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
|
|
|
|
|
"LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python && \
|
|
|
|
|
make -s -j "$(nproc)" install
|
|
|
|
|
) > "${build_log}" 2>&1; then
|
|
|
|
|
echo
|
|
|
|
|
echo "ERROR! Python build failed. Build output:"
|
|
|
|
|
echo
|
|
|
|
|
cat "${build_log}"
|
|
|
|
|
rm -f "${build_log}"
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
rm -f "${build_log}"
|
2025-08-30 20:18:30 +05:30
|
|
|
cd /
|
|
|
|
|
rm -rf /usr/src/python
|
|
|
|
|
find /usr/python -depth \
|
|
|
|
|
\( \
|
|
|
|
|
\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
|
2025-12-02 16:04:52 +01:00
|
|
|
-o \( -type f -a \( -name 'libpython*.a' \) \) \
|
2025-08-30 20:18:30 +05:30
|
|
|
\) -exec rm -rf '{}' +
|
|
|
|
|
link_python
|
2025-07-11 21:18:10 +05:30
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function install_golang() {
|
|
|
|
|
curl "https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg --print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz"
|
|
|
|
|
rm -rf /usr/local/go && tar -C /usr/local -xzf go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz
|
|
|
|
|
}
|
|
|
|
|
|
2025-08-30 20:18:30 +05:30
|
|
|
function apt_clean() {
|
|
|
|
|
apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false
|
|
|
|
|
rm -rf /var/lib/apt/lists/* /var/log/*
|
|
|
|
|
}
|
|
|
|
|
|
2022-08-21 14:58:21 +02:00
|
|
|
if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then
|
|
|
|
|
get_runtime_apt_deps
|
|
|
|
|
install_debian_runtime_dependencies
|
2022-10-31 18:25:03 +01:00
|
|
|
install_docker_cli
|
2025-08-30 20:18:30 +05:30
|
|
|
apt_clean
|
2022-08-21 14:58:21 +02:00
|
|
|
else
|
|
|
|
|
get_dev_apt_deps
|
|
|
|
|
install_debian_dev_dependencies
|
2025-07-11 21:18:10 +05:30
|
|
|
install_python
|
2025-09-01 23:16:20 +02:00
|
|
|
install_additional_dev_dependencies
|
2025-07-11 21:18:10 +05:30
|
|
|
if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then
|
|
|
|
|
install_golang
|
|
|
|
|
fi
|
2022-10-31 18:25:03 +01:00
|
|
|
install_docker_cli
|
2025-08-30 20:18:30 +05:30
|
|
|
apt_clean
|
2022-08-21 14:58:21 +02:00
|
|
|
fi
|
2022-03-27 19:19:02 +02:00
|
|
|
EOF
|
|
|
|
|
|
|
|
|
|
# The content below is automatically copied from scripts/docker/install_mysql.sh
|
2022-03-31 13:39:46 +02:00
|
|
|
COPY <<"EOF" /install_mysql.sh
|
2023-08-21 06:12:48 +08:00
|
|
|
#!/usr/bin/env bash
|
2023-12-18 18:14:04 +04:00
|
|
|
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
|
|
|
|
|
|
2022-03-27 19:19:02 +02:00
|
|
|
set -euo pipefail
|
|
|
|
|
|
2023-12-18 18:14:04 +04:00
|
|
|
common::get_colors
|
|
|
|
|
declare -a packages
|
2022-03-27 19:19:02 +02:00
|
|
|
|
2023-12-18 18:14:04 +04:00
|
|
|
readonly MARIADB_LTS_VERSION="10.11"
|
2022-03-27 19:19:02 +02:00
|
|
|
|
|
|
|
|
: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
|
2023-12-15 19:13:00 +01:00
|
|
|
: "${INSTALL_MYSQL_CLIENT_TYPE:-mariadb}"
|
2022-03-27 19:19:02 +02:00
|
|
|
|
2025-10-23 14:07:28 +02:00
|
|
|
if [[ "${INSTALL_MYSQL_CLIENT}" != "true" && "${INSTALL_MYSQL_CLIENT}" != "false" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_RED}INSTALL_MYSQL_CLIENT must be either true or false${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" != "mysql" && "${INSTALL_MYSQL_CLIENT_TYPE}" != "mariadb" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_RED}INSTALL_MYSQL_CLIENT_TYPE must be either mysql or mariadb${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" == "mysql" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_RED}The 'mysql' client type is not supported any more. Use 'mariadb' instead.${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
echo "The MySQL drivers are wrongly packaged and released by Oracle with an expiration date on their GPG keys,"
|
|
|
|
|
echo "which causes builds to fail after the expiration date. MariaDB client is protocol-compatible with MySQL client."
|
|
|
|
|
echo ""
|
|
|
|
|
echo "Every two years the MySQL packages fail and Oracle team is always surprised and struggling"
|
|
|
|
|
echo "with fixes and re-signing the packages which lasts few days"
|
|
|
|
|
echo "See https://bugs.mysql.com/bug.php?id=113432 for more details."
|
|
|
|
|
echo "As a community we are not able to support this broken packaging practice from Oracle"
|
|
|
|
|
echo "Feel free however to install MySQL drivers on your own as extension of the image."
|
|
|
|
|
echo
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
|
2025-05-07 14:30:20 +05:30
|
|
|
retry() {
|
|
|
|
|
local retries=3
|
|
|
|
|
local count=0
|
|
|
|
|
# adding delay of 10 seconds
|
|
|
|
|
local delay=10
|
|
|
|
|
until "$@"; do
|
|
|
|
|
exit_code=$?
|
|
|
|
|
count=$((count + 1))
|
|
|
|
|
if [[ $count -lt $retries ]]; then
|
|
|
|
|
echo "Command failed. Attempt $count/$retries. Retrying in ${delay}s..."
|
|
|
|
|
sleep $delay
|
|
|
|
|
else
|
|
|
|
|
echo "Command failed after $retries attempts."
|
|
|
|
|
return $exit_code
|
|
|
|
|
fi
|
|
|
|
|
done
|
|
|
|
|
}
|
|
|
|
|
|
2023-02-22 19:24:02 +04:00
|
|
|
install_mariadb_client() {
|
2023-09-16 21:09:33 +04:00
|
|
|
# List of compatible package Oracle MySQL -> MariaDB:
|
|
|
|
|
# `mysql-client` -> `mariadb-client` or `mariadb-client-compat` (11+)
|
|
|
|
|
# `libmysqlclientXX` (where XX is a number) -> `libmariadb3-compat`
|
|
|
|
|
# `libmysqlclient-dev` -> `libmariadb-dev-compat`
|
|
|
|
|
#
|
|
|
|
|
# Different naming against Debian repo which we used before
|
|
|
|
|
# that some of packages might contains `-compat` suffix, Debian repo -> MariaDB repo:
|
|
|
|
|
# `libmariadb-dev` -> `libmariadb-dev-compat`
|
|
|
|
|
# `mariadb-client-core` -> `mariadb-client` or `mariadb-client-compat` (11+)
|
2023-02-22 19:24:02 +04:00
|
|
|
if [[ "${1}" == "dev" ]]; then
|
2023-09-16 21:09:33 +04:00
|
|
|
packages=("libmariadb-dev-compat" "mariadb-client")
|
2023-02-22 19:24:02 +04:00
|
|
|
elif [[ "${1}" == "prod" ]]; then
|
2023-09-16 21:09:33 +04:00
|
|
|
packages=("libmariadb3-compat" "mariadb-client")
|
2023-02-22 19:24:02 +04:00
|
|
|
else
|
|
|
|
|
echo
|
2023-09-16 21:09:33 +04:00
|
|
|
echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}"
|
2023-02-22 19:24:02 +04:00
|
|
|
echo
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
|
2023-12-18 18:14:04 +04:00
|
|
|
common::import_trusted_gpg "0xF1656F24C74CD1D8" "mariadb"
|
2023-09-16 21:09:33 +04:00
|
|
|
|
2023-02-22 19:24:02 +04:00
|
|
|
echo
|
2023-09-16 21:09:33 +04:00
|
|
|
echo "${COLOR_BLUE}Installing MariaDB client version ${MARIADB_LTS_VERSION}: ${1}${COLOR_RESET}"
|
|
|
|
|
echo "${COLOR_YELLOW}MariaDB client protocol-compatible with MySQL client.${COLOR_RESET}"
|
2023-02-22 19:24:02 +04:00
|
|
|
echo
|
2023-09-16 21:09:33 +04:00
|
|
|
|
|
|
|
|
echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${MARIADB_LTS_VERSION}/repo/debian/ $(lsb_release -cs) main" > \
|
|
|
|
|
/etc/apt/sources.list.d/mariadb.list
|
|
|
|
|
# Make sure that dependencies from MariaDB repo are preferred over Debian dependencies
|
|
|
|
|
printf "Package: *\nPin: release o=MariaDB\nPin-Priority: 999\n" > /etc/apt/preferences.d/mariadb
|
2025-05-07 14:30:20 +05:30
|
|
|
retry apt-get update
|
|
|
|
|
retry apt-get install --no-install-recommends -y "${packages[@]}"
|
2023-02-22 19:24:02 +04:00
|
|
|
apt-get autoremove -yqq --purge
|
|
|
|
|
apt-get clean && rm -rf /var/lib/apt/lists/*
|
|
|
|
|
}
|
2022-03-27 19:19:02 +02:00
|
|
|
|
|
|
|
|
if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then
|
2025-10-23 14:07:28 +02:00
|
|
|
install_mariadb_client "${@}"
|
2022-03-27 19:19:02 +02:00
|
|
|
fi
|
|
|
|
|
EOF
|
|
|
|
|
|
|
|
|
|
# The content below is automatically copied from scripts/docker/install_mssql.sh
|
2022-03-31 13:39:46 +02:00
|
|
|
COPY <<"EOF" /install_mssql.sh
|
2023-08-21 06:12:48 +08:00
|
|
|
#!/usr/bin/env bash
|
2023-12-18 18:14:04 +04:00
|
|
|
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
|
|
|
|
|
|
2022-03-27 19:19:02 +02:00
|
|
|
set -euo pipefail
|
|
|
|
|
|
2023-12-18 18:14:04 +04:00
|
|
|
common::get_colors
|
|
|
|
|
declare -a packages
|
2023-07-21 19:27:51 +02:00
|
|
|
|
2022-03-27 19:19:02 +02:00
|
|
|
: "${INSTALL_MSSQL_CLIENT:?Should be true or false}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function install_mssql_client() {
|
|
|
|
|
# Install MsSQL client from Microsoft repositories
|
|
|
|
|
if [[ ${INSTALL_MSSQL_CLIENT:="true"} != "true" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Skip installing mssql client${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
return
|
|
|
|
|
fi
|
2023-12-18 18:14:04 +04:00
|
|
|
packages=("msodbcsql18")
|
|
|
|
|
|
|
|
|
|
common::import_trusted_gpg "EB3E94ADBE1229CF" "microsoft"
|
|
|
|
|
|
2022-03-27 19:19:02 +02:00
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Installing mssql client${COLOR_RESET}"
|
|
|
|
|
echo
|
2023-12-18 18:14:04 +04:00
|
|
|
|
|
|
|
|
echo "deb [arch=amd64,arm64] https://packages.microsoft.com/debian/$(lsb_release -rs)/prod $(lsb_release -cs) main" > \
|
2025-03-21 18:33:47 +10:00
|
|
|
/etc/apt/sources.list.d/mssql-release.list &&
|
|
|
|
|
mkdir -p /opt/microsoft/msodbcsql18 &&
|
|
|
|
|
touch /opt/microsoft/msodbcsql18/ACCEPT_EULA &&
|
|
|
|
|
apt-get update -yqq &&
|
|
|
|
|
apt-get upgrade -yqq &&
|
|
|
|
|
apt-get -yqq install --no-install-recommends "${packages[@]}" &&
|
|
|
|
|
apt-get autoremove -yqq --purge &&
|
|
|
|
|
apt-get clean &&
|
2022-03-27 19:19:02 +02:00
|
|
|
rm -rf /var/lib/apt/lists/*
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
install_mssql_client "${@}"
|
|
|
|
|
EOF
|
|
|
|
|
|
|
|
|
|
# The content below is automatically copied from scripts/docker/install_postgres.sh
|
2022-03-31 13:39:46 +02:00
|
|
|
COPY <<"EOF" /install_postgres.sh
|
2023-08-21 06:12:48 +08:00
|
|
|
#!/usr/bin/env bash
|
2023-12-18 18:14:04 +04:00
|
|
|
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
|
2022-03-27 19:19:02 +02:00
|
|
|
set -euo pipefail
|
|
|
|
|
|
2023-12-18 18:14:04 +04:00
|
|
|
common::get_colors
|
|
|
|
|
declare -a packages
|
2022-03-27 19:19:02 +02:00
|
|
|
|
|
|
|
|
: "${INSTALL_POSTGRES_CLIENT:?Should be true or false}"
|
|
|
|
|
|
|
|
|
|
install_postgres_client() {
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Installing postgres client${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
|
|
|
|
|
if [[ "${1}" == "dev" ]]; then
|
|
|
|
|
packages=("libpq-dev" "postgresql-client")
|
|
|
|
|
elif [[ "${1}" == "prod" ]]; then
|
|
|
|
|
packages=("postgresql-client")
|
|
|
|
|
else
|
|
|
|
|
echo
|
|
|
|
|
echo "Specify either prod or dev"
|
|
|
|
|
echo
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
|
2023-12-18 18:14:04 +04:00
|
|
|
common::import_trusted_gpg "7FCC7D46ACCC4CF8" "postgres"
|
|
|
|
|
|
|
|
|
|
echo "deb [arch=amd64,arm64] https://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > \
|
|
|
|
|
/etc/apt/sources.list.d/pgdg.list
|
2022-03-27 19:19:02 +02:00
|
|
|
apt-get update
|
|
|
|
|
apt-get install --no-install-recommends -y "${packages[@]}"
|
|
|
|
|
apt-get autoremove -yqq --purge
|
|
|
|
|
apt-get clean && rm -rf /var/lib/apt/lists/*
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if [[ ${INSTALL_POSTGRES_CLIENT:="true"} == "true" ]]; then
|
|
|
|
|
install_postgres_client "${@}"
|
|
|
|
|
fi
|
|
|
|
|
EOF
|
|
|
|
|
|
2024-03-02 15:07:06 +01:00
|
|
|
# The content below is automatically copied from scripts/docker/install_packaging_tools.sh
|
|
|
|
|
COPY <<"EOF" /install_packaging_tools.sh
|
2023-08-21 06:12:48 +08:00
|
|
|
#!/usr/bin/env bash
|
2022-03-27 19:19:02 +02:00
|
|
|
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
|
|
|
|
|
|
|
|
|
|
common::get_colors
|
2024-02-26 13:10:31 +01:00
|
|
|
common::get_packaging_tool
|
|
|
|
|
common::show_packaging_tool_version_and_location
|
2024-03-02 15:07:06 +01:00
|
|
|
common::install_packaging_tools
|
2022-03-27 19:19:02 +02:00
|
|
|
EOF
|
|
|
|
|
|
|
|
|
|
# The content below is automatically copied from scripts/docker/common.sh
|
2022-03-31 13:39:46 +02:00
|
|
|
COPY <<"EOF" /common.sh
|
2023-08-21 06:12:48 +08:00
|
|
|
#!/usr/bin/env bash
|
2022-03-27 19:19:02 +02:00
|
|
|
set -euo pipefail
|
|
|
|
|
|
|
|
|
|
function common::get_colors() {
|
|
|
|
|
COLOR_BLUE=$'\e[34m'
|
|
|
|
|
COLOR_GREEN=$'\e[32m'
|
|
|
|
|
COLOR_RED=$'\e[31m'
|
|
|
|
|
COLOR_RESET=$'\e[0m'
|
|
|
|
|
COLOR_YELLOW=$'\e[33m'
|
|
|
|
|
export COLOR_BLUE
|
|
|
|
|
export COLOR_GREEN
|
|
|
|
|
export COLOR_RED
|
|
|
|
|
export COLOR_RESET
|
|
|
|
|
export COLOR_YELLOW
|
|
|
|
|
}
|
|
|
|
|
|
2024-02-26 13:10:31 +01:00
|
|
|
function common::get_packaging_tool() {
|
2024-02-29 21:33:41 +01:00
|
|
|
: "${AIRFLOW_USE_UV:?Should be set}"
|
|
|
|
|
|
2024-02-26 13:10:31 +01:00
|
|
|
## IMPORTANT: IF YOU MODIFY THIS FUNCTION YOU SHOULD ALSO MODIFY CORRESPONDING FUNCTION IN
|
|
|
|
|
## `scripts/in_container/_in_container_utils.sh`
|
|
|
|
|
if [[ ${AIRFLOW_USE_UV} == "true" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
export PACKAGING_TOOL="uv"
|
|
|
|
|
export PACKAGING_TOOL_CMD="uv pip"
|
2025-07-11 15:49:00 +02:00
|
|
|
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
|
|
|
|
|
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
|
|
|
|
|
# See https://bugs.launchpad.net/lxml/+bug/2110068
|
|
|
|
|
if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -f "./pyproject.toml" ]]; then
|
2025-06-02 10:45:51 +02:00
|
|
|
# for uv only install dev group when we install from sources
|
2025-07-11 15:49:00 +02:00
|
|
|
export EXTRA_INSTALL_FLAGS="--group=dev --no-binary lxml --no-binary xmlsec"
|
2025-06-02 10:45:51 +02:00
|
|
|
else
|
2025-07-11 15:49:00 +02:00
|
|
|
export EXTRA_INSTALL_FLAGS="--no-binary lxml --no-binary xmlsec"
|
2025-06-02 10:45:51 +02:00
|
|
|
fi
|
2025-03-05 23:04:00 +01:00
|
|
|
export EXTRA_UNINSTALL_FLAGS=""
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --resolution highest"
|
2024-03-13 15:59:19 +01:00
|
|
|
export UPGRADE_IF_NEEDED="--upgrade"
|
2024-05-26 19:15:57 +02:00
|
|
|
UV_CONCURRENT_DOWNLOADS=$(nproc --all)
|
|
|
|
|
export UV_CONCURRENT_DOWNLOADS
|
2025-04-16 14:53:29 +02:00
|
|
|
if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then
|
Replace chicken-egg providers with automated use of unreleased packages (#49799)
* Replace chicken-egg providers with automated use of unreleased packages
When we got rid of the .dev0 suffix, it is now possible to entirely
rely on building the packages locally using exsting mechanisms, that
check if packages have been already released - for CI builds, and can
rely on the fact that we need at least pre-release version of packages
if we are building pre-release version of airflow.
It works as follows:
* for CI builds (generate constraints and PROD image builds) - we are
alwasys attempt to build ALL provider packages, but without
--skip-tag-check - which means that if provider has been already
released and it's version did not change in main, we are not going
to build it locally and we will use it from PyPI. However if provider
version is updated and the provider has not yet been released (checked
by tag) - it will be build locally from sources and it will be used
for constraint generation.
* for release PROD images build, on the other hand we NEVER build
packages locally - we always rely on PyPI released packages, however
if we are building pre-release version of airflow, we automatically
add --pre flag that looks for pre-release packages in PyPI - this way
pre-release version of airflow can be built with pre-release version
of providers. We are still attempting to use constraints for that,
however first - so unless there are no limits in apache airflow
that prevent it from using released versions of providers, the
constraint versions will be used - only if it fails, PROD images
will fall back to non-constraint installation that will allow to
use freely pre-release versions of packages from PyPI. This means
for example that if we cherry-pick a change from main that increases
minimum version of provider for apache-airflow to one that does not
even have a pre-release version, building of rc version image for
airflow will fail (which is a good thing). Lack of --pre flag
for "release" version of Airlfow also means that if airlfow has
a min version of provider that has no "released" version yet (only
rc) - it will also fail (which is also a good thing)
* Update scripts/in_container/run_generate_constraints.py
2025-04-26 15:23:21 +02:00
|
|
|
EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --prerelease if-necessary"
|
2025-04-16 14:53:29 +02:00
|
|
|
fi
|
2024-02-26 13:10:31 +01:00
|
|
|
else
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
export PACKAGING_TOOL="pip"
|
|
|
|
|
export PACKAGING_TOOL_CMD="pip"
|
2025-07-10 16:08:23 +02:00
|
|
|
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
|
|
|
|
|
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
|
|
|
|
|
# See https://bugs.launchpad.net/lxml/+bug/2110068
|
|
|
|
|
export EXTRA_INSTALL_FLAGS="--root-user-action ignore --no-binary lxml,xmlsec"
|
2024-02-26 13:10:31 +01:00
|
|
|
export EXTRA_UNINSTALL_FLAGS="--yes"
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --upgrade-strategy eager"
|
2024-03-06 01:27:15 +01:00
|
|
|
export UPGRADE_IF_NEEDED="--upgrade --upgrade-strategy only-if-needed"
|
2025-04-16 14:53:29 +02:00
|
|
|
if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then
|
|
|
|
|
EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --pre"
|
|
|
|
|
fi
|
2024-02-26 13:10:31 +01:00
|
|
|
fi
|
|
|
|
|
}
|
2022-03-27 19:19:02 +02:00
|
|
|
|
|
|
|
|
function common::get_airflow_version_specification() {
|
|
|
|
|
if [[ -z ${AIRFLOW_VERSION_SPECIFICATION=}
|
|
|
|
|
&& -n ${AIRFLOW_VERSION}
|
|
|
|
|
&& ${AIRFLOW_INSTALLATION_METHOD} != "." ]]; then
|
|
|
|
|
AIRFLOW_VERSION_SPECIFICATION="==${AIRFLOW_VERSION}"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function common::get_constraints_location() {
|
2026-03-15 22:39:45 +01:00
|
|
|
# When installing from sources without upgrade, generate constraints from uv.lock
|
2026-03-15 15:20:53 +01:00
|
|
|
if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -z "${UPGRADE_RANDOM_INDICATOR_STRING=}" ]]; then
|
|
|
|
|
echo
|
2026-03-15 22:39:45 +01:00
|
|
|
echo "${COLOR_BLUE}Installing from sources with uv.lock - generating constraints from uv.lock${COLOR_RESET}"
|
2026-03-15 15:20:53 +01:00
|
|
|
echo
|
2026-03-15 22:39:45 +01:00
|
|
|
uv export --frozen --no-hashes --no-emit-project --no-editable --no-header \
|
|
|
|
|
--no-annotate > "${HOME}/constraints.txt" 2>/dev/null || true
|
2026-03-15 15:20:53 +01:00
|
|
|
return
|
|
|
|
|
fi
|
|
|
|
|
|
2022-03-27 19:19:02 +02:00
|
|
|
# auto-detect Airflow-constraint reference and location
|
|
|
|
|
if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then
|
2025-05-02 20:21:33 +02:00
|
|
|
if [[ ${AIRFLOW_VERSION} =~ v?2.* || ${AIRFLOW_VERSION} =~ v?3.* ]]; then
|
2022-03-27 19:19:02 +02:00
|
|
|
AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION}
|
|
|
|
|
else
|
|
|
|
|
AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH}
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [[ -z ${AIRFLOW_CONSTRAINTS_LOCATION=} ]]; then
|
|
|
|
|
local constraints_base="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${AIRFLOW_CONSTRAINTS_REFERENCE}"
|
|
|
|
|
local python_version
|
2024-03-02 11:29:57 +02:00
|
|
|
python_version=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
|
2022-05-09 23:02:25 +02:00
|
|
|
AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS_MODE}-${python_version}.txt"
|
2022-03-27 19:19:02 +02:00
|
|
|
fi
|
2024-03-02 11:23:58 +01:00
|
|
|
|
|
|
|
|
if [[ ${AIRFLOW_CONSTRAINTS_LOCATION} =~ http.* ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Downloading constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}"
|
|
|
|
|
echo
|
Add Python 3.14 Support (#63520)
* Update python version exclusion to 3.15
* Add 3.14 metadata version classifiers and related constants
* Regenerate Breeze command help screenshots
* Assorted workarounds to fix breeze image building
- constraints are skipped entirely
- greenlet pin updated
* Exclude cassandra
* Exclude amazon
* Exclude google
* CI: Only add pydantic extra to Airflow 2 migration tests
Before this fix there were two separate issues in the migration-test setup for Python 3.14:
1. The migration workflow always passes --airflow-extras pydantic.
2. For Python 3.14, the minimum Airflow version is resolved to 3.2.0 by get_min_airflow_version_for_python.py, and apache-airflow[pydantic]==3.2.0 is not a valid thing to install.
So when constraints installation fails, the fallback path tries to install an invalid spec.
* Disable DB migration tests for python 3.14
* Enforce werkzeug 3.x for python 3.14
* Increase K8s executor test timeout for Python 3.14
Python 3.14 changed the default multiprocessing start method from 'fork' to 'forkserver' on Linux. The forkserver start method is slower because each new process must import modules from scratch rather than copying the parent's address space. This makes `multiprocessing.Manager()` initialization take longer, causing the test to exceed its 10s timeout.
* Adapt LocalExecutor tests for Python 3.14 forkserver default
Python 3.14 changed the default multiprocessing start method from
'fork' to 'forkserver' on Linux. Like 'spawn', 'forkserver' doesn't
share the parent's address space, so mock patches applied in the test
process are invisible to worker subprocesses.
- Skip tests that mock across process boundaries on non-fork methods
- Add test_executor_lazy_worker_spawning to verify that non-fork start
methods defer worker creation and skip gc.freeze
- Make test_multiple_team_executors_isolation and
test_global_executor_without_team_name assert the correct worker
count for each start method instead of assuming pre-spawning
- Remove skip from test_clean_stop_on_signal (works on all methods)
and increase timeout from 5s to 30s for forkserver overhead
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* Bump dependencies to versions supporting 3.14
* Fix PROD image build failing on Python 3.14 due to excluded providers
The PROD image build installed all provider wheels regardless of Python
version compatibility. Providers like google and amazon that exclude
Python 3.14 were still passed to pip, causing resolution failures (e.g.
ray has no cp314 wheel on PyPI).
Two fixes:
- get_distribution_specs.py now reads each wheel's Requires-Python
metadata and skips incompatible wheels instead of passing them to pip.
- The requires-python specifier generation used !=3.14 which per PEP 440
only excludes 3.14.0, not 3.14.3. Changed to !=3.14.* wildcard.
* Split core test types into 2 matrix groups to avoid OOM on Python 3.14
Non-DB core tests use xdist which runs all test types in a single pytest
process. With 2059 items across 4 workers, memory accumulates until the
OOM killer strikes at ~86% completion (exit code 137).
Split core test types into 2 groups (API/Always/CLI and
Core/Other/Serialization), similar to how provider tests already use
_split_list with NUMBER_OF_LOW_DEP_SLICES. Each group gets ~1000 items,
well under the ~1770 threshold where OOM occurs.
Update selective_checks test expectations to reflect the 2-group split.
* Gracefully handle an already removed password file in fixture
The old code had a check-then-act race (if `os.path.exists` → `os.remove`), which fails when the file doesn't exist at removal time. `contextlib.suppress(FileNotFoundError)` handles this atomically — if the file is missing (never created in this xdist worker, or removed between check and delete), it's silently ignored.
* Fix OOM and flaky tests in test_process_utils
Replace multiprocessing.Process with subprocess.Popen running minimal
inline scripts. multiprocessing.Process uses fork(), which duplicates
the entire xdist worker memory. At 95% test completion the worker has
accumulated hundreds of MBs; forking it triggers the OOM killer
(exit code 137) on Python 3.14.
subprocess.Popen starts a fresh lightweight process (~10MB) without
copying the parent's memory, avoiding the OOM entirely.
Also replace the racy ps -ax process counting in
TestKillChildProcessesByPids with psutil.pid_exists() checks on the
specific PID — the old approach was non-deterministic because unrelated
processes could start/stop between measurements.
* Add prek hook to validate python_version markers for excluded providers
When a provider declares excluded-python-versions in provider.yaml,
every dependency string referencing that provider in pyproject.toml
must carry a matching python_version marker. Missing markers cause
excluded providers to be silently installed as transitive dependencies
(e.g. aiobotocore pulling in amazon on Python 3.14).
The new check-excluded-provider-markers hook reads exclusions from
provider.yaml and validates all dependency strings in pyproject.toml
at commit time, preventing regressions like the one fixed in the
previous commit.
* Update `uv.lock`
---------
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-19 23:03:46 +02:00
|
|
|
if ! curl -sSf -o "${HOME}/constraints.txt" "${AIRFLOW_CONSTRAINTS_LOCATION}"; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_YELLOW}Constraints file not found at ${AIRFLOW_CONSTRAINTS_LOCATION} (new Python version being bootstrapped?).${COLOR_RESET}"
|
|
|
|
|
echo "${COLOR_YELLOW}Falling back to no-constraints installation.${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
AIRFLOW_CONSTRAINTS_LOCATION=""
|
|
|
|
|
# Create an empty constraints file so --constraint flag still works
|
|
|
|
|
touch "${HOME}/constraints.txt"
|
|
|
|
|
fi
|
2024-03-02 11:23:58 +01:00
|
|
|
else
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Copying constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
cp "${AIRFLOW_CONSTRAINTS_LOCATION}" "${HOME}/constraints.txt"
|
|
|
|
|
fi
|
2022-03-27 19:19:02 +02:00
|
|
|
}
|
|
|
|
|
|
2024-02-26 13:10:31 +01:00
|
|
|
function common::show_packaging_tool_version_and_location() {
|
2022-03-27 19:19:02 +02:00
|
|
|
echo "PATH=${PATH}"
|
2024-03-02 15:07:06 +01:00
|
|
|
echo "Installed pip: $(pip --version): $(which pip)"
|
2024-02-26 13:10:31 +01:00
|
|
|
if [[ ${PACKAGING_TOOL} == "pip" ]]; then
|
|
|
|
|
echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}"
|
|
|
|
|
else
|
|
|
|
|
echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}"
|
2024-03-02 15:07:06 +01:00
|
|
|
echo "Installed uv: $(uv --version 2>/dev/null || echo "Not installed yet"): $(which uv 2>/dev/null)"
|
2024-02-26 13:10:31 +01:00
|
|
|
fi
|
2022-03-27 19:19:02 +02:00
|
|
|
}
|
2023-01-05 11:31:57 +01:00
|
|
|
|
2024-03-02 15:07:06 +01:00
|
|
|
function common::install_packaging_tools() {
|
2024-11-15 18:46:57 +01:00
|
|
|
: "${AIRFLOW_USE_UV:?Should be set}"
|
2024-03-06 01:27:15 +01:00
|
|
|
if [[ "${VIRTUAL_ENV=}" != "" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Checking packaging tools in venv: ${VIRTUAL_ENV}${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
else
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Checking packaging tools for system Python installation: $(which python)${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
fi
|
2024-10-17 23:43:48 +02:00
|
|
|
if [[ ${AIRFLOW_PIP_VERSION=} == "" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Installing latest pip version${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
pip install --root-user-action ignore --disable-pip-version-check --upgrade pip
|
2024-10-27 20:46:10 +01:00
|
|
|
elif [[ ! ${AIRFLOW_PIP_VERSION} =~ ^[0-9].* ]]; then
|
2024-03-02 15:07:06 +01:00
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Installing pip version from spec ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
|
|
|
|
|
echo
|
2024-02-26 13:10:31 +01:00
|
|
|
# shellcheck disable=SC2086
|
|
|
|
|
pip install --root-user-action ignore --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}"
|
2023-01-05 11:31:57 +01:00
|
|
|
else
|
2024-03-02 15:07:06 +01:00
|
|
|
local installed_pip_version
|
|
|
|
|
installed_pip_version=$(python -c 'from importlib.metadata import version; print(version("pip"))')
|
|
|
|
|
if [[ ${installed_pip_version} != "${AIRFLOW_PIP_VERSION}" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}(Re)Installing pip version: ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
pip install --root-user-action ignore --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
|
|
|
|
|
fi
|
2024-02-26 13:10:31 +01:00
|
|
|
fi
|
2024-10-17 23:43:48 +02:00
|
|
|
if [[ ${AIRFLOW_UV_VERSION=} == "" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Installing latest uv version${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
pip install --root-user-action ignore --disable-pip-version-check --upgrade uv
|
2024-10-27 20:46:10 +01:00
|
|
|
elif [[ ! ${AIRFLOW_UV_VERSION} =~ ^[0-9].* ]]; then
|
2024-02-26 13:10:31 +01:00
|
|
|
echo
|
2024-03-02 15:07:06 +01:00
|
|
|
echo "${COLOR_BLUE}Installing uv version from spec ${AIRFLOW_UV_VERSION}${COLOR_RESET}"
|
2024-02-26 13:10:31 +01:00
|
|
|
echo
|
2024-03-02 15:07:06 +01:00
|
|
|
# shellcheck disable=SC2086
|
|
|
|
|
pip install --root-user-action ignore --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}"
|
|
|
|
|
else
|
|
|
|
|
local installed_uv_version
|
|
|
|
|
installed_uv_version=$(python -c 'from importlib.metadata import version; print(version("uv"))' 2>/dev/null || echo "Not installed yet")
|
|
|
|
|
if [[ ${installed_uv_version} != "${AIRFLOW_UV_VERSION}" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}(Re)Installing uv version: ${AIRFLOW_UV_VERSION}${COLOR_RESET}"
|
|
|
|
|
echo
|
2024-02-26 13:10:31 +01:00
|
|
|
# shellcheck disable=SC2086
|
|
|
|
|
pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}"
|
|
|
|
|
fi
|
2023-01-05 11:31:57 +01:00
|
|
|
fi
|
2025-08-17 09:00:14 +02:00
|
|
|
if [[ ${AIRFLOW_PREK_VERSION=} == "" ]]; then
|
2024-11-15 18:46:57 +01:00
|
|
|
echo
|
2025-08-17 15:07:07 +02:00
|
|
|
echo "${COLOR_BLUE}Installing latest prek, uv${COLOR_RESET}"
|
2024-11-15 18:46:57 +01:00
|
|
|
echo
|
2025-08-17 09:00:14 +02:00
|
|
|
uv tool install prek --with uv
|
2024-11-15 18:46:57 +01:00
|
|
|
# make sure that the venv/user in .local exists
|
|
|
|
|
mkdir -p "${HOME}/.local/bin"
|
|
|
|
|
else
|
|
|
|
|
echo
|
2025-08-17 09:00:14 +02:00
|
|
|
echo "${COLOR_BLUE}Installing predefined versions of prek, uv:${COLOR_RESET}"
|
|
|
|
|
echo "${COLOR_BLUE}prek(${AIRFLOW_PREK_VERSION}) uv(${AIRFLOW_UV_VERSION})${COLOR_RESET}"
|
2024-11-15 18:46:57 +01:00
|
|
|
echo
|
2025-08-17 09:00:14 +02:00
|
|
|
uv tool install "prek==${AIRFLOW_PREK_VERSION}" --with "uv==${AIRFLOW_UV_VERSION}"
|
2024-11-15 18:46:57 +01:00
|
|
|
# make sure that the venv/user in .local exists
|
|
|
|
|
mkdir -p "${HOME}/.local/bin"
|
|
|
|
|
fi
|
2023-01-05 11:31:57 +01:00
|
|
|
}
|
2023-12-18 18:14:04 +04:00
|
|
|
|
|
|
|
|
function common::import_trusted_gpg() {
|
|
|
|
|
common::get_colors
|
|
|
|
|
|
|
|
|
|
local key=${1:?${COLOR_RED}First argument expects OpenPGP Key ID${COLOR_RESET}}
|
|
|
|
|
local name=${2:?${COLOR_RED}Second argument expected trust storage name${COLOR_RESET}}
|
|
|
|
|
# Please note that not all servers could be used for retrieve keys
|
|
|
|
|
# sks-keyservers.net: Unmaintained and DNS taken down due to GDPR requests.
|
|
|
|
|
# keys.openpgp.org: User ID Mandatory, not suitable for APT repositories
|
|
|
|
|
# keyring.debian.org: Only accept keys in Debian keyring.
|
|
|
|
|
# pgp.mit.edu: High response time.
|
|
|
|
|
local keyservers=(
|
|
|
|
|
"hkps://keyserver.ubuntu.com"
|
|
|
|
|
"hkps://pgp.surf.nl"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
GNUPGHOME="$(mktemp -d)"
|
|
|
|
|
export GNUPGHOME
|
|
|
|
|
set +e
|
|
|
|
|
for keyserver in $(shuf -e "${keyservers[@]}"); do
|
|
|
|
|
echo "${COLOR_BLUE}Try to receive GPG public key ${key} from ${keyserver}${COLOR_RESET}"
|
|
|
|
|
gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break
|
|
|
|
|
echo "${COLOR_YELLOW}Unable to receive GPG public key ${key} from ${keyserver}${COLOR_RESET}"
|
|
|
|
|
done
|
|
|
|
|
set -e
|
|
|
|
|
gpg --export "${key}" > "/etc/apt/trusted.gpg.d/${name}.gpg"
|
|
|
|
|
gpgconf --kill all
|
|
|
|
|
rm -rf "${GNUPGHOME}"
|
|
|
|
|
unset GNUPGHOME
|
|
|
|
|
}
|
2022-03-27 19:19:02 +02:00
|
|
|
EOF
|
|
|
|
|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
# The content below is automatically copied from scripts/docker/install_airflow_when_building_images.sh
|
|
|
|
|
COPY <<"EOF" /install_airflow_when_building_images.sh
|
2023-08-21 06:12:48 +08:00
|
|
|
#!/usr/bin/env bash
|
2022-03-27 19:19:02 +02:00
|
|
|
|
|
|
|
|
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
|
|
|
|
|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
function install_from_sources() {
|
2025-04-15 10:55:14 +02:00
|
|
|
local extra_sync_flags
|
|
|
|
|
extra_sync_flags=""
|
|
|
|
|
if [[ ${VIRTUAL_ENV=} != "" ]]; then
|
|
|
|
|
extra_sync_flags="--active"
|
|
|
|
|
fi
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
|
|
|
|
|
if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then
|
|
|
|
|
set +x
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_RED}We only support uv not pip installation for upgrading dependencies!.${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
set +x
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Attempting to upgrade all packages to highest versions.${COLOR_RESET}"
|
|
|
|
|
echo
|
2025-07-11 15:49:00 +02:00
|
|
|
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
|
|
|
|
|
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
|
|
|
|
|
# See https://bugs.launchpad.net/lxml/+bug/2110068
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
set -x
|
2025-07-11 15:49:00 +02:00
|
|
|
uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \
|
2025-09-06 16:54:43 +02:00
|
|
|
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
|
2025-09-06 16:39:41 +01:00
|
|
|
--no-python-downloads --no-managed-python
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
else
|
|
|
|
|
set +x
|
|
|
|
|
echo
|
2026-03-15 15:20:53 +01:00
|
|
|
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
echo
|
2026-03-15 15:20:53 +01:00
|
|
|
# Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving.
|
|
|
|
|
# --no-binary-package is needed in order to avoid libxml and xmlsec using different version of
|
|
|
|
|
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one).
|
|
|
|
|
# See https://bugs.launchpad.net/lxml/+bug/2110068
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
set -x
|
2026-03-15 15:20:53 +01:00
|
|
|
if ! uv sync --all-packages --frozen --group dev --group docs --group docs-gen \
|
|
|
|
|
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
|
|
|
|
|
--no-python-downloads --no-managed-python; then
|
|
|
|
|
set +x
|
2026-02-24 06:43:41 +01:00
|
|
|
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
|
|
|
|
|
echo
|
2026-03-15 15:20:53 +01:00
|
|
|
echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}"
|
2026-02-24 06:43:41 +01:00
|
|
|
echo
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
echo
|
2026-03-15 15:20:53 +01:00
|
|
|
echo "${COLOR_YELLOW}Likely pyproject.toml has new dependencies not reflected in uv.lock.${COLOR_RESET}"
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
echo
|
2026-03-15 15:20:53 +01:00
|
|
|
echo "${COLOR_BLUE}Falling back to re-resolving dependencies (uv sync without --frozen).${COLOR_RESET}"
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
echo
|
|
|
|
|
set -x
|
2025-07-11 15:49:00 +02:00
|
|
|
uv sync --all-packages --group dev --group docs --group docs-gen \
|
2025-09-06 16:54:43 +02:00
|
|
|
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
|
|
|
|
|
--no-python-downloads --no-managed-python
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
set +x
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function install_from_external_spec() {
|
|
|
|
|
local installation_command_flags
|
|
|
|
|
if [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then
|
2024-03-06 01:27:15 +01:00
|
|
|
installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
|
|
|
|
|
else
|
|
|
|
|
echo
|
2026-03-09 17:57:08 -07:00
|
|
|
echo "${COLOR_RED}The '${AIRFLOW_INSTALLATION_METHOD}' installation method is not supported${COLOR_RESET}"
|
2024-03-06 01:27:15 +01:00
|
|
|
echo
|
2025-08-30 20:18:30 +05:30
|
|
|
echo "${COLOR_YELLOW}Supported methods are ('.', 'apache-airflow')${COLOR_RESET}"
|
2022-03-27 19:19:02 +02:00
|
|
|
echo
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
|
2022-03-27 19:19:02 +02:00
|
|
|
echo
|
2025-03-21 14:25:26 +01:00
|
|
|
echo "${COLOR_BLUE}Remove airflow and all provider distributions installed before potentially${COLOR_RESET}"
|
2022-03-27 19:19:02 +02:00
|
|
|
echo
|
2024-02-14 03:08:15 +01:00
|
|
|
set -x
|
2024-02-26 13:10:31 +01:00
|
|
|
${PACKAGING_TOOL_CMD} freeze | grep apache-airflow | xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true
|
2024-02-14 03:08:15 +01:00
|
|
|
set +x
|
|
|
|
|
echo
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
echo "${COLOR_BLUE}Installing all packages with highest resolutions. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
|
2024-02-14 03:08:15 +01:00
|
|
|
echo
|
|
|
|
|
set -x
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
|
2024-02-14 03:08:15 +01:00
|
|
|
set +x
|
|
|
|
|
else
|
2022-03-27 19:19:02 +02:00
|
|
|
echo
|
2024-03-06 01:27:15 +01:00
|
|
|
echo "${COLOR_BLUE}Installing all packages with constraints. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
|
2022-03-27 19:19:02 +02:00
|
|
|
echo
|
2022-05-09 23:02:25 +02:00
|
|
|
set -x
|
2024-03-06 01:27:15 +01:00
|
|
|
if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
|
2024-03-02 15:07:06 +01:00
|
|
|
set +x
|
2026-02-24 06:43:41 +01:00
|
|
|
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_RED}Failing because constraints installation failed and fallback is disabled.${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
2024-03-02 11:23:58 +01:00
|
|
|
echo
|
|
|
|
|
echo "${COLOR_YELLOW}Likely pyproject.toml has new dependencies conflicting with constraints.${COLOR_RESET}"
|
|
|
|
|
echo
|
2024-03-13 15:59:19 +01:00
|
|
|
echo "${COLOR_BLUE}Falling back to no-constraints installation.${COLOR_RESET}"
|
2024-03-02 11:23:58 +01:00
|
|
|
echo
|
2024-03-02 15:07:06 +01:00
|
|
|
set -x
|
2024-03-06 01:27:15 +01:00
|
|
|
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
set +x
|
2024-03-02 11:23:58 +01:00
|
|
|
fi
|
2022-03-27 19:19:02 +02:00
|
|
|
fi
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
}
|
|
|
|
|
|
2022-03-27 19:19:02 +02:00
|
|
|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
function install_airflow_when_building_images() {
|
|
|
|
|
# Remove mysql from extras if client is not going to be installed
|
|
|
|
|
if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then
|
|
|
|
|
AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}
|
|
|
|
|
echo "${COLOR_YELLOW}MYSQL client installation is disabled. Extra 'mysql' installations were therefore omitted.${COLOR_RESET}"
|
|
|
|
|
fi
|
|
|
|
|
# Remove postgres from extras if client is not going to be installed
|
|
|
|
|
if [[ ${INSTALL_POSTGRES_CLIENT} != "true" ]]; then
|
|
|
|
|
AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/postgres,}
|
|
|
|
|
echo "${COLOR_YELLOW}Postgres client installation is disabled. Extra 'postgres' installations were therefore omitted.${COLOR_RESET}"
|
|
|
|
|
fi
|
|
|
|
|
# Determine the installation_command_flags based on AIRFLOW_INSTALLATION_METHOD method
|
|
|
|
|
if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then
|
|
|
|
|
install_from_sources
|
|
|
|
|
else
|
|
|
|
|
install_from_external_spec
|
|
|
|
|
fi
|
|
|
|
|
set +x
|
|
|
|
|
common::install_packaging_tools
|
|
|
|
|
echo
|
2025-12-20 17:42:51 +01:00
|
|
|
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
echo
|
2025-12-26 23:25:21 +01:00
|
|
|
# We use pip check here to make sure that whatever `uv` installs, is also "correct" according to `pip`
|
|
|
|
|
pip check
|
2022-03-27 19:19:02 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
common::get_colors
|
2024-02-26 13:10:31 +01:00
|
|
|
common::get_packaging_tool
|
2022-03-27 19:19:02 +02:00
|
|
|
common::get_airflow_version_specification
|
|
|
|
|
common::get_constraints_location
|
2024-02-26 13:10:31 +01:00
|
|
|
common::show_packaging_tool_version_and_location
|
2022-03-27 19:19:02 +02:00
|
|
|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
install_airflow_when_building_images
|
2022-03-27 19:19:02 +02:00
|
|
|
EOF
|
|
|
|
|
|
|
|
|
|
# The content below is automatically copied from scripts/docker/install_additional_dependencies.sh
|
2022-03-31 13:39:46 +02:00
|
|
|
COPY <<"EOF" /install_additional_dependencies.sh
|
2023-08-21 06:12:48 +08:00
|
|
|
#!/usr/bin/env bash
|
2022-03-27 19:19:02 +02:00
|
|
|
set -euo pipefail
|
|
|
|
|
|
|
|
|
|
: "${ADDITIONAL_PYTHON_DEPS:?Should be set}"
|
|
|
|
|
|
|
|
|
|
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
|
|
|
|
|
|
|
|
|
|
function install_additional_dependencies() {
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
|
2022-03-27 19:19:02 +02:00
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}"
|
|
|
|
|
echo
|
2022-05-09 23:02:25 +02:00
|
|
|
set -x
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} \
|
2022-07-27 17:07:53 +02:00
|
|
|
${ADDITIONAL_PIP_INSTALL_FLAGS} \
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
${ADDITIONAL_PYTHON_DEPS}
|
2022-05-09 23:02:25 +02:00
|
|
|
set +x
|
2024-03-02 15:07:06 +01:00
|
|
|
common::install_packaging_tools
|
2022-03-27 19:19:02 +02:00
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
|
|
|
|
|
echo
|
2025-12-26 23:25:21 +01:00
|
|
|
# We use pip check here to make sure that whatever `uv` installs, is also "correct" according to `pip`
|
|
|
|
|
pip check
|
2022-03-27 19:19:02 +02:00
|
|
|
else
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}"
|
|
|
|
|
echo
|
2022-05-09 23:02:25 +02:00
|
|
|
set -x
|
2024-03-06 01:27:15 +01:00
|
|
|
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} \
|
2022-07-27 17:07:53 +02:00
|
|
|
${ADDITIONAL_PIP_INSTALL_FLAGS} \
|
2022-03-27 19:19:02 +02:00
|
|
|
${ADDITIONAL_PYTHON_DEPS}
|
2022-05-09 23:02:25 +02:00
|
|
|
set +x
|
2024-03-02 15:07:06 +01:00
|
|
|
common::install_packaging_tools
|
2022-03-27 19:19:02 +02:00
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
|
|
|
|
|
echo
|
2025-12-26 23:25:21 +01:00
|
|
|
# We use pip check here to make sure that whatever `uv` installs, is also "correct" according to `pip`
|
|
|
|
|
pip check
|
2022-03-27 19:19:02 +02:00
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
common::get_colors
|
2024-02-26 13:10:31 +01:00
|
|
|
common::get_packaging_tool
|
2022-03-27 19:19:02 +02:00
|
|
|
common::get_airflow_version_specification
|
|
|
|
|
common::get_constraints_location
|
2024-02-26 13:10:31 +01:00
|
|
|
common::show_packaging_tool_version_and_location
|
2022-03-27 19:19:02 +02:00
|
|
|
|
|
|
|
|
install_additional_dependencies
|
|
|
|
|
EOF
|
|
|
|
|
|
|
|
|
|
# The content below is automatically copied from scripts/docker/entrypoint_ci.sh
|
2022-03-31 13:39:46 +02:00
|
|
|
COPY <<"EOF" /entrypoint_ci.sh
|
2022-03-27 19:19:02 +02:00
|
|
|
#!/usr/bin/env bash
|
Add Python 3.13 support for Airflow. (#46891)
* Breeze changes for Python 3.13
* Pyproject.toml provider changes for Python 3.13
* README.rst changes for providers for Python 3.13
* Add Python 3.13 support for Airflow.
Added Python 3.13 support across the codebase, including:
* CI/CD workflow updates to handle Python 3.13.
* Dockerfile and environment variable adjustments for compatibility.
* Documentation updates to list Python 3.13 as supported.
* Dependency and code changes for compatibility with new/changed
libraries (e.g., numpy 2.x, greenlet, pendulum).
* Test and plugin manager adjustments for Python 3.13-specific behaviors.
* Removal or skipping of tests and features not compatible with Python 3.13.
* Improved error handling and logging for unsupported plugins/providers
on Python 3.13.
* Updated all provider README.rst files to:
* List Python 3.13 as a supported version.
* Adjust dependency requirements for Python 3.13 (e.g., pandas,
pyarrow, ray, etc.).
* Add or update notes about excluded or conditionally supported
dependencies for Python 3.13.
* Document any known incompatibilities or workarounds for specific
providers.
* Updated all provider pyproject.toml files and automation to:
* Add Python 3.13 to the supported classifiers.
* Adjust requires-python and dependency constraints for Python 3.13.
* Exclude Python 3.13 for providers that are not yet compatible (e.g.,
those depending on Flask AppBuilder).
* Add conditional dependencies for new versions of libraries required
by Python 3.13.
* Updated Breeze and related scripts to:
* Build and test with Python 3.13.
* Update documentation and release scripts to include Python 3.13.
* Ensure all dev tooling and test environments work with Python 3.13.
* Removed remaining FAB related code from core and removed tests for
older FAB provider versions (<1.3.0)
* some of the tests in core still depend on FAB - but we make them
optional - both for parsing (we need to skip some imports) and
execution - because they test a legacy behaviour of embedding
Flask plugins that is still supported in core.
* k8s tests now run also with SimpleAuthManager and SimpleAuthManager
is used automatically when runnin tests in Python 3.13
* migration tests had to be excluded fo Python 3.13 because we cannot
migrate down on Python 3.13 without FAB tables created. The tests
will be brought back when FAB is supported or when we release 3.1
and start working on 3.2, we should be able to migrate down to 3.1
* Weaviate tests have been updated to handle case where weaviate-client
is > 2.10.0 - because it started to use httpx instead of requests,
and Python 3.13 actually unblocks using older version of weaviate
(it was previously blocked indirectly via Flask dependencies and
grpcio).
* Added doc change in our rules on what is the "default" Python version
for airflow images - handling the case where we cannot use "newest"
Python version as default where not all "regular" image providers are
supported in the latest version
* Connection is now flushed in `configure_git_connection_for_dag_bundle`
just before `clear_db_connections` - in order to handle Python 3.13
case where FAB app is not initialized by other fixtures - performing
implicit flush along the way
* Cleaning db connections was added to test_dag_run and test_dags test
in fast_api as the test implicitly rely on having connections
cleared (no test git connections added) - this will make the tests
side-effect free.
* Conditionally skipping some serialization tests involving yandex
and ray on Python 3.13 until they support Python 3.13
2025-07-17 16:53:49 +02:00
|
|
|
function set_verbose() {
|
|
|
|
|
if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then
|
|
|
|
|
set -x
|
|
|
|
|
else
|
|
|
|
|
set +x
|
|
|
|
|
fi
|
|
|
|
|
}
|
2022-03-27 19:19:02 +02:00
|
|
|
|
Add Python 3.13 support for Airflow. (#46891)
* Breeze changes for Python 3.13
* Pyproject.toml provider changes for Python 3.13
* README.rst changes for providers for Python 3.13
* Add Python 3.13 support for Airflow.
Added Python 3.13 support across the codebase, including:
* CI/CD workflow updates to handle Python 3.13.
* Dockerfile and environment variable adjustments for compatibility.
* Documentation updates to list Python 3.13 as supported.
* Dependency and code changes for compatibility with new/changed
libraries (e.g., numpy 2.x, greenlet, pendulum).
* Test and plugin manager adjustments for Python 3.13-specific behaviors.
* Removal or skipping of tests and features not compatible with Python 3.13.
* Improved error handling and logging for unsupported plugins/providers
on Python 3.13.
* Updated all provider README.rst files to:
* List Python 3.13 as a supported version.
* Adjust dependency requirements for Python 3.13 (e.g., pandas,
pyarrow, ray, etc.).
* Add or update notes about excluded or conditionally supported
dependencies for Python 3.13.
* Document any known incompatibilities or workarounds for specific
providers.
* Updated all provider pyproject.toml files and automation to:
* Add Python 3.13 to the supported classifiers.
* Adjust requires-python and dependency constraints for Python 3.13.
* Exclude Python 3.13 for providers that are not yet compatible (e.g.,
those depending on Flask AppBuilder).
* Add conditional dependencies for new versions of libraries required
by Python 3.13.
* Updated Breeze and related scripts to:
* Build and test with Python 3.13.
* Update documentation and release scripts to include Python 3.13.
* Ensure all dev tooling and test environments work with Python 3.13.
* Removed remaining FAB related code from core and removed tests for
older FAB provider versions (<1.3.0)
* some of the tests in core still depend on FAB - but we make them
optional - both for parsing (we need to skip some imports) and
execution - because they test a legacy behaviour of embedding
Flask plugins that is still supported in core.
* k8s tests now run also with SimpleAuthManager and SimpleAuthManager
is used automatically when runnin tests in Python 3.13
* migration tests had to be excluded fo Python 3.13 because we cannot
migrate down on Python 3.13 without FAB tables created. The tests
will be brought back when FAB is supported or when we release 3.1
and start working on 3.2, we should be able to migrate down to 3.1
* Weaviate tests have been updated to handle case where weaviate-client
is > 2.10.0 - because it started to use httpx instead of requests,
and Python 3.13 actually unblocks using older version of weaviate
(it was previously blocked indirectly via Flask dependencies and
grpcio).
* Added doc change in our rules on what is the "default" Python version
for airflow images - handling the case where we cannot use "newest"
Python version as default where not all "regular" image providers are
supported in the latest version
* Connection is now flushed in `configure_git_connection_for_dag_bundle`
just before `clear_db_connections` - in order to handle Python 3.13
case where FAB app is not initialized by other fixtures - performing
implicit flush along the way
* Cleaning db connections was added to test_dag_run and test_dags test
in fast_api as the test implicitly rely on having connections
cleared (no test git connections added) - this will make the tests
side-effect free.
* Conditionally skipping some serialization tests involving yandex
and ray on Python 3.13 until they support Python 3.13
2025-07-17 16:53:49 +02:00
|
|
|
set_verbose
|
2022-08-15 18:05:46 +03:00
|
|
|
. "${AIRFLOW_SOURCES:-/opt/airflow}"/scripts/in_container/_in_container_script_init.sh
|
2022-03-27 19:19:02 +02:00
|
|
|
|
|
|
|
|
LD_PRELOAD="/usr/lib/$(uname -m)-linux-gnu/libstdc++.so.6"
|
|
|
|
|
export LD_PRELOAD
|
|
|
|
|
|
|
|
|
|
chmod 1777 /tmp
|
|
|
|
|
|
|
|
|
|
AIRFLOW_SOURCES=$(cd "${IN_CONTAINER_DIR}/../.." || exit 1; pwd)
|
|
|
|
|
|
2025-06-29 17:29:46 +02:00
|
|
|
PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.10}
|
2022-03-27 19:19:02 +02:00
|
|
|
|
|
|
|
|
export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}}
|
|
|
|
|
|
2023-12-10 16:07:06 +01:00
|
|
|
mkdir "${AIRFLOW_HOME}/sqlite" -p || true
|
2022-03-27 19:19:02 +02:00
|
|
|
|
2023-07-30 03:49:27 +05:30
|
|
|
ASSET_COMPILATION_WAIT_MULTIPLIER=${ASSET_COMPILATION_WAIT_MULTIPLIER:=1}
|
|
|
|
|
|
2025-06-29 14:23:17 +02:00
|
|
|
if [[ "${CI=}" == "true" ]]; then
|
|
|
|
|
export COLUMNS="202"
|
|
|
|
|
fi
|
|
|
|
|
|
2024-11-27 13:14:52 +00:00
|
|
|
. "${IN_CONTAINER_DIR}/check_connectivity.sh"
|
|
|
|
|
|
2022-12-24 14:48:14 +01:00
|
|
|
function wait_for_asset_compilation() {
|
2025-02-28 11:01:03 +01:00
|
|
|
if [[ -f "${AIRFLOW_SOURCES}/.build/ui/.asset_compile.lock" ]]; then
|
2022-12-24 14:48:14 +01:00
|
|
|
echo
|
|
|
|
|
echo "${COLOR_YELLOW}Waiting for asset compilation to complete in the background.${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
local counter=0
|
2025-02-28 11:01:03 +01:00
|
|
|
while [[ -f "${AIRFLOW_SOURCES}/.build/ui/.asset_compile.lock" ]]; do
|
2023-04-11 04:06:16 +08:00
|
|
|
if (( counter % 5 == 2 )); then
|
|
|
|
|
echo "${COLOR_BLUE}Still waiting .....${COLOR_RESET}"
|
|
|
|
|
fi
|
2022-12-24 14:48:14 +01:00
|
|
|
sleep 1
|
|
|
|
|
((counter=counter+1))
|
2023-07-30 03:49:27 +05:30
|
|
|
if [[ ${counter} == 30*$ASSET_COMPILATION_WAIT_MULTIPLIER ]]; then
|
2022-12-24 14:48:14 +01:00
|
|
|
echo
|
|
|
|
|
echo "${COLOR_YELLOW}The asset compilation is taking too long.${COLOR_YELLOW}"
|
|
|
|
|
echo """
|
|
|
|
|
If it does not complete soon, you might want to stop it and remove file lock:
|
|
|
|
|
* press Ctrl-C
|
2025-02-28 11:01:03 +01:00
|
|
|
* run 'rm ${AIRFLOW_SOURCES}/.build/ui/.asset_compile.lock'
|
2022-12-24 14:48:14 +01:00
|
|
|
"""
|
|
|
|
|
fi
|
2023-07-30 03:49:27 +05:30
|
|
|
if [[ ${counter} == 60*$ASSET_COMPILATION_WAIT_MULTIPLIER ]]; then
|
2022-12-24 14:48:14 +01:00
|
|
|
echo
|
|
|
|
|
echo "${COLOR_RED}The asset compilation is taking too long. Exiting.${COLOR_RED}"
|
2024-01-21 21:41:54 +01:00
|
|
|
echo "${COLOR_RED}refer to dev/breeze/doc/04_troubleshooting.rst for resolution steps.${COLOR_RED}"
|
2022-12-24 14:48:14 +01:00
|
|
|
echo
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
done
|
|
|
|
|
fi
|
2025-02-28 11:01:03 +01:00
|
|
|
if [ -f "${AIRFLOW_SOURCES}/.build/ui/asset_compile.out" ]; then
|
2023-01-26 23:40:56 +01:00
|
|
|
echo
|
|
|
|
|
echo "${COLOR_RED}The asset compilation failed. Exiting.${COLOR_RESET}"
|
|
|
|
|
echo
|
2025-02-28 11:01:03 +01:00
|
|
|
cat "${AIRFLOW_SOURCES}/.build/ui/asset_compile.out"
|
|
|
|
|
rm "${AIRFLOW_SOURCES}/.build/ui/asset_compile.out"
|
2023-01-26 23:40:56 +01:00
|
|
|
echo
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
2022-12-24 14:48:14 +01:00
|
|
|
}
|
|
|
|
|
|
2023-12-10 16:07:06 +01:00
|
|
|
function environment_initialization() {
|
|
|
|
|
if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} == "true" ]]; then
|
|
|
|
|
return
|
|
|
|
|
fi
|
2022-03-27 19:19:02 +02:00
|
|
|
echo
|
2022-04-03 22:35:26 +02:00
|
|
|
echo "${COLOR_BLUE}Running Initialization. Your basic configuration is:${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
echo " * ${COLOR_BLUE}Airflow home:${COLOR_RESET} ${AIRFLOW_HOME}"
|
|
|
|
|
echo " * ${COLOR_BLUE}Airflow sources:${COLOR_RESET} ${AIRFLOW_SOURCES}"
|
2024-03-14 19:14:58 +01:00
|
|
|
echo " * ${COLOR_BLUE}Airflow core SQL connection:${COLOR_RESET} ${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN:=}"
|
|
|
|
|
if [[ ${BACKEND=} == "postgres" ]]; then
|
|
|
|
|
echo " * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Postgres: ${POSTGRES_VERSION}"
|
|
|
|
|
elif [[ ${BACKEND=} == "mysql" ]]; then
|
|
|
|
|
echo " * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} MySQL: ${MYSQL_VERSION}"
|
|
|
|
|
elif [[ ${BACKEND=} == "sqlite" ]]; then
|
|
|
|
|
echo " * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Sqlite"
|
2026-03-12 16:15:35 -03:00
|
|
|
elif [[ ${BACKEND=} == "custom" ]]; then
|
|
|
|
|
local _conn_url="${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN:-}"
|
|
|
|
|
local _masked_url
|
|
|
|
|
_masked_url=$(echo "${_conn_url}" | sed -E 's|://([^:]+):([^@]+)@|://\1:***@|')
|
|
|
|
|
echo " * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Custom (${_masked_url})"
|
2024-03-14 19:14:58 +01:00
|
|
|
fi
|
2022-03-27 19:19:02 +02:00
|
|
|
echo
|
|
|
|
|
|
2023-10-23 21:30:45 +02:00
|
|
|
if [[ ${STANDALONE_DAG_PROCESSOR=} == "true" ]]; then
|
|
|
|
|
echo
|
2025-03-05 17:27:22 +05:30
|
|
|
echo "${COLOR_BLUE}Forcing scheduler/standalone_dag_processor to True${COLOR_RESET}"
|
2023-10-23 21:30:45 +02:00
|
|
|
echo
|
|
|
|
|
export AIRFLOW__SCHEDULER__STANDALONE_DAG_PROCESSOR=True
|
|
|
|
|
fi
|
2023-12-10 16:07:06 +01:00
|
|
|
|
2026-03-21 20:43:46 +05:30
|
|
|
if [[ ${GO_WORKER=} == "true" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Starting go worker${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
export AIRFLOW__SCHEDULER__GO_WORKER=True
|
|
|
|
|
fi
|
|
|
|
|
|
2024-07-30 23:34:32 +02:00
|
|
|
RUN_TESTS=${RUN_TESTS:="false"}
|
2022-03-27 19:19:02 +02:00
|
|
|
CI=${CI:="false"}
|
|
|
|
|
|
|
|
|
|
# Added to have run-tests on path
|
2025-07-11 21:18:10 +05:30
|
|
|
export PATH=${PATH}:${AIRFLOW_SOURCES}:/usr/local/go/bin/
|
2022-03-27 19:19:02 +02:00
|
|
|
|
|
|
|
|
mkdir -pv "${AIRFLOW_HOME}/logs/"
|
|
|
|
|
|
|
|
|
|
# Change the default worker_concurrency for tests
|
|
|
|
|
export AIRFLOW__CELERY__WORKER_CONCURRENCY=8
|
|
|
|
|
|
|
|
|
|
set +e
|
|
|
|
|
|
2025-08-10 22:14:35 +02:00
|
|
|
# shellcheck source=scripts/in_container/configure_environment.sh
|
|
|
|
|
. "${IN_CONTAINER_DIR}/configure_environment.sh"
|
|
|
|
|
# shellcheck source=scripts/in_container/run_init_script.sh
|
|
|
|
|
. "${IN_CONTAINER_DIR}/run_init_script.sh"
|
|
|
|
|
|
2022-03-27 19:19:02 +02:00
|
|
|
"${IN_CONTAINER_DIR}/check_environment.sh"
|
|
|
|
|
ENVIRONMENT_EXIT_CODE=$?
|
|
|
|
|
set -e
|
|
|
|
|
if [[ ${ENVIRONMENT_EXIT_CODE} != 0 ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "Error: check_environment returned ${ENVIRONMENT_EXIT_CODE}. Exiting."
|
|
|
|
|
echo
|
|
|
|
|
exit ${ENVIRONMENT_EXIT_CODE}
|
|
|
|
|
fi
|
2025-08-10 22:14:35 +02:00
|
|
|
|
2022-03-27 19:19:02 +02:00
|
|
|
mkdir -p /usr/lib/google-cloud-sdk/bin
|
|
|
|
|
touch /usr/lib/google-cloud-sdk/bin/gcloud
|
|
|
|
|
ln -s -f /usr/bin/gcloud /usr/lib/google-cloud-sdk/bin/gcloud
|
|
|
|
|
|
|
|
|
|
if [[ ${SKIP_SSH_SETUP="false"} == "false" ]]; then
|
|
|
|
|
# Set up ssh keys
|
|
|
|
|
echo 'yes' | ssh-keygen -t rsa -C your_email@youremail.com -m PEM -P '' -f ~/.ssh/id_rsa \
|
|
|
|
|
>"${AIRFLOW_HOME}/logs/ssh-keygen.log" 2>&1
|
|
|
|
|
|
|
|
|
|
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
|
|
|
|
|
ln -s -f ~/.ssh/authorized_keys ~/.ssh/authorized_keys2
|
|
|
|
|
chmod 600 ~/.ssh/*
|
|
|
|
|
|
|
|
|
|
# SSH Service
|
|
|
|
|
sudo service ssh restart >/dev/null 2>&1
|
|
|
|
|
|
|
|
|
|
# Sometimes the server is not quick enough to load the keys!
|
|
|
|
|
while [[ $(ssh-keyscan -H localhost 2>/dev/null | wc -l) != "3" ]] ; do
|
|
|
|
|
echo "Not all keys yet loaded by the server"
|
|
|
|
|
sleep 0.05
|
|
|
|
|
done
|
|
|
|
|
|
|
|
|
|
ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null
|
|
|
|
|
fi
|
|
|
|
|
|
2025-09-13 12:02:52 -05:00
|
|
|
if [[ ${INTEGRATION_LOCALSTACK:-"false"} == "true" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Configuring LocalStack integration${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
|
|
|
|
|
# Define LocalStack AWS configuration
|
|
|
|
|
declare -A localstack_config=(
|
|
|
|
|
["AWS_ENDPOINT_URL"]="http://localstack:4566"
|
|
|
|
|
["AWS_ACCESS_KEY_ID"]="test"
|
|
|
|
|
["AWS_SECRET_ACCESS_KEY"]="test"
|
|
|
|
|
["AWS_DEFAULT_REGION"]="us-east-1"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Export each configuration variable and log it
|
|
|
|
|
for key in "${!localstack_config[@]}"; do
|
|
|
|
|
export "$key"="${localstack_config[$key]}"
|
|
|
|
|
echo " * ${COLOR_BLUE}${key}:${COLOR_RESET} ${localstack_config[$key]}"
|
|
|
|
|
done
|
|
|
|
|
echo
|
|
|
|
|
fi
|
|
|
|
|
|
2022-03-27 19:19:02 +02:00
|
|
|
cd "${AIRFLOW_SOURCES}"
|
|
|
|
|
|
2025-04-30 21:45:09 +02:00
|
|
|
# Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags
|
|
|
|
|
# in the UI when testing in Breeze. This might be solved differently in the future
|
|
|
|
|
if [[ -d /opt/airflow/providers/standard/tests ]]; then
|
|
|
|
|
export PYTHONPATH=${PYTHONPATH=}:/opt/airflow/providers/standard/tests
|
|
|
|
|
fi
|
|
|
|
|
|
2022-04-03 22:35:26 +02:00
|
|
|
if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then
|
2025-11-26 12:07:48 +01:00
|
|
|
if [[ ${BREEZE_DEBUG_CELERY_WORKER=} == "true" ]]; then
|
|
|
|
|
export AIRFLOW__CELERY__POOL=${AIRFLOW__CELERY__POOL:-solo}
|
|
|
|
|
fi
|
2022-03-27 19:19:02 +02:00
|
|
|
export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES}
|
2026-01-29 21:33:56 -05:00
|
|
|
if [[ ${SKIP_ASSETS_COMPILATION:="false"} == "false" ]]; then
|
|
|
|
|
wait_for_asset_compilation
|
|
|
|
|
fi
|
2026-01-23 00:40:52 +01:00
|
|
|
if [[ ${TERMINAL_MULTIPLEXER:="mprocs"} == "mprocs" ]]; then
|
2025-11-26 12:07:48 +01:00
|
|
|
# shellcheck source=scripts/in_container/bin/run_mprocs
|
|
|
|
|
exec run_mprocs
|
|
|
|
|
else
|
|
|
|
|
# shellcheck source=scripts/in_container/bin/run_tmux
|
|
|
|
|
exec run_tmux
|
|
|
|
|
fi
|
2022-03-27 19:19:02 +02:00
|
|
|
fi
|
2023-12-10 16:07:06 +01:00
|
|
|
}
|
|
|
|
|
|
2025-01-15 22:51:32 +01:00
|
|
|
function handle_mount_sources() {
|
|
|
|
|
if [[ ${MOUNT_SOURCES=} == "remove" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Mounted sources are removed, cleaning up mounted dist-info files${COLOR_RESET}"
|
|
|
|
|
echo
|
2025-06-30 16:17:47 +03:00
|
|
|
rm -rf /usr/local/lib/python"${PYTHON_MAJOR_MINOR_VERSION}"/site-packages/apache_airflow*.dist-info/
|
2025-01-15 22:51:32 +01:00
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
2023-12-10 16:07:06 +01:00
|
|
|
function determine_airflow_to_use() {
|
|
|
|
|
USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION:=""}"
|
2025-11-28 03:07:18 +01:00
|
|
|
if [[ "${USE_AIRFLOW_VERSION}" == "" && "${USE_DISTRIBUTIONS_FROM_DIST=}" != "true" ]]; then
|
2023-12-10 16:07:06 +01:00
|
|
|
export PYTHONPATH=${AIRFLOW_SOURCES}
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Using airflow version from current sources${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
# Cleanup the logs, tmp when entering the environment
|
|
|
|
|
sudo rm -rf "${AIRFLOW_SOURCES}"/logs/*
|
|
|
|
|
sudo rm -rf "${AIRFLOW_SOURCES}"/tmp/*
|
|
|
|
|
mkdir -p "${AIRFLOW_SOURCES}"/logs/
|
|
|
|
|
mkdir -p "${AIRFLOW_SOURCES}"/tmp/
|
|
|
|
|
else
|
2024-08-19 00:12:29 +02:00
|
|
|
if [[ ${CLEAN_AIRFLOW_INSTALLATION=} == "true" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Uninstalling all packages first${COLOR_RESET}"
|
|
|
|
|
echo
|
2024-12-29 22:58:27 +01:00
|
|
|
# shellcheck disable=SC2086
|
2025-05-11 21:55:29 +02:00
|
|
|
${PACKAGING_TOOL_CMD} freeze | grep -ve "^-e" | grep -ve "^#" | grep -ve "^uv" | grep -v "@" | \
|
2024-12-29 22:58:27 +01:00
|
|
|
xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS}
|
2024-08-19 00:12:29 +02:00
|
|
|
# Now install rich ad click first to use the installation script
|
2024-12-29 22:58:27 +01:00
|
|
|
# shellcheck disable=SC2086
|
2025-03-05 23:04:00 +01:00
|
|
|
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} rich rich-click click \
|
2024-08-19 00:12:29 +02:00
|
|
|
--constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt
|
|
|
|
|
fi
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Reinstalling all development dependencies${COLOR_RESET}"
|
|
|
|
|
echo
|
2026-03-15 15:20:53 +01:00
|
|
|
# Generate constraints from uv.lock and use them to install development dependencies
|
|
|
|
|
# via the Python script. --no-cache is needed - otherwise there is possibility of
|
|
|
|
|
# overriding temporary environments by multiple parallel processes
|
|
|
|
|
local constraint_file="/tmp/constraints-from-lock.txt"
|
2026-03-21 09:38:47 +01:00
|
|
|
uv export --frozen --no-hashes --no-emit-project --no-emit-workspace --no-editable --no-header \
|
2026-03-15 15:20:53 +01:00
|
|
|
--no-annotate > "${constraint_file}" 2>/dev/null || true
|
2025-05-22 12:03:59 +02:00
|
|
|
uv run --no-cache /opt/airflow/scripts/in_container/install_development_dependencies.py \
|
2026-03-15 15:20:53 +01:00
|
|
|
--constraint "${constraint_file}"
|
2024-05-15 16:16:09 -04:00
|
|
|
# Some packages might leave legacy typing module which causes test issues
|
2024-12-29 22:58:27 +01:00
|
|
|
# shellcheck disable=SC2086
|
|
|
|
|
${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} typing || true
|
2025-03-05 23:04:00 +01:00
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Installing airflow and providers ${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
python "${IN_CONTAINER_DIR}/install_airflow_and_providers.py"
|
2023-12-10 16:07:06 +01:00
|
|
|
fi
|
2025-03-28 14:44:36 +01:00
|
|
|
if [[ "${USE_AIRFLOW_VERSION}" =~ ^2.* ]]; then
|
|
|
|
|
# Remove auth manager setting
|
|
|
|
|
unset AIRFLOW__CORE__AUTH_MANAGER
|
|
|
|
|
fi
|
2022-03-27 19:19:02 +02:00
|
|
|
|
2023-12-10 16:07:06 +01:00
|
|
|
if [[ "${USE_AIRFLOW_VERSION}" =~ ^2\.2\..*|^2\.1\..*|^2\.0\..* && "${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=}" != "" ]]; then
|
|
|
|
|
# make sure old variable is used for older airflow versions
|
|
|
|
|
export AIRFLOW__CORE__SQL_ALCHEMY_CONN="${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN}"
|
|
|
|
|
fi
|
|
|
|
|
}
|
2023-08-22 01:51:26 +04:00
|
|
|
|
2023-12-10 16:07:06 +01:00
|
|
|
function check_boto_upgrade() {
|
|
|
|
|
if [[ ${UPGRADE_BOTO=} != "true" ]]; then
|
|
|
|
|
return
|
|
|
|
|
fi
|
2023-09-12 11:49:56 +02:00
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
|
|
|
|
|
echo
|
2024-02-26 13:10:31 +01:00
|
|
|
# shellcheck disable=SC2086
|
2025-02-07 01:23:55 +05:30
|
|
|
${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs || true
|
2025-12-06 11:10:51 +01:00
|
|
|
|
|
|
|
|
# Urllib 2.6.0 breaks kubernetes client because kubernetes client uses deprecated in 2.0.0 and
|
|
|
|
|
# removed in 2.6.0 `getheaders()` call (instead of `headers` property.
|
|
|
|
|
# Tracked in https://github.com/kubernetes-client/python/issues/2477
|
2025-02-07 01:23:55 +05:30
|
|
|
# shellcheck disable=SC2086
|
2025-12-06 11:10:51 +01:00
|
|
|
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "boto3<1.38.3" "botocore<1.38.3" "urllib3<2.6.0"
|
2023-12-10 16:07:06 +01:00
|
|
|
}
|
|
|
|
|
|
2025-06-30 16:17:47 +03:00
|
|
|
function check_upgrade_sqlalchemy() {
|
2025-08-10 20:25:27 +03:00
|
|
|
# The python version constraint is a TEMPORARY WORKAROUND to exclude all FAB tests. Is should be removed once we
|
|
|
|
|
# upgrade FAB to v5 (PR #50960).
|
2025-11-28 03:07:18 +01:00
|
|
|
if [[ "${UPGRADE_SQLALCHEMY=}" != "true" || ${PYTHON_MAJOR_MINOR_VERSION} != "3.13" ]]; then
|
2025-06-30 16:17:47 +03:00
|
|
|
return
|
|
|
|
|
fi
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Upgrading sqlalchemy to the latest version to run tests with it${COLOR_RESET}"
|
|
|
|
|
echo
|
2025-09-06 16:54:43 +02:00
|
|
|
uv sync --all-packages --no-install-package apache-airflow-providers-fab --resolution highest \
|
|
|
|
|
--no-python-downloads --no-managed-python
|
2025-06-30 16:17:47 +03:00
|
|
|
}
|
|
|
|
|
|
2024-03-25 08:06:12 +01:00
|
|
|
function check_downgrade_sqlalchemy() {
|
2023-12-10 16:07:06 +01:00
|
|
|
if [[ ${DOWNGRADE_SQLALCHEMY=} != "true" ]]; then
|
|
|
|
|
return
|
|
|
|
|
fi
|
2025-03-21 14:25:26 +01:00
|
|
|
local min_sqlalchemy_version
|
2025-04-23 19:49:08 +02:00
|
|
|
min_sqlalchemy_version=$(grep "sqlalchemy\[asyncio\]>=" airflow-core/pyproject.toml | sed "s/.*>=\([0-9\.]*\).*/\1/" | xargs)
|
2023-09-12 11:49:56 +02:00
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Downgrading sqlalchemy to minimum supported version: ${min_sqlalchemy_version}${COLOR_RESET}"
|
|
|
|
|
echo
|
2024-02-26 13:10:31 +01:00
|
|
|
# shellcheck disable=SC2086
|
2025-04-23 19:49:08 +02:00
|
|
|
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "sqlalchemy[asyncio]==${min_sqlalchemy_version}"
|
2025-12-20 17:42:51 +01:00
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
|
|
|
|
|
echo
|
2025-12-26 23:25:21 +01:00
|
|
|
# We use pip check here to make sure that whatever `uv` installs, is also "correct" according to `pip`
|
|
|
|
|
pip check
|
2023-12-10 16:07:06 +01:00
|
|
|
}
|
2023-03-31 07:45:09 +02:00
|
|
|
|
2024-03-25 08:06:12 +01:00
|
|
|
function check_downgrade_pendulum() {
|
2024-06-04 06:20:00 +02:00
|
|
|
if [[ ${DOWNGRADE_PENDULUM=} != "true" || ${PYTHON_MAJOR_MINOR_VERSION} == "3.12" ]]; then
|
2024-01-12 14:24:12 +04:00
|
|
|
return
|
|
|
|
|
fi
|
2025-03-21 14:25:26 +01:00
|
|
|
local min_pendulum_version
|
|
|
|
|
min_pendulum_version=$(grep "pendulum>=" airflow-core/pyproject.toml | head -1 | sed "s/.*>=\([0-9\.]*\).*/\1/" | xargs)
|
2024-01-12 14:24:12 +04:00
|
|
|
echo
|
2025-03-21 14:25:26 +01:00
|
|
|
echo "${COLOR_BLUE}Downgrading pendulum to minimum supported version: ${min_pendulum_version}${COLOR_RESET}"
|
2024-01-12 14:24:12 +04:00
|
|
|
echo
|
2024-02-26 13:10:31 +01:00
|
|
|
# shellcheck disable=SC2086
|
2025-03-21 14:25:26 +01:00
|
|
|
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "pendulum==${min_pendulum_version}"
|
2025-12-20 17:42:51 +01:00
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
|
|
|
|
|
echo
|
2025-12-26 23:25:21 +01:00
|
|
|
# We use pip check here to make sure that whatever `uv` installs, is also "correct" according to `pip`
|
|
|
|
|
pip check
|
2024-01-12 14:24:12 +04:00
|
|
|
}
|
|
|
|
|
|
2023-12-10 16:07:06 +01:00
|
|
|
function check_run_tests() {
|
|
|
|
|
if [[ ${RUN_TESTS=} != "true" ]]; then
|
|
|
|
|
return
|
|
|
|
|
fi
|
2023-10-31 21:52:06 +01:00
|
|
|
|
2024-11-15 09:57:32 +01:00
|
|
|
if [[ ${TEST_GROUP:=""} == "system" ]]; then
|
2023-12-10 16:07:06 +01:00
|
|
|
exec "${IN_CONTAINER_DIR}/run_system_tests.sh" "${@}"
|
|
|
|
|
else
|
|
|
|
|
exec "${IN_CONTAINER_DIR}/run_ci_tests.sh" "${@}"
|
|
|
|
|
fi
|
|
|
|
|
}
|
2022-03-27 19:19:02 +02:00
|
|
|
|
2024-06-04 06:20:00 +02:00
|
|
|
function check_force_lowest_dependencies() {
|
|
|
|
|
if [[ ${FORCE_LOWEST_DEPENDENCIES=} != "true" ]]; then
|
|
|
|
|
return
|
|
|
|
|
fi
|
|
|
|
|
if [[ ${TEST_TYPE=} =~ Providers\[.*\] ]]; then
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
local provider_id
|
2024-06-04 06:20:00 +02:00
|
|
|
# shellcheck disable=SC2001
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
provider_id=$(echo "${TEST_TYPE}" | sed 's/Providers\[\(.*\)\]/\1/')
|
2024-06-04 06:20:00 +02:00
|
|
|
echo
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
echo "${COLOR_BLUE}Forcing dependencies to lowest versions for provider: ${provider_id}${COLOR_RESET}"
|
2024-06-04 06:20:00 +02:00
|
|
|
echo
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
if ! /opt/airflow/scripts/in_container/is_provider_excluded.py "${provider_id}"; then
|
2024-09-04 22:46:20 +02:00
|
|
|
echo
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
echo "S${COLOR_YELLOW}Skipping ${provider_id} provider check on Python ${PYTHON_MAJOR_MINOR_VERSION}!${COLOR_RESET}"
|
2024-09-04 22:46:20 +02:00
|
|
|
echo
|
|
|
|
|
exit 0
|
|
|
|
|
fi
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
cd "${AIRFLOW_SOURCES}/providers/${provider_id/.//}" || exit 1
|
2025-07-11 15:49:00 +02:00
|
|
|
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
|
|
|
|
|
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
|
|
|
|
|
# See https://bugs.launchpad.net/lxml/+bug/2110068
|
2025-09-06 16:54:43 +02:00
|
|
|
uv sync --resolution lowest-direct --no-binary-package lxml --no-binary-package xmlsec --all-extras \
|
|
|
|
|
--no-python-downloads --no-managed-python
|
2024-06-04 06:20:00 +02:00
|
|
|
else
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Forcing dependencies to lowest versions for Airflow.${COLOR_RESET}"
|
|
|
|
|
echo
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
cd "${AIRFLOW_SOURCES}/airflow-core"
|
2025-07-11 15:49:00 +02:00
|
|
|
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
|
|
|
|
|
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
|
|
|
|
|
# See https://bugs.launchpad.net/lxml/+bug/2110068
|
2025-09-06 16:54:43 +02:00
|
|
|
uv sync --resolution lowest-direct --no-binary-package lxml --no-binary-package xmlsec --all-extras \
|
|
|
|
|
--no-python-downloads --no-managed-python
|
2024-06-04 06:20:00 +02:00
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
2024-11-27 13:14:52 +00:00
|
|
|
function check_airflow_python_client_installation() {
|
|
|
|
|
if [[ ${INSTALL_AIRFLOW_PYTHON_CLIENT=} != "true" ]]; then
|
|
|
|
|
return
|
|
|
|
|
fi
|
|
|
|
|
python "${IN_CONTAINER_DIR}/install_airflow_python_client.py"
|
|
|
|
|
}
|
|
|
|
|
|
2025-04-29 14:55:35 -07:00
|
|
|
function initialize_db() {
|
|
|
|
|
# If we are going to start the api server OR we are a system test (which may or may not start the api server,
|
|
|
|
|
# depending on the Airflow version being used to run the tests), then migrate the DB.
|
|
|
|
|
if [[ ${START_API_SERVER_WITH_EXAMPLES=} == "true" || ${TEST_GROUP:=""} == "system" ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Initializing database${COLOR_RESET}"
|
|
|
|
|
echo
|
|
|
|
|
airflow db migrate
|
|
|
|
|
echo
|
|
|
|
|
echo "${COLOR_BLUE}Database initialized${COLOR_RESET}"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
2025-02-26 20:53:35 -05:00
|
|
|
function start_api_server_with_examples(){
|
2025-04-29 14:55:35 -07:00
|
|
|
USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION:=""}"
|
|
|
|
|
# Do not start the api server if either START_API_SERVER_WITH_EXAMPLES is false or the TEST_GROUP env var is not
|
|
|
|
|
# equal to "system".
|
2025-04-07 08:59:50 -07:00
|
|
|
if [[ ${START_API_SERVER_WITH_EXAMPLES=} != "true" && ${TEST_GROUP:=""} != "system" ]]; then
|
2024-11-27 13:14:52 +00:00
|
|
|
return
|
|
|
|
|
fi
|
2025-04-29 14:55:35 -07:00
|
|
|
# If the use Airflow version is set and it is <= 3.0.0 (which does not have the API server anyway) also return
|
|
|
|
|
if [[ ${USE_AIRFLOW_VERSION} != "" && ${USE_AIRFLOW_VERSION} < "3.0.0" ]]; then
|
|
|
|
|
return
|
|
|
|
|
fi
|
2024-11-27 13:14:52 +00:00
|
|
|
export AIRFLOW__CORE__LOAD_EXAMPLES=True
|
2025-05-06 11:05:35 +02:00
|
|
|
export AIRFLOW__API__EXPOSE_CONFIG=True
|
2025-01-16 14:45:40 -07:00
|
|
|
airflow dags reserialize
|
2024-11-27 13:14:52 +00:00
|
|
|
echo "Example dags parsing finished"
|
2025-03-20 09:15:19 -04:00
|
|
|
if airflow config get-value core auth_manager | grep -q "FabAuthManager"; then
|
|
|
|
|
echo "Create admin user"
|
|
|
|
|
airflow users create -u admin -p admin -f Thor -l Administrator -r Admin -e admin@email.domain || true
|
|
|
|
|
echo "Admin user created"
|
|
|
|
|
else
|
|
|
|
|
echo "Skipping user creation as auth manager different from Fab is used"
|
|
|
|
|
fi
|
2024-11-27 13:14:52 +00:00
|
|
|
echo
|
2025-02-26 20:53:35 -05:00
|
|
|
echo "${COLOR_BLUE}Starting airflow api server${COLOR_RESET}"
|
2024-11-27 13:14:52 +00:00
|
|
|
echo
|
2026-01-12 16:03:07 +01:00
|
|
|
if [[ ${START_API_SERVER_DAEMON:-"true"} == "false" ]]; then
|
|
|
|
|
airflow api-server --port 8080 &
|
|
|
|
|
else
|
|
|
|
|
airflow api-server --port 8080 --daemon
|
|
|
|
|
fi
|
2024-11-27 13:14:52 +00:00
|
|
|
echo
|
2025-02-26 20:53:35 -05:00
|
|
|
echo "${COLOR_BLUE}Waiting for api-server to start${COLOR_RESET}"
|
2024-11-27 13:14:52 +00:00
|
|
|
echo
|
2025-03-03 21:29:14 -07:00
|
|
|
check_service_connection "Airflow api-server" "run_nc localhost 8080" 100
|
2024-11-27 13:14:52 +00:00
|
|
|
EXIT_CODE=$?
|
|
|
|
|
if [[ ${EXIT_CODE} != 0 ]]; then
|
|
|
|
|
echo
|
2025-02-26 20:53:35 -05:00
|
|
|
echo "${COLOR_RED}Api server did not start properly${COLOR_RESET}"
|
2024-11-27 13:14:52 +00:00
|
|
|
echo
|
|
|
|
|
exit ${EXIT_CODE}
|
|
|
|
|
fi
|
|
|
|
|
echo
|
2025-02-26 20:53:35 -05:00
|
|
|
echo "${COLOR_BLUE}Airflow api-server started${COLOR_RESET}"
|
2024-11-27 13:14:52 +00:00
|
|
|
}
|
|
|
|
|
|
2025-01-15 22:51:32 +01:00
|
|
|
handle_mount_sources
|
2023-12-10 16:07:06 +01:00
|
|
|
determine_airflow_to_use
|
|
|
|
|
environment_initialization
|
|
|
|
|
check_boto_upgrade
|
2025-06-30 16:17:47 +03:00
|
|
|
check_upgrade_sqlalchemy
|
2024-03-25 08:06:12 +01:00
|
|
|
check_downgrade_sqlalchemy
|
|
|
|
|
check_downgrade_pendulum
|
2024-06-04 06:20:00 +02:00
|
|
|
check_force_lowest_dependencies
|
2024-11-27 13:14:52 +00:00
|
|
|
check_airflow_python_client_installation
|
2025-04-29 14:55:35 -07:00
|
|
|
initialize_db
|
2025-02-26 20:53:35 -05:00
|
|
|
start_api_server_with_examples
|
2023-12-10 16:07:06 +01:00
|
|
|
check_run_tests "${@}"
|
|
|
|
|
|
|
|
|
|
exec /bin/bash "${@}"
|
2022-03-27 19:19:02 +02:00
|
|
|
EOF
|
|
|
|
|
|
|
|
|
|
# The content below is automatically copied from scripts/docker/entrypoint_exec.sh
|
2022-03-31 13:39:46 +02:00
|
|
|
COPY <<"EOF" /entrypoint_exec.sh
|
2022-03-27 19:19:02 +02:00
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
. /opt/airflow/scripts/in_container/_in_container_script_init.sh
|
|
|
|
|
|
2022-03-30 18:24:22 +02:00
|
|
|
. /opt/airflow/scripts/in_container/configure_environment.sh
|
2022-03-27 19:19:02 +02:00
|
|
|
|
2022-03-30 18:24:22 +02:00
|
|
|
. /opt/airflow/scripts/in_container/run_init_script.sh
|
2022-03-27 19:19:02 +02:00
|
|
|
|
|
|
|
|
exec /bin/bash "${@}"
|
|
|
|
|
EOF
|
|
|
|
|
|
2025-07-11 21:18:10 +05:30
|
|
|
FROM ${BASE_IMAGE} as main
|
2019-09-18 13:43:30 +02:00
|
|
|
|
2022-01-10 06:46:09 +01:00
|
|
|
# Nolog bash flag is currently ignored - but you can replace it with other flags (for example
|
|
|
|
|
# xtrace - to show commands executed)
|
|
|
|
|
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
|
2019-04-13 15:03:02 +02:00
|
|
|
|
2025-07-11 21:18:10 +05:30
|
|
|
ARG BASE_IMAGE
|
2021-08-04 23:32:12 +02:00
|
|
|
ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
|
|
|
|
|
|
2024-10-13 04:46:16 +02:00
|
|
|
# By increasing this number we can do force build of all dependencies.
|
|
|
|
|
# NOTE! When you want to make sure dependencies are installed from scratch in your PR after removing
|
|
|
|
|
# some dependencies, you also need to set "disable image cache" in your PR to make sure the image is
|
|
|
|
|
# not built using the "main" version of those dependencies.
|
2025-06-30 23:11:01 +02:00
|
|
|
ARG DEPENDENCIES_EPOCH_NUMBER="15"
|
2019-04-13 15:03:02 +02:00
|
|
|
|
2019-09-21 00:56:02 -04:00
|
|
|
# Make sure noninteractive debian install is used and language variables set
|
2025-07-11 21:18:10 +05:30
|
|
|
ENV BASE_IMAGE=${BASE_IMAGE} \
|
2021-04-23 16:08:39 +02:00
|
|
|
DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \
|
2025-09-01 05:14:03 +02:00
|
|
|
LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \
|
2022-01-10 06:46:09 +01:00
|
|
|
DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER} \
|
|
|
|
|
INSTALL_MYSQL_CLIENT="true" \
|
2022-02-17 19:49:06 +01:00
|
|
|
INSTALL_MSSQL_CLIENT="true" \
|
2024-12-29 22:58:27 +01:00
|
|
|
INSTALL_POSTGRES_CLIENT="true" \
|
|
|
|
|
PIP_CACHE_DIR=/root/.cache/pip \
|
|
|
|
|
UV_CACHE_DIR=/root/.cache/uv
|
|
|
|
|
|
2019-04-13 15:03:02 +02:00
|
|
|
|
2025-07-11 21:18:10 +05:30
|
|
|
RUN echo "Base image version: ${BASE_IMAGE}"
|
2019-04-13 15:03:02 +02:00
|
|
|
|
2022-08-21 14:58:21 +02:00
|
|
|
ARG DEV_APT_COMMAND=""
|
2020-09-29 15:30:00 +02:00
|
|
|
ARG ADDITIONAL_DEV_APT_COMMAND=""
|
|
|
|
|
ARG ADDITIONAL_DEV_ENV_VARS=""
|
2025-09-01 23:16:20 +02:00
|
|
|
ARG ADDITIONAL_DEV_APT_DEPS="bash-completion dumb-init git gdb graphviz krb5-user \
|
2025-02-26 20:53:35 -05:00
|
|
|
less libenchant-2-2 libgcc-11-dev libgeos-dev libpq-dev net-tools netcat-openbsd \
|
|
|
|
|
openssh-server postgresql-client software-properties-common rsync tmux unzip vim xxd"
|
2022-08-21 14:58:21 +02:00
|
|
|
|
|
|
|
|
ARG ADDITIONAL_DEV_APT_ENV=""
|
2020-09-29 15:30:00 +02:00
|
|
|
|
2021-04-23 16:08:39 +02:00
|
|
|
ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \
|
|
|
|
|
ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} \
|
|
|
|
|
ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND}
|
|
|
|
|
|
2026-02-24 06:43:41 +01:00
|
|
|
|
2026-03-03 16:51:33 +01:00
|
|
|
ARG AIRFLOW_PYTHON_VERSION="3.12.13"
|
2025-08-30 20:18:30 +05:30
|
|
|
ENV AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION}
|
2026-03-06 17:44:17 +01:00
|
|
|
ENV GOLANG_MAJOR_MINOR_VERSION="1.26.1"
|
2025-07-11 21:18:10 +05:30
|
|
|
|
2025-11-29 03:23:17 +05:30
|
|
|
ARG PYTHON_LTO
|
|
|
|
|
|
2025-08-30 20:18:30 +05:30
|
|
|
COPY --from=scripts install_os_dependencies.sh /scripts/docker/
|
2025-07-11 21:18:10 +05:30
|
|
|
|
2025-11-29 03:23:17 +05:30
|
|
|
RUN PYTHON_LTO=${PYTHON_LTO} bash /scripts/docker/install_os_dependencies.sh ci
|
2022-08-21 14:58:21 +02:00
|
|
|
|
2025-09-01 23:16:20 +02:00
|
|
|
# In case system python is installed, setting LD_LIBRARY_PATH prevents any case the system python
|
|
|
|
|
# libraries will be accidentally used before the library installed from sources (which is newer and
|
|
|
|
|
# python interpreter might break if accidentally the old system libraries are used.
|
|
|
|
|
ENV LD_LIBRARY_PATH="/usr/python/lib"
|
|
|
|
|
|
2023-07-21 19:27:51 +02:00
|
|
|
COPY --from=scripts common.sh /scripts/docker/
|
|
|
|
|
|
2022-01-10 06:46:09 +01:00
|
|
|
# Only copy mysql/mssql installation scripts for now - so that changing the other
|
2022-01-20 17:44:27 +01:00
|
|
|
# scripts which are needed much later will not invalidate the docker layer here.
|
2022-03-29 15:03:50 +02:00
|
|
|
COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scripts/docker/
|
2022-03-27 19:19:02 +02:00
|
|
|
|
2022-08-21 14:58:21 +02:00
|
|
|
ARG HOME=/root
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
ARG AIRFLOW_IMAGE_TYPE="ci"
|
2022-08-21 14:58:21 +02:00
|
|
|
ARG AIRFLOW_HOME=/root/airflow
|
|
|
|
|
ARG AIRFLOW_SOURCES=/opt/airflow
|
2026-01-17 13:54:45 +02:00
|
|
|
ARG INSTALL_MYSQL_CLIENT_TYPE="mariadb"
|
2022-08-21 14:58:21 +02:00
|
|
|
|
|
|
|
|
ENV HOME=${HOME} \
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
AIRFLOW_IMAGE_TYPE=${AIRFLOW_IMAGE_TYPE} \
|
2022-08-21 14:58:21 +02:00
|
|
|
AIRFLOW_HOME=${AIRFLOW_HOME} \
|
2023-10-24 12:54:09 +04:00
|
|
|
AIRFLOW_SOURCES=${AIRFLOW_SOURCES} \
|
|
|
|
|
INSTALL_MYSQL_CLIENT_TYPE=${INSTALL_MYSQL_CLIENT_TYPE}
|
2022-08-21 14:58:21 +02:00
|
|
|
|
2022-01-29 19:08:10 +01:00
|
|
|
# We run scripts with bash here to make sure we can execute the scripts. Changing to +x might have an
|
|
|
|
|
# unexpected result - the cache for Dockerfiles might get invalidated in case the host system
|
|
|
|
|
# had different umask set and group x bit was not set. In Azure the bit might be not set at all.
|
2023-07-28 06:47:40 +02:00
|
|
|
# That also protects against AUFS Docker backend problem where changing the executable bit required sync
|
2022-02-17 19:49:06 +01:00
|
|
|
RUN bash /scripts/docker/install_mysql.sh prod \
|
|
|
|
|
&& bash /scripts/docker/install_mysql.sh dev \
|
2023-07-21 19:27:51 +02:00
|
|
|
&& bash /scripts/docker/install_mssql.sh dev \
|
2023-08-16 08:30:29 +02:00
|
|
|
&& bash /scripts/docker/install_postgres.sh dev \
|
|
|
|
|
# The user is added to allow ssh debugging (you can connect with airflow/airflow over ssh)
|
|
|
|
|
&& adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \
|
2025-02-26 20:53:35 -05:00
|
|
|
--quiet "airflow" --home "/home/airflow" \
|
2023-08-16 08:30:29 +02:00
|
|
|
&& echo -e "airflow\nairflow" | passwd airflow 2>&1 \
|
|
|
|
|
&& echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \
|
|
|
|
|
&& chmod 0440 /etc/sudoers.d/airflow
|
2019-04-13 15:03:02 +02:00
|
|
|
|
2021-08-04 23:32:12 +02:00
|
|
|
# Install Helm
|
2026-01-18 17:54:55 +02:00
|
|
|
ARG HELM_VERSION="v3.19.0"
|
2021-08-04 23:32:12 +02:00
|
|
|
|
|
|
|
|
RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \
|
2022-03-10 07:47:42 +01:00
|
|
|
&& PLATFORM=$([ "$(uname -m)" = "aarch64" ] && echo "arm64" || echo "amd64" ) \
|
|
|
|
|
&& HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-${PLATFORM}.tar.gz" \
|
|
|
|
|
&& curl --silent --location "${HELM_URL}" | tar -xz -O "${SYSTEM}-${PLATFORM}/helm" > /usr/local/bin/helm \
|
2021-08-04 23:32:12 +02:00
|
|
|
&& chmod +x /usr/local/bin/helm
|
|
|
|
|
|
2025-11-26 12:07:48 +01:00
|
|
|
# Install mprocs - a modern process manager for managing multiple Airflow components
|
2026-03-26 17:50:21 +08:00
|
|
|
ARG MPROCS_VERSION="0.9.2"
|
2025-11-26 12:07:48 +01:00
|
|
|
|
|
|
|
|
RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \
|
|
|
|
|
&& PLATFORM="$(uname -m)" \
|
|
|
|
|
&& MPROCS_URL="https://github.com/pvolok/mprocs/releases/download/v${MPROCS_VERSION}/mprocs-${MPROCS_VERSION}-${SYSTEM}-${PLATFORM}-musl.tar.gz" \
|
|
|
|
|
&& echo "Downloading mprocs from ${MPROCS_URL}" \
|
|
|
|
|
&& curl --silent --location "${MPROCS_URL}" | tar -xz -C /usr/local/bin/ mprocs \
|
|
|
|
|
&& chmod +x /usr/local/bin/mprocs
|
|
|
|
|
|
2019-08-20 05:41:51 -04:00
|
|
|
WORKDIR ${AIRFLOW_SOURCES}
|
|
|
|
|
|
2020-06-04 19:12:09 +02:00
|
|
|
RUN mkdir -pv ${AIRFLOW_HOME} && \
|
|
|
|
|
mkdir -pv ${AIRFLOW_HOME}/dags && \
|
2019-11-12 21:47:20 +01:00
|
|
|
mkdir -pv ${AIRFLOW_HOME}/logs
|
2019-01-13 17:09:20 +01:00
|
|
|
|
2019-07-17 22:42:43 +02:00
|
|
|
ARG AIRFLOW_REPO=apache/airflow
|
2021-06-01 12:16:18 +02:00
|
|
|
ARG AIRFLOW_BRANCH=main
|
2019-07-17 22:42:43 +02:00
|
|
|
# Airflow Extras installed
|
2025-03-05 23:04:00 +01:00
|
|
|
ARG AIRFLOW_EXTRAS="all"
|
2020-05-27 17:09:11 +02:00
|
|
|
ARG ADDITIONAL_AIRFLOW_EXTRAS=""
|
2021-02-21 18:56:55 +01:00
|
|
|
# Allows to override constraints source
|
|
|
|
|
ARG CONSTRAINTS_GITHUB_REPOSITORY="apache/airflow"
|
2022-05-09 23:02:25 +02:00
|
|
|
ARG AIRFLOW_CONSTRAINTS_MODE="constraints-source-providers"
|
2021-03-23 04:13:17 +01:00
|
|
|
ARG AIRFLOW_CONSTRAINTS_REFERENCE=""
|
|
|
|
|
ARG AIRFLOW_CONSTRAINTS_LOCATION=""
|
2021-06-01 12:16:18 +02:00
|
|
|
ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main"
|
2026-02-24 06:43:41 +01:00
|
|
|
# By default fallback to installation without constraints because in CI image it should always be tried
|
|
|
|
|
ARG AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION="true"
|
|
|
|
|
|
2021-08-18 18:39:04 +02:00
|
|
|
# By changing the epoch we can force reinstalling Airflow and pip all dependencies
|
2019-08-01 13:56:58 +02:00
|
|
|
# It can also be overwritten manually by setting the AIRFLOW_CI_BUILD_EPOCH environment variable.
|
2024-02-12 00:15:53 +01:00
|
|
|
ARG AIRFLOW_CI_BUILD_EPOCH="10"
|
2021-04-23 16:08:39 +02:00
|
|
|
# Setup PIP
|
|
|
|
|
# By default PIP has progress bar but you can disable it.
|
|
|
|
|
ARG PIP_PROGRESS_BAR="on"
|
|
|
|
|
# Optimizing installation of Cassandra driver (in case there are no prebuilt wheels which is the
|
|
|
|
|
# case as of 20.04.2021 with Python 3.9
|
|
|
|
|
# Speeds up building the image - cassandra driver without CYTHON saves around 10 minutes
|
|
|
|
|
ARG CASS_DRIVER_NO_CYTHON="1"
|
|
|
|
|
# Build cassandra driver on multiple CPUs
|
|
|
|
|
ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
|
2021-01-08 20:11:35 +01:00
|
|
|
|
2023-08-14 10:41:48 +02:00
|
|
|
# This value should be set by the CI image build system to the current timestamp
|
|
|
|
|
ARG AIRFLOW_VERSION=""
|
2022-01-30 21:07:32 +01:00
|
|
|
|
2022-07-27 17:07:53 +02:00
|
|
|
# Additional PIP flags passed to all pip install commands except reinstalling pip itself
|
|
|
|
|
ARG ADDITIONAL_PIP_INSTALL_FLAGS=""
|
|
|
|
|
|
2024-02-29 21:33:41 +01:00
|
|
|
ARG AIRFLOW_USE_UV="true"
|
|
|
|
|
|
2021-04-23 16:08:39 +02:00
|
|
|
ENV AIRFLOW_REPO=${AIRFLOW_REPO}\
|
|
|
|
|
AIRFLOW_BRANCH=${AIRFLOW_BRANCH} \
|
|
|
|
|
AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} \
|
|
|
|
|
CONSTRAINTS_GITHUB_REPOSITORY=${CONSTRAINTS_GITHUB_REPOSITORY} \
|
2022-05-09 23:02:25 +02:00
|
|
|
AIRFLOW_CONSTRAINTS_MODE=${AIRFLOW_CONSTRAINTS_MODE} \
|
2021-04-23 16:08:39 +02:00
|
|
|
AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE} \
|
|
|
|
|
AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \
|
2026-02-24 06:43:41 +01:00
|
|
|
AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION=${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} \
|
2021-04-23 16:08:39 +02:00
|
|
|
DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \
|
|
|
|
|
AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH} \
|
2022-01-30 21:07:32 +01:00
|
|
|
AIRFLOW_VERSION=${AIRFLOW_VERSION} \
|
2021-04-23 16:08:39 +02:00
|
|
|
AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
|
2024-02-26 13:10:31 +01:00
|
|
|
AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \
|
|
|
|
|
AIRFLOW_USE_UV=${AIRFLOW_USE_UV} \
|
2025-03-05 23:04:00 +01:00
|
|
|
UV_SYSTEM_PYTHON="true" \
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
UV_PROJECT_ENVIRONMENT="/usr/local" \
|
2021-04-23 16:08:39 +02:00
|
|
|
INSTALL_MYSQL_CLIENT="true" \
|
2023-10-24 12:54:09 +04:00
|
|
|
INSTALL_MYSQL_CLIENT_TYPE=${INSTALL_MYSQL_CLIENT_TYPE} \
|
2021-09-21 11:27:47 +02:00
|
|
|
INSTALL_MSSQL_CLIENT="true" \
|
2022-02-17 19:49:06 +01:00
|
|
|
INSTALL_POSTGRES_CLIENT="true" \
|
2021-04-23 16:08:39 +02:00
|
|
|
AIRFLOW_INSTALLATION_METHOD="." \
|
|
|
|
|
AIRFLOW_VERSION_SPECIFICATION="" \
|
|
|
|
|
PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \
|
2022-07-27 17:07:53 +02:00
|
|
|
ADDITIONAL_PIP_INSTALL_FLAGS=${ADDITIONAL_PIP_INSTALL_FLAGS} \
|
Add Python 3.14 Support (#63520)
* Update python version exclusion to 3.15
* Add 3.14 metadata version classifiers and related constants
* Regenerate Breeze command help screenshots
* Assorted workarounds to fix breeze image building
- constraints are skipped entirely
- greenlet pin updated
* Exclude cassandra
* Exclude amazon
* Exclude google
* CI: Only add pydantic extra to Airflow 2 migration tests
Before this fix there were two separate issues in the migration-test setup for Python 3.14:
1. The migration workflow always passes --airflow-extras pydantic.
2. For Python 3.14, the minimum Airflow version is resolved to 3.2.0 by get_min_airflow_version_for_python.py, and apache-airflow[pydantic]==3.2.0 is not a valid thing to install.
So when constraints installation fails, the fallback path tries to install an invalid spec.
* Disable DB migration tests for python 3.14
* Enforce werkzeug 3.x for python 3.14
* Increase K8s executor test timeout for Python 3.14
Python 3.14 changed the default multiprocessing start method from 'fork' to 'forkserver' on Linux. The forkserver start method is slower because each new process must import modules from scratch rather than copying the parent's address space. This makes `multiprocessing.Manager()` initialization take longer, causing the test to exceed its 10s timeout.
* Adapt LocalExecutor tests for Python 3.14 forkserver default
Python 3.14 changed the default multiprocessing start method from
'fork' to 'forkserver' on Linux. Like 'spawn', 'forkserver' doesn't
share the parent's address space, so mock patches applied in the test
process are invisible to worker subprocesses.
- Skip tests that mock across process boundaries on non-fork methods
- Add test_executor_lazy_worker_spawning to verify that non-fork start
methods defer worker creation and skip gc.freeze
- Make test_multiple_team_executors_isolation and
test_global_executor_without_team_name assert the correct worker
count for each start method instead of assuming pre-spawning
- Remove skip from test_clean_stop_on_signal (works on all methods)
and increase timeout from 5s to 30s for forkserver overhead
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* Bump dependencies to versions supporting 3.14
* Fix PROD image build failing on Python 3.14 due to excluded providers
The PROD image build installed all provider wheels regardless of Python
version compatibility. Providers like google and amazon that exclude
Python 3.14 were still passed to pip, causing resolution failures (e.g.
ray has no cp314 wheel on PyPI).
Two fixes:
- get_distribution_specs.py now reads each wheel's Requires-Python
metadata and skips incompatible wheels instead of passing them to pip.
- The requires-python specifier generation used !=3.14 which per PEP 440
only excludes 3.14.0, not 3.14.3. Changed to !=3.14.* wildcard.
* Split core test types into 2 matrix groups to avoid OOM on Python 3.14
Non-DB core tests use xdist which runs all test types in a single pytest
process. With 2059 items across 4 workers, memory accumulates until the
OOM killer strikes at ~86% completion (exit code 137).
Split core test types into 2 groups (API/Always/CLI and
Core/Other/Serialization), similar to how provider tests already use
_split_list with NUMBER_OF_LOW_DEP_SLICES. Each group gets ~1000 items,
well under the ~1770 threshold where OOM occurs.
Update selective_checks test expectations to reflect the 2-group split.
* Gracefully handle an already removed password file in fixture
The old code had a check-then-act race (if `os.path.exists` → `os.remove`), which fails when the file doesn't exist at removal time. `contextlib.suppress(FileNotFoundError)` handles this atomically — if the file is missing (never created in this xdist worker, or removed between check and delete), it's silently ignored.
* Fix OOM and flaky tests in test_process_utils
Replace multiprocessing.Process with subprocess.Popen running minimal
inline scripts. multiprocessing.Process uses fork(), which duplicates
the entire xdist worker memory. At 95% test completion the worker has
accumulated hundreds of MBs; forking it triggers the OOM killer
(exit code 137) on Python 3.14.
subprocess.Popen starts a fresh lightweight process (~10MB) without
copying the parent's memory, avoiding the OOM entirely.
Also replace the racy ps -ax process counting in
TestKillChildProcessesByPids with psutil.pid_exists() checks on the
specific PID — the old approach was non-deterministic because unrelated
processes could start/stop between measurements.
* Add prek hook to validate python_version markers for excluded providers
When a provider declares excluded-python-versions in provider.yaml,
every dependency string referencing that provider in pyproject.toml
must carry a matching python_version marker. Missing markers cause
excluded providers to be silently installed as transitive dependencies
(e.g. aiobotocore pulling in amazon on Python 3.14).
The new check-excluded-provider-markers hook reads exclusions from
provider.yaml and validates all dependency strings in pyproject.toml
at commit time, preventing regressions like the one fixed in the
previous commit.
* Update `uv.lock`
---------
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-19 23:03:46 +02:00
|
|
|
INCLUDE_PRE_RELEASE="true" \
|
2021-04-23 16:08:39 +02:00
|
|
|
CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY} \
|
|
|
|
|
CASS_DRIVER_NO_CYTHON=${CASS_DRIVER_NO_CYTHON}
|
2021-02-10 00:20:50 +01:00
|
|
|
|
2022-01-30 21:07:32 +01:00
|
|
|
RUN echo "Airflow version: ${AIRFLOW_VERSION}"
|
|
|
|
|
|
2022-01-11 10:38:34 +01:00
|
|
|
# Copy all scripts required for installation - changing any of those should lead to
|
|
|
|
|
# rebuilding from here
|
2024-12-29 22:58:27 +01:00
|
|
|
COPY --from=scripts common.sh install_packaging_tools.sh install_additional_dependencies.sh /scripts/docker/
|
2022-01-10 06:46:09 +01:00
|
|
|
|
2022-01-11 10:38:34 +01:00
|
|
|
# We are first creating a venv where all python packages and .so binaries needed by those are
|
|
|
|
|
# installed.
|
2024-10-17 23:43:48 +02:00
|
|
|
|
|
|
|
|
# Here we fix the versions so all subsequent commands will use the versions
|
|
|
|
|
# from the sources
|
2024-10-27 20:46:10 +01:00
|
|
|
# You can swap comments between those two args to test pip from the main version
|
|
|
|
|
# When you attempt to test if the version of `pip` from specified branch works for our builds
|
|
|
|
|
# Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
|
2026-02-05 17:16:07 +05:30
|
|
|
ARG AIRFLOW_PIP_VERSION=26.0.1
|
2024-10-27 20:46:10 +01:00
|
|
|
# ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main"
|
2026-03-27 19:03:24 +01:00
|
|
|
ARG AIRFLOW_UV_VERSION=0.10.12
|
|
|
|
|
ARG AIRFLOW_PREK_VERSION="0.3.6"
|
2024-10-17 23:43:48 +02:00
|
|
|
|
2025-08-30 20:18:30 +05:30
|
|
|
# UV_LINK_MODE=copy is needed since we are using cache mounted from the host
|
2024-10-17 23:43:48 +02:00
|
|
|
ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
|
2024-11-15 18:46:57 +01:00
|
|
|
AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \
|
2024-12-29 22:58:27 +01:00
|
|
|
UV_LINK_MODE=copy \
|
2025-08-17 09:00:14 +02:00
|
|
|
AIRFLOW_PREK_VERSION=${AIRFLOW_PREK_VERSION}
|
2024-10-17 23:43:48 +02:00
|
|
|
|
2025-08-30 20:18:30 +05:30
|
|
|
# The PATH is needed for python to find installed and cargo to build the wheels
|
|
|
|
|
ENV PATH="/usr/python/bin:/root/.local/bin:/root/.cargo/bin:${PATH}"
|
2024-12-29 22:58:27 +01:00
|
|
|
# Useful for creating a cache id based on the underlying architecture, preventing the use of cached python packages from
|
|
|
|
|
# an incorrect architecture.
|
|
|
|
|
ARG TARGETARCH
|
|
|
|
|
# Value to be able to easily change cache id and therefore use a bare new cache
|
2025-09-22 23:15:02 +02:00
|
|
|
ARG DEPENDENCY_CACHE_EPOCH="2"
|
2024-12-29 22:58:27 +01:00
|
|
|
|
2022-02-12 21:46:47 +01:00
|
|
|
# Install useful command line tools in their own virtualenv so that they do not clash with
|
2024-10-17 23:43:48 +02:00
|
|
|
# dependencies installed in Airflow also reinstall PIP and UV to make sure they are installed
|
|
|
|
|
# in the version specified above
|
2025-08-30 20:18:30 +05:30
|
|
|
|
2024-12-30 12:48:19 +01:00
|
|
|
RUN bash /scripts/docker/install_packaging_tools.sh
|
2024-12-29 22:58:27 +01:00
|
|
|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
COPY --from=scripts install_airflow_when_building_images.sh /scripts/docker/
|
2022-01-11 10:38:34 +01:00
|
|
|
|
2024-12-29 22:58:27 +01:00
|
|
|
# We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not
|
|
|
|
|
# copying over stuff that is accidentally generated or that we do not need (such as egg-info)
|
|
|
|
|
# if you want to add something that is missing and you expect to see it in the image you can
|
|
|
|
|
# add it with ! in .dockerignore next to the airflow, test etc. directories there
|
|
|
|
|
COPY . ${AIRFLOW_SOURCES}/
|
|
|
|
|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
ARG UPGRADE_RANDOM_INDICATOR_STRING=""
|
2025-04-27 17:13:14 +02:00
|
|
|
ARG VERSION_SUFFIX=""
|
2024-02-12 21:30:54 +01:00
|
|
|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
ENV UPGRADE_RANDOM_INDICATOR_STRING=${UPGRADE_RANDOM_INDICATOR_STRING} \
|
2025-04-27 17:13:14 +02:00
|
|
|
VERSION_SUFFIX=${VERSION_SUFFIX}
|
2024-02-12 21:30:54 +01:00
|
|
|
|
Standardize airflow build process and switch to Hatchling build backend (#36537)
This PR changes Airflow installation and build backend to use new
standard Python ways of building Python applications.
We've been trying to do it for quite a while. Airflow tranditionally
has been using complex and convoluted build process based on
setuptools and (extremely) custom setup.py file. It survived
migration to Airflow 2.0 and splitting Airlfow monorepo into
Airflow and Providers, adding pre-installed providers and switching
providers to use flit (and follow build standards).
So far tooling in Python ecosystme had not been able to fuflill our
needs and we refrained to develop our own tooling, but finally with
appearance of Hatch (managed by Python Packaging Authority) and
few recent advancements there we are finally able to swtich to
Python standard ways of managing project dependnecy configuration
and project build setup (with a few customizations).
This PR makes airflow build process follow those standard PEPs:
* Airflow has all build configuration stored in pyproject.toml
following PEP 518 which allows any fronted (`pip`, `poetry`,
`hatch`, `flit`, or whatever other frontend is used to
install required build dependendencies to install Airflow
locally and to build distribution pacakges (sdist/wheel)
* Hatchling backend follows PEP 517 for standard source tree and build
backend implementation that allows to execute the build in a
frontend-independent way
* We store all project metadata in pyprooject.toml - following
PEP 621 where all necessary project metadata components were
defined.
* We plug-in into Hatchling "editable build" hooks following
PEP 660. Hatchling internally builds editable wheel that
is used as ephemeral step and communication between backend
and frontend (and this ephemeral wheel is used to make
editable installation of the projeect - suitable for fast
iteration of code without reinstalling the package.
With Airflow having many provider packages in single source tree
where we want to be able to install and develop airflow and
providers together, this is not a small feat to implement the
case wher editable installation has to behave quite a bit
differently when it comes to packaging and dependencies for
editable install (when you want to edit sources directly) and
installable package (where you want to have separate Airflow
package and provider packages). Fortunately the standardisation
efforts in the Python Packaging community and tooling implementing
it had finally made it possible.
Some of the important ways bow this has been achieved:
* We continue using provider.yaml in providers as the single source
of trutgh for per-provider dependencies. We added a possibility
to specify "devel-dependencies" in provider.yaml so that all
per-provider dependencies in `generated/provider_dependencies.json`
and `pyproject.toml` are generated from those dependencies via
update-providers-dependencies pre-commit.
* Pyproject.toml is generally managed manually, but the part where
provider dependencies and bundle dependencies are used is
automatically updated by a pre-commit whenever provider
dependencies change. Those generated provider dependencies contain
just dependencies of providers - not the provider packages, but
in the final "standard" wheel file they are replaced with
"apache-airflow-providers-PROVIDER" dependencies - so that the
wheel package will only install the provider and use the
dependencies of that version of provider it installs.
* We are utilising custom hatchiling build hooks (PEP 660 standard)
that allow to modify 'standard' wheel package on-the-fly when
the wheel is being prepared by adding preinstalled package
dependencies (which are not needed in editable build) and by
removing all devel extras (that are not needed in the PyPI
distributed wheel package). This allows to solve the conundrum
of having different "editable" and "standard" behaviour while
keeping the same project specification in pyproject.toml.
* We added description of how `Hatch` can be employed as build
frontend in order to manage local virtualenv and install Airflow
in editable way easily - while keeping all properties of the
installed application (including working airflow cli and
package metadata discovery) as well as how to use PEP-standard
ways of bulding wheel and sdist packages.
* We have a custom step (following PEP-standards) to inject
airflow-specific build steps - compiling www assets and
generating git commit hash version to display it in the UI
* We also show how all this makes it possible to make it easy to
manage local virtualenvs and editable installations for Airflow
contributors - without vendor lock-in of the build tools as
by following standard PEPs Airflow can be locally and editably
installed by anyone using any build front-end tools following
the standards - whether you use `pip`, `poetry`, `Hatch`, `flit`
or any other frontent build tools, Airflow local installation
and package building will work the same way for all of them,
where both "editable" and "standard" package prepration is
managed by `hatchling` backend in the same way.
* Previously our extras contained a "." which is not normalized
name for extras - `pip` and other tools replaced it automatically
with `_'. This change updates the extra names to contain
'-' rather than '.' in the name, following PEP-685. This should be
fully backwards compatible, users will still be able to use "." but it
will be normalized to "-" in Airflow packages. This is also future
proof as it is expected that all package managers and tools
will eventually use PEP-685 applied to extras, even if currently
some of the tools (pip + setuptools) might generate warnings.
* Additionally, this change organizes the documentation around
the extras and dependencies, explaining the reasoning behind
all the different extras we have.
* As a bonus (and this is what we used to test it all) we are
documenting how to use Hatch frontend to:
* manage multiple Python installations
* manage multiple Pythob virtualenv environments
* build Airflow packages for release management
2024-01-10 21:19:02 +01:00
|
|
|
# The goal of this line is to install the dependencies from the most current pyproject.toml from sources
|
2019-07-17 22:42:43 +02:00
|
|
|
# This will be usually incremental small set of packages in CI optimized build, so it will be very fast
|
|
|
|
|
# In non-CI optimized build this will install all dependencies before installing sources.
|
Standardize airflow build process and switch to Hatchling build backend (#36537)
This PR changes Airflow installation and build backend to use new
standard Python ways of building Python applications.
We've been trying to do it for quite a while. Airflow tranditionally
has been using complex and convoluted build process based on
setuptools and (extremely) custom setup.py file. It survived
migration to Airflow 2.0 and splitting Airlfow monorepo into
Airflow and Providers, adding pre-installed providers and switching
providers to use flit (and follow build standards).
So far tooling in Python ecosystme had not been able to fuflill our
needs and we refrained to develop our own tooling, but finally with
appearance of Hatch (managed by Python Packaging Authority) and
few recent advancements there we are finally able to swtich to
Python standard ways of managing project dependnecy configuration
and project build setup (with a few customizations).
This PR makes airflow build process follow those standard PEPs:
* Airflow has all build configuration stored in pyproject.toml
following PEP 518 which allows any fronted (`pip`, `poetry`,
`hatch`, `flit`, or whatever other frontend is used to
install required build dependendencies to install Airflow
locally and to build distribution pacakges (sdist/wheel)
* Hatchling backend follows PEP 517 for standard source tree and build
backend implementation that allows to execute the build in a
frontend-independent way
* We store all project metadata in pyprooject.toml - following
PEP 621 where all necessary project metadata components were
defined.
* We plug-in into Hatchling "editable build" hooks following
PEP 660. Hatchling internally builds editable wheel that
is used as ephemeral step and communication between backend
and frontend (and this ephemeral wheel is used to make
editable installation of the projeect - suitable for fast
iteration of code without reinstalling the package.
With Airflow having many provider packages in single source tree
where we want to be able to install and develop airflow and
providers together, this is not a small feat to implement the
case wher editable installation has to behave quite a bit
differently when it comes to packaging and dependencies for
editable install (when you want to edit sources directly) and
installable package (where you want to have separate Airflow
package and provider packages). Fortunately the standardisation
efforts in the Python Packaging community and tooling implementing
it had finally made it possible.
Some of the important ways bow this has been achieved:
* We continue using provider.yaml in providers as the single source
of trutgh for per-provider dependencies. We added a possibility
to specify "devel-dependencies" in provider.yaml so that all
per-provider dependencies in `generated/provider_dependencies.json`
and `pyproject.toml` are generated from those dependencies via
update-providers-dependencies pre-commit.
* Pyproject.toml is generally managed manually, but the part where
provider dependencies and bundle dependencies are used is
automatically updated by a pre-commit whenever provider
dependencies change. Those generated provider dependencies contain
just dependencies of providers - not the provider packages, but
in the final "standard" wheel file they are replaced with
"apache-airflow-providers-PROVIDER" dependencies - so that the
wheel package will only install the provider and use the
dependencies of that version of provider it installs.
* We are utilising custom hatchiling build hooks (PEP 660 standard)
that allow to modify 'standard' wheel package on-the-fly when
the wheel is being prepared by adding preinstalled package
dependencies (which are not needed in editable build) and by
removing all devel extras (that are not needed in the PyPI
distributed wheel package). This allows to solve the conundrum
of having different "editable" and "standard" behaviour while
keeping the same project specification in pyproject.toml.
* We added description of how `Hatch` can be employed as build
frontend in order to manage local virtualenv and install Airflow
in editable way easily - while keeping all properties of the
installed application (including working airflow cli and
package metadata discovery) as well as how to use PEP-standard
ways of bulding wheel and sdist packages.
* We have a custom step (following PEP-standards) to inject
airflow-specific build steps - compiling www assets and
generating git commit hash version to display it in the UI
* We also show how all this makes it possible to make it easy to
manage local virtualenvs and editable installations for Airflow
contributors - without vendor lock-in of the build tools as
by following standard PEPs Airflow can be locally and editably
installed by anyone using any build front-end tools following
the standards - whether you use `pip`, `poetry`, `Hatch`, `flit`
or any other frontent build tools, Airflow local installation
and package building will work the same way for all of them,
where both "editable" and "standard" package prepration is
managed by `hatchling` backend in the same way.
* Previously our extras contained a "." which is not normalized
name for extras - `pip` and other tools replaced it automatically
with `_'. This change updates the extra names to contain
'-' rather than '.' in the name, following PEP-685. This should be
fully backwards compatible, users will still be able to use "." but it
will be normalized to "-" in Airflow packages. This is also future
proof as it is expected that all package managers and tools
will eventually use PEP-685 applied to extras, even if currently
some of the tools (pip + setuptools) might generate warnings.
* Additionally, this change organizes the documentation around
the extras and dependencies, explaining the reasoning behind
all the different extras we have.
* As a bonus (and this is what we used to test it all) we are
documenting how to use Hatch frontend to:
* manage multiple Python installations
* manage multiple Pythob virtualenv environments
* build Airflow packages for release management
2024-01-10 21:19:02 +01:00
|
|
|
# Usually we will install versions based on the dependencies in pyproject.toml and upgraded only if needed.
|
|
|
|
|
# But in cron job we will install latest versions matching pyproject.toml to see if there is no breaking change
|
2020-07-20 14:36:03 +02:00
|
|
|
# and push the constraints if everything is successful
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
RUN --mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.cache/ bash /scripts/docker/install_airflow_when_building_images.sh
|
2022-01-11 10:38:34 +01:00
|
|
|
|
2024-03-02 15:07:06 +01:00
|
|
|
COPY --from=scripts install_packaging_tools.sh install_additional_dependencies.sh /scripts/docker/
|
2021-05-14 14:10:28 +02:00
|
|
|
|
2021-08-04 23:32:12 +02:00
|
|
|
ARG ADDITIONAL_PYTHON_DEPS=""
|
|
|
|
|
|
2024-12-29 22:58:27 +01:00
|
|
|
ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS}
|
|
|
|
|
|
|
|
|
|
RUN --mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.cache/ \
|
|
|
|
|
bash /scripts/docker/install_packaging_tools.sh; \
|
2021-08-04 23:32:12 +02:00
|
|
|
if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
|
2025-02-26 20:53:35 -05:00
|
|
|
bash /scripts/docker/install_additional_dependencies.sh; \
|
2021-08-04 23:32:12 +02:00
|
|
|
fi
|
2019-11-12 22:40:03 +01:00
|
|
|
|
2024-12-29 22:58:27 +01:00
|
|
|
COPY --from=scripts entrypoint_ci.sh /entrypoint
|
|
|
|
|
COPY --from=scripts entrypoint_exec.sh /entrypoint-exec
|
|
|
|
|
RUN chmod a+x /entrypoint /entrypoint-exec
|
2020-01-11 16:25:19 +01:00
|
|
|
|
2023-03-31 07:45:09 +02:00
|
|
|
|
2024-12-29 22:58:27 +01:00
|
|
|
# Install autocomplete for airflow and kubectl
|
2025-12-01 06:48:24 +01:00
|
|
|
# hadolint ignore=SC2028
|
2024-12-29 22:58:27 +01:00
|
|
|
RUN if command -v airflow; then \
|
2025-02-26 20:53:35 -05:00
|
|
|
register-python-argcomplete airflow >> ~/.bashrc ; \
|
2024-12-29 22:58:27 +01:00
|
|
|
fi; \
|
2025-12-01 06:48:24 +01:00
|
|
|
echo "source /etc/bash_completion" >> ~/.bashrc ; \
|
|
|
|
|
echo 'export PS1="\[\033[1;36m\][Breeze:\$(python --version 2>&1 | cut -d\" \" -f2)]\[\033[0m\] \[\033[1;32m\]\u@\h\[\033[0m\]:\[\033[1;34m\]\w\[\033[0m\]\$ "' >> ~/.bashrc
|
2019-11-12 22:40:03 +01:00
|
|
|
|
2021-08-04 23:32:12 +02:00
|
|
|
WORKDIR ${AIRFLOW_SOURCES}
|
2019-11-12 22:40:03 +01:00
|
|
|
|
2020-08-19 02:03:22 +02:00
|
|
|
ARG BUILD_ID
|
|
|
|
|
ARG COMMIT_SHA
|
2021-01-21 16:16:09 +01:00
|
|
|
ARG AIRFLOW_IMAGE_DATE_CREATED
|
|
|
|
|
|
2022-01-11 10:38:34 +01:00
|
|
|
ENV PATH="/files/bin/:/opt/airflow/scripts/in_container/bin/:${PATH}" \
|
2021-04-23 16:08:39 +02:00
|
|
|
GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm/" \
|
|
|
|
|
BUILD_ID=${BUILD_ID} \
|
2024-12-29 22:58:27 +01:00
|
|
|
COMMIT_SHA=${COMMIT_SHA} \
|
|
|
|
|
# When we enter the image, the /root/.cache is not mounted from temporary mount cache.
|
|
|
|
|
# We do not want to share the cache from host to avoid all kinds of problems where cache
|
|
|
|
|
# is different with different platforms / python versions. We want to have a clean cache
|
|
|
|
|
# in the image - and in this case /root/.cache is on the same filesystem as the installed packages.
|
|
|
|
|
# so we can go back to the default link mode being hardlink.
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
UV_LINK_MODE=hardlink \
|
|
|
|
|
MYPY_FORCE_COLOR="true"
|
2021-04-23 16:08:39 +02:00
|
|
|
|
2022-01-11 10:38:34 +01:00
|
|
|
# Link dumb-init for backwards compatibility (so that older images also work)
|
|
|
|
|
RUN ln -sf /usr/bin/dumb-init /usr/local/bin/dumb-init
|
|
|
|
|
|
|
|
|
|
EXPOSE 8080
|
|
|
|
|
|
2020-12-09 05:19:38 +00:00
|
|
|
LABEL org.apache.airflow.distro="debian" \
|
2025-02-26 20:53:35 -05:00
|
|
|
org.apache.airflow.module="airflow" \
|
|
|
|
|
org.apache.airflow.component="airflow" \
|
|
|
|
|
org.apache.airflow.image="airflow-ci" \
|
|
|
|
|
org.apache.airflow.version="${AIRFLOW_VERSION}" \
|
2025-08-30 20:18:30 +05:30
|
|
|
org.apache.airflow.python.version="${AIRFLOW_PYTHON_VERSION}" \
|
2025-02-26 20:53:35 -05:00
|
|
|
org.apache.airflow.uid="0" \
|
|
|
|
|
org.apache.airflow.gid="0" \
|
|
|
|
|
org.apache.airflow.build-id="${BUILD_ID}" \
|
|
|
|
|
org.apache.airflow.commit-sha="${COMMIT_SHA}" \
|
|
|
|
|
org.opencontainers.image.source="${AIRFLOW_IMAGE_REPOSITORY}" \
|
|
|
|
|
org.opencontainers.image.created="${AIRFLOW_IMAGE_DATE_CREATED}" \
|
|
|
|
|
org.opencontainers.image.authors="dev@airflow.apache.org" \
|
|
|
|
|
org.opencontainers.image.url="https://airflow.apache.org" \
|
|
|
|
|
org.opencontainers.image.documentation="https://airflow.apache.org/docs/docker-stack/index.html" \
|
|
|
|
|
org.opencontainers.image.source="https://github.com/apache/airflow" \
|
|
|
|
|
org.opencontainers.image.version="${AIRFLOW_VERSION}" \
|
|
|
|
|
org.opencontainers.image.revision="${COMMIT_SHA}" \
|
|
|
|
|
org.opencontainers.image.vendor="Apache Software Foundation" \
|
|
|
|
|
org.opencontainers.image.licenses="Apache-2.0" \
|
|
|
|
|
org.opencontainers.image.ref.name="airflow-ci-image" \
|
|
|
|
|
org.opencontainers.image.title="Continuous Integration Airflow Image" \
|
|
|
|
|
org.opencontainers.image.description="Installed Apache Airflow with Continuous Integration dependencies"
|
2020-08-19 02:03:22 +02:00
|
|
|
|
2020-06-16 12:36:46 +02:00
|
|
|
ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]
|
2022-01-11 10:38:34 +01:00
|
|
|
CMD []
|
