2025-04-06 18:29:56 +02:00
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
|
|
|
# or more contributor license agreements. See the NOTICE file
|
|
|
|
|
# distributed with this work for additional information
|
|
|
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
|
|
|
# to you under the Apache License, Version 2.0 (the
|
|
|
|
|
# "License"); you may not use this file except in compliance
|
|
|
|
|
# with the License. You may obtain a copy of the License at
|
|
|
|
|
#
|
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
#
|
|
|
|
|
# Unless required by applicable law or agreed to in writing,
|
|
|
|
|
# software distributed under the License is distributed on an
|
|
|
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
|
# KIND, either express or implied. See the License for the
|
|
|
|
|
# specific language governing permissions and limitations
|
|
|
|
|
# under the License.
|
|
|
|
|
|
|
|
|
|
[build-system]
|
|
|
|
|
requires = [
|
2026-02-27 21:05:52 +01:00
|
|
|
"hatchling==1.29.0",
|
|
|
|
|
"packaging==26.0",
|
|
|
|
|
"pathspec==1.0.4",
|
|
|
|
|
"pluggy==1.6.0",
|
2026-03-27 19:03:24 +01:00
|
|
|
"tomli==2.4.0; python_version < '3.11'",
|
2026-02-27 21:05:52 +01:00
|
|
|
"trove-classifiers==2026.1.14.14",
|
2025-04-06 18:29:56 +02:00
|
|
|
]
|
|
|
|
|
build-backend = "hatchling.build"
|
|
|
|
|
|
|
|
|
|
[project]
|
2025-08-30 20:18:30 +05:30
|
|
|
name = "docker-stack"
|
2025-04-06 18:29:56 +02:00
|
|
|
description = "Programmatically author, schedule and monitor data pipelines"
|
Add Python 3.14 Support (#63520)
* Update python version exclusion to 3.15
* Add 3.14 metadata version classifiers and related constants
* Regenerate Breeze command help screenshots
* Assorted workarounds to fix breeze image building
- constraints are skipped entirely
- greenlet pin updated
* Exclude cassandra
* Exclude amazon
* Exclude google
* CI: Only add pydantic extra to Airflow 2 migration tests
Before this fix there were two separate issues in the migration-test setup for Python 3.14:
1. The migration workflow always passes --airflow-extras pydantic.
2. For Python 3.14, the minimum Airflow version is resolved to 3.2.0 by get_min_airflow_version_for_python.py, and apache-airflow[pydantic]==3.2.0 is not a valid thing to install.
So when constraints installation fails, the fallback path tries to install an invalid spec.
* Disable DB migration tests for python 3.14
* Enforce werkzeug 3.x for python 3.14
* Increase K8s executor test timeout for Python 3.14
Python 3.14 changed the default multiprocessing start method from 'fork' to 'forkserver' on Linux. The forkserver start method is slower because each new process must import modules from scratch rather than copying the parent's address space. This makes `multiprocessing.Manager()` initialization take longer, causing the test to exceed its 10s timeout.
* Adapt LocalExecutor tests for Python 3.14 forkserver default
Python 3.14 changed the default multiprocessing start method from
'fork' to 'forkserver' on Linux. Like 'spawn', 'forkserver' doesn't
share the parent's address space, so mock patches applied in the test
process are invisible to worker subprocesses.
- Skip tests that mock across process boundaries on non-fork methods
- Add test_executor_lazy_worker_spawning to verify that non-fork start
methods defer worker creation and skip gc.freeze
- Make test_multiple_team_executors_isolation and
test_global_executor_without_team_name assert the correct worker
count for each start method instead of assuming pre-spawning
- Remove skip from test_clean_stop_on_signal (works on all methods)
and increase timeout from 5s to 30s for forkserver overhead
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* Bump dependencies to versions supporting 3.14
* Fix PROD image build failing on Python 3.14 due to excluded providers
The PROD image build installed all provider wheels regardless of Python
version compatibility. Providers like google and amazon that exclude
Python 3.14 were still passed to pip, causing resolution failures (e.g.
ray has no cp314 wheel on PyPI).
Two fixes:
- get_distribution_specs.py now reads each wheel's Requires-Python
metadata and skips incompatible wheels instead of passing them to pip.
- The requires-python specifier generation used !=3.14 which per PEP 440
only excludes 3.14.0, not 3.14.3. Changed to !=3.14.* wildcard.
* Split core test types into 2 matrix groups to avoid OOM on Python 3.14
Non-DB core tests use xdist which runs all test types in a single pytest
process. With 2059 items across 4 workers, memory accumulates until the
OOM killer strikes at ~86% completion (exit code 137).
Split core test types into 2 groups (API/Always/CLI and
Core/Other/Serialization), similar to how provider tests already use
_split_list with NUMBER_OF_LOW_DEP_SLICES. Each group gets ~1000 items,
well under the ~1770 threshold where OOM occurs.
Update selective_checks test expectations to reflect the 2-group split.
* Gracefully handle an already removed password file in fixture
The old code had a check-then-act race (if `os.path.exists` → `os.remove`), which fails when the file doesn't exist at removal time. `contextlib.suppress(FileNotFoundError)` handles this atomically — if the file is missing (never created in this xdist worker, or removed between check and delete), it's silently ignored.
* Fix OOM and flaky tests in test_process_utils
Replace multiprocessing.Process with subprocess.Popen running minimal
inline scripts. multiprocessing.Process uses fork(), which duplicates
the entire xdist worker memory. At 95% test completion the worker has
accumulated hundreds of MBs; forking it triggers the OOM killer
(exit code 137) on Python 3.14.
subprocess.Popen starts a fresh lightweight process (~10MB) without
copying the parent's memory, avoiding the OOM entirely.
Also replace the racy ps -ax process counting in
TestKillChildProcessesByPids with psutil.pid_exists() checks on the
specific PID — the old approach was non-deterministic because unrelated
processes could start/stop between measurements.
* Add prek hook to validate python_version markers for excluded providers
When a provider declares excluded-python-versions in provider.yaml,
every dependency string referencing that provider in pyproject.toml
must carry a matching python_version marker. Missing markers cause
excluded providers to be silently installed as transitive dependencies
(e.g. aiobotocore pulling in amazon on Python 3.14).
The new check-excluded-provider-markers hook reads exclusions from
provider.yaml and validates all dependency strings in pyproject.toml
at commit time, preventing regressions like the one fixed in the
previous commit.
* Update `uv.lock`
---------
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-19 23:03:46 +02:00
|
|
|
requires-python = ">=3.10,!=3.15"
|
2025-04-06 18:29:56 +02:00
|
|
|
authors = [
|
|
|
|
|
{ name = "Apache Software Foundation", email = "dev@airflow.apache.org" },
|
|
|
|
|
]
|
|
|
|
|
maintainers = [
|
|
|
|
|
{ name = "Apache Software Foundation", email="dev@airflow.apache.org" },
|
|
|
|
|
]
|
|
|
|
|
keywords = [ "airflow", "orchestration", "workflow", "dag", "pipelines", "automation", "data" ]
|
|
|
|
|
classifiers = [
|
|
|
|
|
"Development Status :: 5 - Production/Stable",
|
|
|
|
|
"Environment :: Console",
|
|
|
|
|
"Environment :: Web Environment",
|
|
|
|
|
"Framework :: Apache Airflow",
|
|
|
|
|
"Intended Audience :: Developers",
|
|
|
|
|
"Intended Audience :: System Administrators",
|
|
|
|
|
"Programming Language :: Python :: 3.10",
|
|
|
|
|
"Programming Language :: Python :: 3.11",
|
|
|
|
|
"Programming Language :: Python :: 3.12",
|
Add Python 3.13 support for Airflow. (#46891)
* Breeze changes for Python 3.13
* Pyproject.toml provider changes for Python 3.13
* README.rst changes for providers for Python 3.13
* Add Python 3.13 support for Airflow.
Added Python 3.13 support across the codebase, including:
* CI/CD workflow updates to handle Python 3.13.
* Dockerfile and environment variable adjustments for compatibility.
* Documentation updates to list Python 3.13 as supported.
* Dependency and code changes for compatibility with new/changed
libraries (e.g., numpy 2.x, greenlet, pendulum).
* Test and plugin manager adjustments for Python 3.13-specific behaviors.
* Removal or skipping of tests and features not compatible with Python 3.13.
* Improved error handling and logging for unsupported plugins/providers
on Python 3.13.
* Updated all provider README.rst files to:
* List Python 3.13 as a supported version.
* Adjust dependency requirements for Python 3.13 (e.g., pandas,
pyarrow, ray, etc.).
* Add or update notes about excluded or conditionally supported
dependencies for Python 3.13.
* Document any known incompatibilities or workarounds for specific
providers.
* Updated all provider pyproject.toml files and automation to:
* Add Python 3.13 to the supported classifiers.
* Adjust requires-python and dependency constraints for Python 3.13.
* Exclude Python 3.13 for providers that are not yet compatible (e.g.,
those depending on Flask AppBuilder).
* Add conditional dependencies for new versions of libraries required
by Python 3.13.
* Updated Breeze and related scripts to:
* Build and test with Python 3.13.
* Update documentation and release scripts to include Python 3.13.
* Ensure all dev tooling and test environments work with Python 3.13.
* Removed remaining FAB related code from core and removed tests for
older FAB provider versions (<1.3.0)
* some of the tests in core still depend on FAB - but we make them
optional - both for parsing (we need to skip some imports) and
execution - because they test a legacy behaviour of embedding
Flask plugins that is still supported in core.
* k8s tests now run also with SimpleAuthManager and SimpleAuthManager
is used automatically when runnin tests in Python 3.13
* migration tests had to be excluded fo Python 3.13 because we cannot
migrate down on Python 3.13 without FAB tables created. The tests
will be brought back when FAB is supported or when we release 3.1
and start working on 3.2, we should be able to migrate down to 3.1
* Weaviate tests have been updated to handle case where weaviate-client
is > 2.10.0 - because it started to use httpx instead of requests,
and Python 3.13 actually unblocks using older version of weaviate
(it was previously blocked indirectly via Flask dependencies and
grpcio).
* Added doc change in our rules on what is the "default" Python version
for airflow images - handling the case where we cannot use "newest"
Python version as default where not all "regular" image providers are
supported in the latest version
* Connection is now flushed in `configure_git_connection_for_dag_bundle`
just before `clear_db_connections` - in order to handle Python 3.13
case where FAB app is not initialized by other fixtures - performing
implicit flush along the way
* Cleaning db connections was added to test_dag_run and test_dags test
in fast_api as the test implicitly rely on having connections
cleared (no test git connections added) - this will make the tests
side-effect free.
* Conditionally skipping some serialization tests involving yandex
and ray on Python 3.13 until they support Python 3.13
2025-07-17 16:53:49 +02:00
|
|
|
"Programming Language :: Python :: 3.13",
|
Add Python 3.14 Support (#63520)
* Update python version exclusion to 3.15
* Add 3.14 metadata version classifiers and related constants
* Regenerate Breeze command help screenshots
* Assorted workarounds to fix breeze image building
- constraints are skipped entirely
- greenlet pin updated
* Exclude cassandra
* Exclude amazon
* Exclude google
* CI: Only add pydantic extra to Airflow 2 migration tests
Before this fix there were two separate issues in the migration-test setup for Python 3.14:
1. The migration workflow always passes --airflow-extras pydantic.
2. For Python 3.14, the minimum Airflow version is resolved to 3.2.0 by get_min_airflow_version_for_python.py, and apache-airflow[pydantic]==3.2.0 is not a valid thing to install.
So when constraints installation fails, the fallback path tries to install an invalid spec.
* Disable DB migration tests for python 3.14
* Enforce werkzeug 3.x for python 3.14
* Increase K8s executor test timeout for Python 3.14
Python 3.14 changed the default multiprocessing start method from 'fork' to 'forkserver' on Linux. The forkserver start method is slower because each new process must import modules from scratch rather than copying the parent's address space. This makes `multiprocessing.Manager()` initialization take longer, causing the test to exceed its 10s timeout.
* Adapt LocalExecutor tests for Python 3.14 forkserver default
Python 3.14 changed the default multiprocessing start method from
'fork' to 'forkserver' on Linux. Like 'spawn', 'forkserver' doesn't
share the parent's address space, so mock patches applied in the test
process are invisible to worker subprocesses.
- Skip tests that mock across process boundaries on non-fork methods
- Add test_executor_lazy_worker_spawning to verify that non-fork start
methods defer worker creation and skip gc.freeze
- Make test_multiple_team_executors_isolation and
test_global_executor_without_team_name assert the correct worker
count for each start method instead of assuming pre-spawning
- Remove skip from test_clean_stop_on_signal (works on all methods)
and increase timeout from 5s to 30s for forkserver overhead
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* Bump dependencies to versions supporting 3.14
* Fix PROD image build failing on Python 3.14 due to excluded providers
The PROD image build installed all provider wheels regardless of Python
version compatibility. Providers like google and amazon that exclude
Python 3.14 were still passed to pip, causing resolution failures (e.g.
ray has no cp314 wheel on PyPI).
Two fixes:
- get_distribution_specs.py now reads each wheel's Requires-Python
metadata and skips incompatible wheels instead of passing them to pip.
- The requires-python specifier generation used !=3.14 which per PEP 440
only excludes 3.14.0, not 3.14.3. Changed to !=3.14.* wildcard.
* Split core test types into 2 matrix groups to avoid OOM on Python 3.14
Non-DB core tests use xdist which runs all test types in a single pytest
process. With 2059 items across 4 workers, memory accumulates until the
OOM killer strikes at ~86% completion (exit code 137).
Split core test types into 2 groups (API/Always/CLI and
Core/Other/Serialization), similar to how provider tests already use
_split_list with NUMBER_OF_LOW_DEP_SLICES. Each group gets ~1000 items,
well under the ~1770 threshold where OOM occurs.
Update selective_checks test expectations to reflect the 2-group split.
* Gracefully handle an already removed password file in fixture
The old code had a check-then-act race (if `os.path.exists` → `os.remove`), which fails when the file doesn't exist at removal time. `contextlib.suppress(FileNotFoundError)` handles this atomically — if the file is missing (never created in this xdist worker, or removed between check and delete), it's silently ignored.
* Fix OOM and flaky tests in test_process_utils
Replace multiprocessing.Process with subprocess.Popen running minimal
inline scripts. multiprocessing.Process uses fork(), which duplicates
the entire xdist worker memory. At 95% test completion the worker has
accumulated hundreds of MBs; forking it triggers the OOM killer
(exit code 137) on Python 3.14.
subprocess.Popen starts a fresh lightweight process (~10MB) without
copying the parent's memory, avoiding the OOM entirely.
Also replace the racy ps -ax process counting in
TestKillChildProcessesByPids with psutil.pid_exists() checks on the
specific PID — the old approach was non-deterministic because unrelated
processes could start/stop between measurements.
* Add prek hook to validate python_version markers for excluded providers
When a provider declares excluded-python-versions in provider.yaml,
every dependency string referencing that provider in pyproject.toml
must carry a matching python_version marker. Missing markers cause
excluded providers to be silently installed as transitive dependencies
(e.g. aiobotocore pulling in amazon on Python 3.14).
The new check-excluded-provider-markers hook reads exclusions from
provider.yaml and validates all dependency strings in pyproject.toml
at commit time, preventing regressions like the one fixed in the
previous commit.
* Update `uv.lock`
---------
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-19 23:03:46 +02:00
|
|
|
"Programming Language :: Python :: 3.14",
|
2025-04-06 18:29:56 +02:00
|
|
|
"Topic :: System :: Monitoring",
|
|
|
|
|
"Topic :: System :: Monitoring",
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
version = "0.0.1"
|
|
|
|
|
|
|
|
|
|
dependencies = [
|
|
|
|
|
"apache-airflow-core",
|
|
|
|
|
"apache-airflow-devel-common",
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
[tool.hatch.build.targets.sdist]
|
|
|
|
|
exclude = ["*"]
|
|
|
|
|
|
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
|
|
|
bypass-selection = true
|
|
|
|
|
|
|
|
|
|
[dependency-groups]
|
|
|
|
|
# To build docs run:
|
|
|
|
|
#
|
|
|
|
|
# uv run --group docs sphinx-build -T --color -b html . _build
|
|
|
|
|
#
|
|
|
|
|
# To check spelling:
|
|
|
|
|
#
|
|
|
|
|
# uv run --group docs sphinx-build -T --color -b spelling . _build
|
|
|
|
|
#
|
|
|
|
|
# To enable auto-refreshing build with server:
|
|
|
|
|
#
|
|
|
|
|
# uv run --group docs sphinx-autobuild -T --color -b html . _build
|
|
|
|
|
#
|
|
|
|
|
docs = [
|
|
|
|
|
"apache-airflow-devel-common[docs]"
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
packages = []
|
