2019-08-14 20:53:28 -04:00
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
|
|
|
# or more contributor license agreements. See the NOTICE file
|
|
|
|
|
# distributed with this work for additional information
|
|
|
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
|
|
|
# to you under the Apache License, Version 2.0 (the
|
|
|
|
|
# "License"); you may not use this file except in compliance
|
|
|
|
|
# with the License. You may obtain a copy of the License at
|
|
|
|
|
#
|
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
#
|
|
|
|
|
# Unless required by applicable law or agreed to in writing,
|
|
|
|
|
# software distributed under the License is distributed on an
|
|
|
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
|
# KIND, either express or implied. See the License for the
|
|
|
|
|
# specific language governing permissions and limitations
|
|
|
|
|
# under the License.
|
|
|
|
|
---
|
2024-10-09 21:31:26 +01:00
|
|
|
default_stages: [pre-commit, pre-push]
|
2026-03-02 21:44:11 +09:00
|
|
|
minimum_prek_version: '0.3.4'
|
2019-08-22 10:13:56 -04:00
|
|
|
default_language_version:
|
|
|
|
|
python: python3
|
2025-08-29 13:40:37 +08:00
|
|
|
node: 22.19.0
|
2025-11-05 11:35:28 +01:00
|
|
|
golang: 1.24.0
|
2025-01-02 20:29:53 +10:00
|
|
|
exclude: ^.*/.*_vendor/
|
2019-08-14 20:53:28 -04:00
|
|
|
repos:
|
2020-12-16 14:14:20 +01:00
|
|
|
- repo: meta
|
|
|
|
|
hooks:
|
|
|
|
|
- id: identity
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Print checked files
|
|
|
|
|
description: Print input to the static check hooks for troubleshooting
|
2020-12-16 14:14:20 +01:00
|
|
|
- id: check-hooks-apply
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Check if all hooks apply to the repository
|
2022-02-15 23:52:50 +01:00
|
|
|
- repo: https://github.com/thlorenz/doctoc.git
|
2026-02-25 07:24:06 +01:00
|
|
|
rev: d7815f1f950f8d5ec933fa4f70208bf316bb13f8 # frozen: v2.3.0
|
2022-02-15 23:52:50 +01:00
|
|
|
hooks:
|
|
|
|
|
- id: doctoc
|
2023-02-13 01:04:11 +10:00
|
|
|
name: Add TOC for Markdown and RST files
|
2022-02-15 23:52:50 +01:00
|
|
|
files:
|
2025-05-12 19:03:22 +02:00
|
|
|
(?x)
|
|
|
|
|
^README\.md$|
|
|
|
|
|
^UPDATING.*\.md$|
|
|
|
|
|
^chart/UPDATING.*\.md$|
|
|
|
|
|
^dev/.*\.md$|
|
|
|
|
|
^dev/.*\.rst$|
|
|
|
|
|
^docs/README\.md$|
|
|
|
|
|
^\.github/.*\.md$|
|
|
|
|
|
^airflow-core/tests/system/README\.md$
|
2026-01-09 15:33:48 +01:00
|
|
|
exclude:
|
2026-02-19 10:01:41 -05:00
|
|
|
(?x)
|
|
|
|
|
.github/PULL_REQUEST_TEMPLATE\.md$|
|
2026-02-25 02:43:21 +00:00
|
|
|
.github/instructions/|
|
2026-02-19 10:01:41 -05:00
|
|
|
.github/skills/
|
2022-02-15 23:52:50 +01:00
|
|
|
args:
|
|
|
|
|
- "--maxlevel"
|
|
|
|
|
- "2"
|
2019-09-04 15:56:03 +02:00
|
|
|
- repo: https://github.com/Lucas-C/pre-commit-hooks
|
2026-01-17 16:43:25 +01:00
|
|
|
rev: ad1b27d73581aa16cca06fc4a0761fc563ffe8e8 # frozen: v1.5.6
|
2019-08-15 19:59:35 -04:00
|
|
|
hooks:
|
|
|
|
|
- id: insert-license
|
2019-10-26 13:31:51 +02:00
|
|
|
name: Add license for all SQL files
|
2019-08-15 19:59:35 -04:00
|
|
|
files: \.sql$
|
2022-10-23 17:17:26 +01:00
|
|
|
exclude: |
|
|
|
|
|
(?x)
|
2026-02-17 23:38:20 +01:00
|
|
|
^\.github/|
|
|
|
|
|
^scripts/ci/license-templates/
|
2019-08-15 19:59:35 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "/*||*/"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-16 00:09:26 -04:00
|
|
|
- id: insert-license
|
2023-02-13 01:04:11 +10:00
|
|
|
name: Add license for all RST files
|
2019-08-16 00:09:26 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "||"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.rst
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-16 00:09:26 -04:00
|
|
|
files: \.rst$
|
2026-02-17 23:38:20 +01:00
|
|
|
exclude:
|
|
|
|
|
(?x)
|
|
|
|
|
^\.github/.*$|
|
|
|
|
|
newsfragments/.*\.rst$|
|
|
|
|
|
^scripts/ci/license-templates/
|
2019-08-15 20:06:18 -04:00
|
|
|
- id: insert-license
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Add license for CSS/JS/JSX/PUML/TS/TSX
|
2023-02-13 01:04:11 +10:00
|
|
|
files: \.(css|jsx?|puml|tsx?)$
|
2025-09-03 09:58:16 +02:00
|
|
|
exclude: ^\.github/.*$|ui/openapi-gen/|www/openapi-gen/|.*/dist/.*
|
2019-08-15 20:06:18 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
2020-10-16 03:59:09 -04:00
|
|
|
- "/*!| *| */"
|
2019-08-15 20:06:18 -04:00
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-20 00:07:39 -04:00
|
|
|
- id: insert-license
|
2023-02-13 01:04:11 +10:00
|
|
|
name: Add license for all Shell files
|
2025-01-02 20:29:53 +10:00
|
|
|
exclude: ^\.github/.*$|^dev/breeze/autocomplete/.*$
|
2022-09-02 23:20:45 +02:00
|
|
|
files: \.bash$|\.sh$
|
2019-08-20 00:07:39 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "|#|"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
Generate Python client in reproducible way (#36763)
Client source code and package generation was done using the code
generated and committed to `airflow-client-python` and while the
repository with such code is useful to have, it's just a convenience
repo, because all sources are (and should be) generated from the
API specification which is present in the Airflow repository.
This also made the reproducible builds and package generation not really
possible, because we never knew if the source generated in the
`airflow-client-python` repository has been generated and not tampered
with.
While implementing it, it turned out that there were some issues in
the past that nade our client generation somewhat broken..
* In 2.7.0 python client, we added the same code twice
(See https://github.com/apache/airflow-client-python/pull/93) on
top of "airflow_client.client" package, we also added copy of the
API client generated in "airflow_client.airflow_client" - that was
likely due to bad bash scripts and tools that were used to generate
it and errors during generation the clients.
* We used to generate the code for "client" package and then moved
the "client" package to "airflow_client.client" package, while
manually modifying imports with `sed` (!?). That was likely due to
limitations in some old version of the client generator. However the
client generator we use now is capable of generating code directly in
the "airflow_client.client" package.
* We also manually (via pre-commit) added Apache Licence to the
generated files. Whieh was completely unnecessary, because ASF rules
do not require licence headers to be added to code automatically
generated from a code that already has ASF licence.
* We also generated source tarball packages from such generated code,
which was completely unnecessary - because sdist packages are already
fulfilling all the reqirements of such source pacakges - the code
in the packages is enough to build the package from the sources and
it does not contain any binary code, moreover the code is generated
out of the API specificiation, which means that anyone can take
the code and genearate the pacakged software from just sources in
sdist. Similarly as in case of provider packages, we do not need
to produce separate -source.tar.gz files.
This PR fixes all of it.
First of all the source that lands in the source repository
`airflow-client-python` and sdist/wheel packages are generated directly
from the openapi specification.
They are generated using breeze release_management command from airflow
source tagged with specific tag in the Airflow repo (including the
source of reproducible build date that is updated together with airflow
release notes. This means that any PMC member can regenerate packages
(binary identical) straight from the Airflow repository - without
going through "airflow-client-python" repository.
No source tarball is generated - it is not needed, sdist is enough.
The `test_python_client.py` has been also moved over to Airflow repo
and updated with handling case when expose_config is not enabled and
it is used to automatically test the API client after it has been
generated.
2024-01-14 18:17:20 +01:00
|
|
|
- id: insert-license
|
|
|
|
|
name: Add license for all toml files
|
2025-01-02 20:29:53 +10:00
|
|
|
exclude: ^\.github/.*$|^dev/breeze/autocomplete/.*$
|
Generate Python client in reproducible way (#36763)
Client source code and package generation was done using the code
generated and committed to `airflow-client-python` and while the
repository with such code is useful to have, it's just a convenience
repo, because all sources are (and should be) generated from the
API specification which is present in the Airflow repository.
This also made the reproducible builds and package generation not really
possible, because we never knew if the source generated in the
`airflow-client-python` repository has been generated and not tampered
with.
While implementing it, it turned out that there were some issues in
the past that nade our client generation somewhat broken..
* In 2.7.0 python client, we added the same code twice
(See https://github.com/apache/airflow-client-python/pull/93) on
top of "airflow_client.client" package, we also added copy of the
API client generated in "airflow_client.airflow_client" - that was
likely due to bad bash scripts and tools that were used to generate
it and errors during generation the clients.
* We used to generate the code for "client" package and then moved
the "client" package to "airflow_client.client" package, while
manually modifying imports with `sed` (!?). That was likely due to
limitations in some old version of the client generator. However the
client generator we use now is capable of generating code directly in
the "airflow_client.client" package.
* We also manually (via pre-commit) added Apache Licence to the
generated files. Whieh was completely unnecessary, because ASF rules
do not require licence headers to be added to code automatically
generated from a code that already has ASF licence.
* We also generated source tarball packages from such generated code,
which was completely unnecessary - because sdist packages are already
fulfilling all the reqirements of such source pacakges - the code
in the packages is enough to build the package from the sources and
it does not contain any binary code, moreover the code is generated
out of the API specificiation, which means that anyone can take
the code and genearate the pacakged software from just sources in
sdist. Similarly as in case of provider packages, we do not need
to produce separate -source.tar.gz files.
This PR fixes all of it.
First of all the source that lands in the source repository
`airflow-client-python` and sdist/wheel packages are generated directly
from the openapi specification.
They are generated using breeze release_management command from airflow
source tagged with specific tag in the Airflow repo (including the
source of reproducible build date that is updated together with airflow
release notes. This means that any PMC member can regenerate packages
(binary identical) straight from the Airflow repository - without
going through "airflow-client-python" repository.
No source tarball is generated - it is not needed, sdist is enough.
The `test_python_client.py` has been also moved over to Airflow repo
and updated with handling case when expose_config is not enabled and
it is used to automatically test the API client after it has been
generated.
2024-01-14 18:17:20 +01:00
|
|
|
files: \.toml$
|
|
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "|#|"
|
|
|
|
|
- --license-filepath
|
|
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
|
|
|
|
- --fuzzy-match-generates-todo
|
2019-09-17 13:16:32 +02:00
|
|
|
- id: insert-license
|
2021-05-06 12:36:32 +10:00
|
|
|
name: Add license for all Python files
|
2025-04-21 20:01:05 +05:30
|
|
|
exclude: ^\.github/.*$|^.*/_vendor/.*$|^airflow-ctl/.*/.*generated\.py$
|
2021-12-27 16:46:22 +01:00
|
|
|
files: \.py$|\.pyi$
|
2019-09-17 13:16:32 +02:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "|#|"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2019-09-17 13:16:32 +02:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-20 10:29:34 -04:00
|
|
|
- id: insert-license
|
2019-10-26 13:31:51 +02:00
|
|
|
name: Add license for all XML files
|
2025-01-02 20:29:53 +10:00
|
|
|
exclude: ^\.github/.*$
|
2020-08-24 22:45:28 +02:00
|
|
|
files: \.xml$
|
2019-08-20 10:29:34 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "<!--||-->"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2023-04-22 23:22:29 +02:00
|
|
|
- id: insert-license
|
|
|
|
|
name: Add license for all YAML files except Helm templates
|
2025-05-20 09:49:44 -04:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
2025-10-26 15:30:54 +01:00
|
|
|
^\.github/.*$|
|
|
|
|
|
^chart/templates/.*|
|
2025-05-20 09:49:44 -04:00
|
|
|
.*reproducible_build\.yaml$|
|
|
|
|
|
^.*/v2.*\.yaml$|
|
|
|
|
|
^.*/openapi/_private_ui.*\.yaml$|
|
2025-05-20 12:48:21 -04:00
|
|
|
^.*/pnpm-lock\.yaml$|
|
2025-05-20 09:49:44 -04:00
|
|
|
.*-generated\.yaml$
|
2019-08-22 10:13:56 -04:00
|
|
|
types: [yaml]
|
2023-02-13 01:04:11 +10:00
|
|
|
files: \.ya?ml$
|
2019-08-22 10:13:56 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "|#|"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-21 23:27:54 -04:00
|
|
|
- id: insert-license
|
2023-02-13 01:04:11 +10:00
|
|
|
name: Add license for all Markdown files
|
2019-08-21 23:27:54 -04:00
|
|
|
files: \.md$
|
|
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "<!--|| -->"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2026-01-09 15:33:48 +01:00
|
|
|
exclude:
|
|
|
|
|
(?x)
|
2026-02-17 23:38:20 +01:00
|
|
|
^\.github/.*\.md$|
|
|
|
|
|
^\.claude/|
|
2026-03-12 21:36:21 +00:00
|
|
|
^(?:.*/)?AGENTS\.md$|
|
|
|
|
|
^(?:.*/)?CLAUDE\.md$|
|
|
|
|
|
^(?:.*/)?SKILL\.md$|
|
2026-02-17 23:38:20 +01:00
|
|
|
^scripts/ci/license-templates/
|
|
|
|
|
- id: insert-license
|
|
|
|
|
name: Add short license for agentic Markdown files
|
|
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "||"
|
|
|
|
|
- --license-filepath
|
|
|
|
|
- scripts/ci/license-templates/SHORT_LICENSE.md
|
|
|
|
|
- --fuzzy-match-generates-todo
|
|
|
|
|
files:
|
|
|
|
|
(?x)
|
|
|
|
|
^\.github/.*\.md$|
|
|
|
|
|
^\.claude/|
|
2026-03-12 21:36:21 +00:00
|
|
|
^(?:.*/)?AGENTS\.md$|
|
|
|
|
|
^(?:.*/)?CLAUDE\.md$|
|
|
|
|
|
^(?:.*/)?SKILL\.md$
|
2026-02-17 23:38:20 +01:00
|
|
|
exclude:
|
2026-02-19 10:01:41 -05:00
|
|
|
(?x)
|
|
|
|
|
^scripts/ci/license-templates/|
|
2026-02-25 02:43:21 +00:00
|
|
|
^\.github/instructions/|
|
2026-02-19 10:01:41 -05:00
|
|
|
^\.github/skills/airflow-translations/SKILL\.md$
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: insert-license
|
|
|
|
|
name: Add license for all other files
|
|
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "|#|"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2022-04-23 21:33:08 +02:00
|
|
|
- --fuzzy-match-generates-todo
|
2026-02-17 23:38:20 +01:00
|
|
|
files: |
|
|
|
|
|
(?x)
|
|
|
|
|
\.cfg$|
|
|
|
|
|
\.conf$|
|
|
|
|
|
\.ini$|
|
|
|
|
|
\.ldif$|
|
|
|
|
|
\.properties$|
|
|
|
|
|
\.service$|
|
|
|
|
|
\.tf$|
|
|
|
|
|
Dockerfile.*$
|
|
|
|
|
exclude:
|
|
|
|
|
(?x)
|
|
|
|
|
^\.github/.*$|
|
|
|
|
|
^scripts/ci/license-templates/
|
2022-12-18 17:04:48 +01:00
|
|
|
- repo: local
|
|
|
|
|
hooks:
|
2024-10-23 14:59:50 +02:00
|
|
|
- id: check-min-python-version
|
|
|
|
|
name: Check minimum Python version
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_min_python_version.py
|
2024-10-23 14:59:50 +02:00
|
|
|
language: python
|
|
|
|
|
require_serial: true
|
2026-01-17 16:20:55 +01:00
|
|
|
- id: check-notice-files
|
|
|
|
|
name: Check NOTICE files for current year and ASF references
|
|
|
|
|
entry: ./scripts/ci/prek/check_notice_files.py
|
|
|
|
|
language: python
|
|
|
|
|
files: ^.*NOTICE$
|
2025-12-16 17:15:36 +01:00
|
|
|
- id: check-version-consistency
|
|
|
|
|
name: Check version consistency
|
|
|
|
|
entry: ./scripts/ci/prek/check_version_consistency.py
|
|
|
|
|
language: python
|
|
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-core/src/airflow/__init__\.py$|
|
|
|
|
|
^airflow-core/pyproject\.toml$|
|
|
|
|
|
^task-sdk/src/airflow/sdk/__init__\.py$|
|
|
|
|
|
^pyproject\.toml$
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
2026-03-15 21:07:51 +01:00
|
|
|
- id: check-distribution-gitignore
|
|
|
|
|
name: Check distribution .gitignore files have *.iml
|
|
|
|
|
entry: ./scripts/ci/prek/check_distribution_gitignore.py
|
|
|
|
|
language: python
|
|
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
^.*/\.gitignore$|
|
|
|
|
|
^\.gitignore$|
|
|
|
|
|
^.*/pyproject\.toml$|
|
|
|
|
|
^pyproject\.toml$
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
2025-08-17 09:00:14 +02:00
|
|
|
- id: upgrade-important-versions
|
|
|
|
|
name: Upgrade important versions (manual)
|
|
|
|
|
entry: ./scripts/ci/prek/upgrade_important_versions.py
|
2024-02-28 00:05:41 +01:00
|
|
|
stages: ['manual']
|
|
|
|
|
language: python
|
2025-06-23 18:33:23 +02:00
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
^\.pre-commit-config\.yaml$|
|
|
|
|
|
^\.github/\.pre-commit-config\.yaml$|
|
2025-08-17 09:00:14 +02:00
|
|
|
^scripts/ci/prek/update_installers_and_prek\.py$
|
2024-02-28 00:05:41 +01:00
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
2025-09-08 22:32:18 +01:00
|
|
|
- repo: https://github.com/adamchainz/blacken-docs
|
2025-12-14 09:36:08 +01:00
|
|
|
rev: fda77690955e9b63c6687d8806bafd56a526e45f # frozen: 1.20.0
|
2021-05-27 07:09:25 +02:00
|
|
|
hooks:
|
|
|
|
|
- id: blacken-docs
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Run black on docs
|
2022-07-01 20:12:15 +02:00
|
|
|
args:
|
|
|
|
|
- --line-length=110
|
|
|
|
|
- --target-version=py310
|
2024-10-23 01:28:50 +01:00
|
|
|
- --target-version=py311
|
|
|
|
|
- --target-version=py312
|
2025-08-19 14:26:08 +02:00
|
|
|
- --target-version=py313
|
2023-10-31 18:51:50 -04:00
|
|
|
alias: blacken-docs
|
2025-08-19 14:26:08 +02:00
|
|
|
additional_dependencies:
|
2026-02-08 13:50:18 +02:00
|
|
|
- 'black==26.1.0'
|
2019-08-16 10:35:34 -04:00
|
|
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
2025-08-11 23:23:42 +02:00
|
|
|
rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0
|
2019-08-16 10:35:34 -04:00
|
|
|
hooks:
|
|
|
|
|
- id: check-merge-conflict
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Check that merge conflicts are not being committed
|
2019-11-26 09:28:42 +01:00
|
|
|
- id: debug-statements
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Detect accidentally committed debug statements
|
2020-06-25 16:35:37 +01:00
|
|
|
- id: check-builtin-literals
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Require literal syntax when initializing builtins
|
2019-08-16 10:35:34 -04:00
|
|
|
- id: detect-private-key
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Detect if private key is added to the repository
|
2025-04-09 01:26:02 +10:00
|
|
|
exclude: ^providers/ssh/docs/connections/ssh\.rst$
|
2019-08-16 10:35:34 -04:00
|
|
|
- id: end-of-file-fixer
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Make sure that there is an empty line at the end
|
2025-06-15 14:13:19 +02:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-core/docs/img/.*\.dot|
|
|
|
|
|
^airflow-core/docs/img/.*\.sha256|
|
|
|
|
|
.*/dist/.*|
|
|
|
|
|
LICENSES-ui\.txt$|
|
|
|
|
|
.*/openapi-gen/.*
|
2019-08-16 10:35:34 -04:00
|
|
|
- id: mixed-line-ending
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Detect if mixed line ending is used (\r vs. \r\n)
|
2019-08-20 10:29:34 -04:00
|
|
|
- id: check-executables-have-shebangs
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Check that executables have shebang
|
2019-08-20 10:29:34 -04:00
|
|
|
- id: check-xml
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Check XML files with xmllint
|
2025-04-24 11:19:09 +01:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
|
|
|
|
^scripts/ci/docker-compose/gremlin/.
|
2019-12-28 19:34:35 +01:00
|
|
|
- id: trailing-whitespace
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Remove trailing whitespace at end of line
|
2025-06-15 14:13:19 +02:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-core/docs/img/.*\.dot$|
|
|
|
|
|
^dev/breeze/doc/images/output.*$|
|
2025-06-29 16:24:54 +02:00
|
|
|
^.*/openapi-gen/.*$|
|
|
|
|
|
^airflow-ctl/docs/images/.*\.svg$
|
2019-09-21 20:47:02 +01:00
|
|
|
- repo: https://github.com/pre-commit/pygrep-hooks
|
2025-08-11 23:23:42 +02:00
|
|
|
rev: 3a6eb0fadf60b3cccfd80bad9dbb6fae7e47b316 # frozen: v1.10.0
|
2019-09-21 20:47:02 +01:00
|
|
|
hooks:
|
|
|
|
|
- id: rst-backticks
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Check if RST files use double backticks for code
|
2019-09-22 08:55:39 +01:00
|
|
|
- id: python-no-log-warn
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Check if there are no deprecate log warn
|
2020-06-14 02:37:30 +02:00
|
|
|
- repo: https://github.com/adrienverge/yamllint
|
2026-01-17 12:06:28 +01:00
|
|
|
rev: cba56bcde1fdd01c1deb3f945e69764c291a6530 # frozen: v1.38.0
|
2019-08-14 20:53:28 -04:00
|
|
|
hooks:
|
2019-08-22 10:13:56 -04:00
|
|
|
- id: yamllint
|
2021-05-06 12:36:32 +10:00
|
|
|
name: Check YAML files with yamllint
|
2020-10-21 12:24:32 +02:00
|
|
|
entry: yamllint -c yamllint-config.yml --strict
|
2019-08-22 10:13:56 -04:00
|
|
|
types: [yaml]
|
2025-03-01 12:43:03 +01:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
|
|
|
|
^.*airflow\.template\.yaml$|
|
|
|
|
|
^.*init_git_sync\.template\.yaml$|
|
|
|
|
|
^chart/(?:templates|files)/.*\.yaml$|
|
2025-05-19 13:13:41 -04:00
|
|
|
^helm-tests/tests/chart_utils/keda.sh_scaledobjects\.yaml$|
|
2025-04-15 15:30:16 +02:00
|
|
|
.*/v1.*\.yaml$|
|
2025-03-01 12:43:03 +01:00
|
|
|
^.*openapi.*\.yaml$|
|
|
|
|
|
^\.pre-commit-config\.yaml$|
|
2025-04-09 01:26:02 +10:00
|
|
|
^.*reproducible_build\.yaml$|
|
2025-12-07 23:53:40 +01:00
|
|
|
^.*pnpm-lock\.yaml$|
|
|
|
|
|
^.*-generated\.yaml$
|
2021-09-10 11:54:15 -06:00
|
|
|
- repo: https://github.com/ikamensh/flynt
|
2025-10-14 16:17:52 +02:00
|
|
|
rev: 97be693bf18bc2f050667dd282d243e2824b81e2 # frozen: 1.0.6
|
2021-08-18 01:56:38 +02:00
|
|
|
hooks:
|
|
|
|
|
- id: flynt
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Run flynt string format converter for Python
|
2021-10-17 18:34:06 +02:00
|
|
|
args:
|
|
|
|
|
# If flynt detects too long text it ignores it. So we set a very large limit to make it easy
|
|
|
|
|
# to split the text by hand. Too long lines are detected by flake8 (below),
|
|
|
|
|
# so the user is informed to take action.
|
|
|
|
|
- --line-length
|
|
|
|
|
- '99999'
|
2021-10-14 19:37:08 +01:00
|
|
|
- repo: https://github.com/codespell-project/codespell
|
2026-03-09 23:16:04 +01:00
|
|
|
rev: 2ccb47ff45ad361a21071a7eedda4c37e6ae8c5a # frozen: v2.4.2
|
2021-10-14 19:37:08 +01:00
|
|
|
hooks:
|
|
|
|
|
- id: codespell
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Run codespell
|
|
|
|
|
description: Run codespell to check for common misspellings in files
|
2021-10-30 15:44:20 +05:30
|
|
|
entry: bash -c 'echo "If you think that this failure is an error, consider adding the word(s)
|
|
|
|
|
to the codespell dictionary at docs/spelling_wordlist.txt.
|
|
|
|
|
The word(s) should be in lowercase." && exec codespell "$@"' --
|
2021-10-14 19:37:08 +01:00
|
|
|
language: python
|
|
|
|
|
types: [text]
|
2025-05-23 23:01:25 +02:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
|
|
|
|
material-icons\.css$|
|
|
|
|
|
^images/.*$|
|
|
|
|
|
^RELEASE_NOTES\.txt$|
|
|
|
|
|
^.*package-lock\.json$|
|
|
|
|
|
^.*/kinglear\.txt$|
|
|
|
|
|
^.*pnpm-lock\.yaml$|
|
|
|
|
|
.*/dist/.*|
|
2026-02-27 01:31:40 +08:00
|
|
|
^airflow-core/src/airflow/ui/public/i18n/locales/(?!en/).+/|
|
|
|
|
|
^\.github/skills/airflow-translations/
|
2021-10-14 19:37:08 +01:00
|
|
|
args:
|
|
|
|
|
- --ignore-words=docs/spelling_wordlist.txt
|
2025-05-08 14:50:47 +02:00
|
|
|
- --skip=providers/.*/src/airflow/providers/*/*.rst,providers/*/docs/changelog.rst,docs/*/commits.rst,providers/*/docs/commits.rst,providers/*/*/docs/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md,*.svg
|
2022-01-24 12:43:09 +00:00
|
|
|
- --exclude-file=.codespellignorelines
|
2025-01-05 15:52:38 +00:00
|
|
|
- repo: https://github.com/woodruffw/zizmor-pre-commit
|
2026-03-12 15:52:51 -07:00
|
|
|
rev: ea2eb407b4cbce87cf0d502f36578950494f5ac9 # frozen: v1.23.1
|
2025-01-05 15:52:38 +00:00
|
|
|
hooks:
|
|
|
|
|
- id: zizmor
|
|
|
|
|
name: Run zizmor to check for github workflow syntax errors
|
|
|
|
|
types: [yaml]
|
2025-04-09 01:26:02 +10:00
|
|
|
files: ^\.github/workflows/.*$|^\.github/actions/.*$
|
2025-01-05 15:52:38 +00:00
|
|
|
require_serial: true
|
|
|
|
|
entry: zizmor
|
2020-06-14 02:37:30 +02:00
|
|
|
- repo: local
|
2023-04-21 17:53:48 +02:00
|
|
|
# Note that this is the 2nd "local" repo group in the .pre-commit-config.yaml file. This is because
|
2025-08-17 09:00:14 +02:00
|
|
|
# we try to minimize the number of passes that must happen to apply some of the changes
|
|
|
|
|
# done by prek-hooks. Some of the prek hooks not only check for errors but also fix them. This means
|
|
|
|
|
# that output from an earlier prek hook becomes input to another prek hook. Splitting the local
|
|
|
|
|
# scripts of our and adding some other non-local prek hook in-between allows us to handle such
|
2023-04-21 17:53:48 +02:00
|
|
|
# changes quickly - especially when we want the early modifications from the first local group
|
2025-08-17 09:00:14 +02:00
|
|
|
# to be applied before the non-local prek hooks are run
|
2020-06-14 02:37:30 +02:00
|
|
|
hooks:
|
2025-07-26 10:22:14 +02:00
|
|
|
- id: check-shared-distributions-structure
|
|
|
|
|
name: Check shared distributions structure
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_shared_distributions_structure.py
|
2025-07-25 16:26:22 +02:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: false
|
2025-07-27 07:51:39 +10:00
|
|
|
files: ^shared/.*$
|
2025-07-26 10:22:14 +02:00
|
|
|
- id: check-shared-distributions-usage
|
|
|
|
|
name: Check shared distributions usage
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_shared_distributions_usage.py
|
2025-07-26 10:22:14 +02:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: false
|
2025-07-27 07:51:39 +10:00
|
|
|
files: ^shared/.*$|^.*/pyproject.toml$|^.*/_shared/.*$
|
2026-02-03 18:42:35 +05:30
|
|
|
- id: check-airflow-imports-in-shared
|
|
|
|
|
name: Check for core/sdk imports in shared libraries
|
|
|
|
|
entry: ./scripts/ci/prek/check_airflow_imports_in_shared.py
|
|
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
files: ^shared/.*/src/.*\.py$
|
|
|
|
|
exclude: |
|
|
|
|
|
(?x)
|
|
|
|
|
^shared/listeners/src/airflow_shared/listeners/spec/taskinstance\.py$|
|
|
|
|
|
^shared/logging/src/airflow_shared/logging/remote\.py$|
|
|
|
|
|
^shared/observability/src/airflow_shared/observability/metrics/stats\.py$|
|
|
|
|
|
^shared/secrets_backend/src/airflow_shared/secrets_backend/base\.py$
|
2026-02-10 19:22:04 +02:00
|
|
|
- id: check-test-only-imports-in-src
|
|
|
|
|
name: Check for test-only imports in production source
|
|
|
|
|
entry: ./scripts/ci/prek/check_test_only_imports_in_src.py
|
|
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-core/src/.*\.py$|
|
|
|
|
|
^airflow-ctl/src/.*\.py$|
|
|
|
|
|
^providers/.*/src/.*\.py$|
|
|
|
|
|
^task-sdk/src/.*\.py$|
|
|
|
|
|
^shared/.*/src/.*\.py$
|
2026-01-09 15:45:58 +05:30
|
|
|
- id: check-secrets-search-path-sync
|
|
|
|
|
name: Check sync between sdk and core
|
|
|
|
|
entry: ./scripts/ci/prek/check_secrets_search_path_sync.py
|
|
|
|
|
language: python
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
files: ^airflow-core/src/airflow/secrets/base_secrets\.py$|^task-sdk/src/airflow/sdk/execution_time/secrets/__init__\.py$
|
2026-03-11 04:15:43 +00:00
|
|
|
- id: check-registry-types-json-sync
|
|
|
|
|
name: Check registry types.json in sync with types.py
|
|
|
|
|
entry: ./scripts/ci/prek/check_registry_types_json_sync.py
|
|
|
|
|
language: python
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
files: ^dev/registry/registry_tools/types\.py$|^registry/src/_data/types\.json$
|
2024-01-07 20:02:08 +01:00
|
|
|
- id: ruff
|
|
|
|
|
name: Run 'ruff' for extremely fast Python linting
|
|
|
|
|
description: "Run 'ruff' for extremely fast Python linting"
|
|
|
|
|
entry: ruff check --force-exclude
|
|
|
|
|
language: python
|
|
|
|
|
types_or: [python, pyi]
|
|
|
|
|
args: [--fix]
|
|
|
|
|
require_serial: true
|
2026-03-20 01:04:32 -07:00
|
|
|
additional_dependencies: ['ruff==0.15.7']
|
2025-04-09 01:26:02 +10:00
|
|
|
exclude: ^airflow-core/tests/unit/dags/test_imports\.py$|^performance/tests/test_.*\.py$
|
2024-01-07 20:02:08 +01:00
|
|
|
- id: ruff-format
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Run 'ruff format'
|
2024-01-07 20:02:08 +01:00
|
|
|
description: "Run 'ruff format' for extremely fast Python formatting"
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/ruff_format.py
|
2024-01-07 20:02:08 +01:00
|
|
|
language: python
|
|
|
|
|
types_or: [python, pyi]
|
|
|
|
|
args: []
|
|
|
|
|
require_serial: true
|
2025-04-09 01:26:02 +10:00
|
|
|
exclude: ^airflow-core/tests/unit/dags/test_imports\.py$
|
2022-07-05 17:08:52 +02:00
|
|
|
- id: replace-bad-characters
|
|
|
|
|
name: Replace bad characters
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/replace_bad_characters.py
|
2022-07-05 17:08:52 +02:00
|
|
|
language: python
|
|
|
|
|
types: [file, text]
|
2025-09-03 09:58:16 +02:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
|
|
|
|
^clients/gen/go\.sh$|
|
|
|
|
|
^\.gitmodules$|
|
|
|
|
|
^airflow-core/src/airflow/ui/openapi-gen/|
|
2026-03-25 10:51:11 -03:00
|
|
|
^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/openapi-gen/|
|
2025-09-03 09:58:16 +02:00
|
|
|
^providers/edge3/src/airflow/providers/edge3/plugins/www/openapi-gen/|
|
|
|
|
|
.*/dist/.*|
|
|
|
|
|
\.go$|
|
|
|
|
|
/go\.(mod|sum)$
|
2019-08-14 20:53:28 -04:00
|
|
|
- id: lint-dockerfile
|
2023-02-13 01:04:11 +10:00
|
|
|
name: Lint Dockerfile
|
2022-09-02 17:21:41 +02:00
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/lint_dockerfile.py
|
2020-09-06 18:06:05 +02:00
|
|
|
files: Dockerfile.*$
|
2019-09-18 13:43:30 +02:00
|
|
|
pass_filenames: true
|
2021-02-27 18:38:43 +01:00
|
|
|
require_serial: true
|
2023-12-12 16:15:48 +01:00
|
|
|
- id: check-airflow-providers-bug-report-template
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Sort airflow-bug-report provider list
|
2023-12-12 16:15:48 +01:00
|
|
|
language: python
|
2025-05-13 12:34:56 +05:30
|
|
|
files: ^\.github/ISSUE_TEMPLATE/3-airflow_providers_bug_report\.yml$
|
2023-12-12 16:15:48 +01:00
|
|
|
require_serial: true
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_airflow_bug_report_template.py
|
2020-03-07 18:12:28 +01:00
|
|
|
- id: update-local-yml-file
|
|
|
|
|
name: Update mounts in the local yml file
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/local_yml_mounts.py
|
2022-05-28 17:19:14 +02:00
|
|
|
language: python
|
|
|
|
|
files: ^dev/breeze/src/airflow_breeze/utils/docker_command_utils\.py$|^scripts/ci/docker_compose/local\.yml$
|
2020-03-07 18:12:28 +01:00
|
|
|
pass_filenames: false
|
2021-09-21 21:22:41 +02:00
|
|
|
- id: check-extras-order
|
|
|
|
|
name: Check order of extras in Dockerfile
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_order_dockerfile_extras.py
|
2021-09-21 21:22:41 +02:00
|
|
|
language: python
|
|
|
|
|
files: ^Dockerfile$
|
|
|
|
|
pass_filenames: false
|
2023-12-05 21:15:16 +01:00
|
|
|
- id: generate-airflow-diagrams
|
|
|
|
|
name: Generate airflow diagrams
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/generate_airflow_diagrams.py
|
2023-12-05 21:15:16 +01:00
|
|
|
language: python
|
2025-05-12 19:03:22 +02:00
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-core/docs/.*/diagram_[^/]*\.py$|
|
2025-09-16 20:46:44 +02:00
|
|
|
^docs/images/.*\.py$|
|
|
|
|
|
^airflow-ctl/docs/images/diagrams/.*\.py$
|
2023-12-20 19:44:33 +01:00
|
|
|
pass_filenames: true
|
2025-06-23 19:25:50 +05:30
|
|
|
- id: prevent-deprecated-sqlalchemy-usage
|
|
|
|
|
name: Prevent deprecated sqlalchemy usage
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/prevent_deprecated_sqlalchemy_usage.py
|
2025-06-23 19:25:50 +05:30
|
|
|
language: python
|
|
|
|
|
files: >
|
|
|
|
|
(?x)
|
2026-01-17 17:44:58 +08:00
|
|
|
^airflow-core/.*\.py$|
|
2026-01-31 21:26:08 +05:30
|
|
|
^airflow-ctl.*\.py$|
|
|
|
|
|
^airflow-ctl-tests.*\.py|
|
|
|
|
|
^dev/.*\.py$|
|
|
|
|
|
^devel-common/.*\.py|
|
|
|
|
|
^providers/.*\.py$|
|
|
|
|
|
^task-sdk.*\.py$|
|
|
|
|
|
^task-sdk-integration-tests.*\.py
|
2025-06-23 19:25:50 +05:30
|
|
|
pass_filenames: true
|
2021-12-30 22:13:10 +01:00
|
|
|
- id: update-supported-versions
|
|
|
|
|
name: Updates supported versions in documentation
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/supported_versions.py
|
2021-12-30 22:13:10 +01:00
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
files: ^airflow-core/docs/installation/supported-versions\.rst$|^scripts/ci/prek/supported_versions\.py$|^README\.md$
|
2021-12-30 22:13:10 +01:00
|
|
|
pass_filenames: false
|
2022-04-08 18:20:54 +01:00
|
|
|
- id: check-revision-heads-map
|
|
|
|
|
name: Check that the REVISION_HEADS_MAP is up-to-date
|
|
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_revision_heads_map.py
|
2022-04-08 18:20:54 +01:00
|
|
|
pass_filenames: false
|
2023-08-29 12:30:26 -07:00
|
|
|
files: >
|
|
|
|
|
(?x)
|
2025-08-17 09:00:14 +02:00
|
|
|
^scripts/ci/prek/version_heads_map\.py$|
|
2025-10-24 17:23:32 +02:00
|
|
|
^airflow-core/src/airflow/migrations/versions/.*$|
|
|
|
|
|
^airflow-core/src/airflow/migrations/versions|
|
|
|
|
|
^airflow-core/src/airflow/utils/db\.py$
|
2021-09-18 10:41:57 +02:00
|
|
|
- id: update-version
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Update versions in docs
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/update_versions.py
|
2021-09-18 10:41:57 +02:00
|
|
|
language: python
|
2025-04-27 17:13:14 +02:00
|
|
|
files: ^docs|^airflow-core/src/airflow/__init__\.py$|.*/pyproject\.toml$
|
2021-09-18 10:41:57 +02:00
|
|
|
pass_filenames: false
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-pydevd-left-in-code
|
2019-11-26 09:28:42 +01:00
|
|
|
language: pygrep
|
|
|
|
|
name: Check for pydevd debug statements accidentally left
|
|
|
|
|
entry: "pydevd.*settrace\\("
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
files: \.py$
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-safe-filter-usage-in-html
|
2020-06-08 22:27:02 +01:00
|
|
|
language: pygrep
|
|
|
|
|
name: Don't use safe in templates
|
|
|
|
|
description: the Safe filter is error-prone, use Markup() in code instead
|
|
|
|
|
entry: "\\|\\s*safe"
|
|
|
|
|
files: \.html$
|
|
|
|
|
pass_filenames: true
|
2023-02-20 10:06:12 +09:00
|
|
|
- id: check-urlparse-usage-in-code
|
|
|
|
|
language: pygrep
|
|
|
|
|
name: Don't use urlparse in code
|
|
|
|
|
description: urlparse is not recommended, use urlsplit() in code instead
|
|
|
|
|
entry: "^\\s*from urllib\\.parse import ((\\|, )(urlparse\\|urlunparse))+$"
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
files: \.py$
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-for-inclusive-language
|
Don't use the term "whitelist" - language matters (#9174)
It's fairly common to say whitelisting and blacklisting to describe
desirable and undesirable things in cyber security. However just because
it is common doesn't mean it's right.
However, there's an issue with the terminology. It only makes sense if
you equate white with 'good, permitted, safe' and black with 'bad,
dangerous, forbidden'. There are some obvious problems with this.
You may not see why this matters. If you're not adversely affected by
racial stereotyping yourself, then please count yourself lucky. For some
of your friends and colleagues (and potential future colleagues), this
really is a change worth making.
From now on, we will use 'allow list' and 'deny list' in place of
'whitelist' and 'blacklist' wherever possible. Which, in fact, is
clearer and less ambiguous. So as well as being more inclusive of all,
this is a net benefit to our understandability.
(Words mostly borrowed from
<https://www.ncsc.gov.uk/blog-post/terminology-its-not-black-and-white>)
Co-authored-by: Jarek Potiuk <jarek@potiuk.com>
2020-06-08 10:01:46 +01:00
|
|
|
language: pygrep
|
|
|
|
|
name: Check for language that we do not accept as community
|
2022-05-09 16:52:29 -05:00
|
|
|
description: Please use more appropriate words for community documentation.
|
|
|
|
|
entry: >
|
2023-02-22 09:45:03 -05:00
|
|
|
(?ix)
|
2022-05-09 16:52:29 -05:00
|
|
|
(black|white)[_-]?list|
|
|
|
|
|
\bshe\b|
|
|
|
|
|
\bhe\b|
|
|
|
|
|
\bher\b|
|
|
|
|
|
\bhis\b|
|
|
|
|
|
\bmaster\b|
|
|
|
|
|
\bslave\b|
|
|
|
|
|
\bsanity\b|
|
|
|
|
|
\bdummy\b
|
Don't use the term "whitelist" - language matters (#9174)
It's fairly common to say whitelisting and blacklisting to describe
desirable and undesirable things in cyber security. However just because
it is common doesn't mean it's right.
However, there's an issue with the terminology. It only makes sense if
you equate white with 'good, permitted, safe' and black with 'bad,
dangerous, forbidden'. There are some obvious problems with this.
You may not see why this matters. If you're not adversely affected by
racial stereotyping yourself, then please count yourself lucky. For some
of your friends and colleagues (and potential future colleagues), this
really is a change worth making.
From now on, we will use 'allow list' and 'deny list' in place of
'whitelist' and 'blacklist' wherever possible. Which, in fact, is
clearer and less ambiguous. So as well as being more inclusive of all,
this is a net benefit to our understandability.
(Words mostly borrowed from
<https://www.ncsc.gov.uk/blog-post/terminology-its-not-black-and-white>)
Co-authored-by: Jarek Potiuk <jarek@potiuk.com>
2020-06-08 10:01:46 +01:00
|
|
|
pass_filenames: true
|
|
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
2025-12-02 18:00:42 +01:00
|
|
|
^README\.md$|
|
Add Python 3.14 Support (#63520)
* Update python version exclusion to 3.15
* Add 3.14 metadata version classifiers and related constants
* Regenerate Breeze command help screenshots
* Assorted workarounds to fix breeze image building
- constraints are skipped entirely
- greenlet pin updated
* Exclude cassandra
* Exclude amazon
* Exclude google
* CI: Only add pydantic extra to Airflow 2 migration tests
Before this fix there were two separate issues in the migration-test setup for Python 3.14:
1. The migration workflow always passes --airflow-extras pydantic.
2. For Python 3.14, the minimum Airflow version is resolved to 3.2.0 by get_min_airflow_version_for_python.py, and apache-airflow[pydantic]==3.2.0 is not a valid thing to install.
So when constraints installation fails, the fallback path tries to install an invalid spec.
* Disable DB migration tests for python 3.14
* Enforce werkzeug 3.x for python 3.14
* Increase K8s executor test timeout for Python 3.14
Python 3.14 changed the default multiprocessing start method from 'fork' to 'forkserver' on Linux. The forkserver start method is slower because each new process must import modules from scratch rather than copying the parent's address space. This makes `multiprocessing.Manager()` initialization take longer, causing the test to exceed its 10s timeout.
* Adapt LocalExecutor tests for Python 3.14 forkserver default
Python 3.14 changed the default multiprocessing start method from
'fork' to 'forkserver' on Linux. Like 'spawn', 'forkserver' doesn't
share the parent's address space, so mock patches applied in the test
process are invisible to worker subprocesses.
- Skip tests that mock across process boundaries on non-fork methods
- Add test_executor_lazy_worker_spawning to verify that non-fork start
methods defer worker creation and skip gc.freeze
- Make test_multiple_team_executors_isolation and
test_global_executor_without_team_name assert the correct worker
count for each start method instead of assuming pre-spawning
- Remove skip from test_clean_stop_on_signal (works on all methods)
and increase timeout from 5s to 30s for forkserver overhead
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* Bump dependencies to versions supporting 3.14
* Fix PROD image build failing on Python 3.14 due to excluded providers
The PROD image build installed all provider wheels regardless of Python
version compatibility. Providers like google and amazon that exclude
Python 3.14 were still passed to pip, causing resolution failures (e.g.
ray has no cp314 wheel on PyPI).
Two fixes:
- get_distribution_specs.py now reads each wheel's Requires-Python
metadata and skips incompatible wheels instead of passing them to pip.
- The requires-python specifier generation used !=3.14 which per PEP 440
only excludes 3.14.0, not 3.14.3. Changed to !=3.14.* wildcard.
* Split core test types into 2 matrix groups to avoid OOM on Python 3.14
Non-DB core tests use xdist which runs all test types in a single pytest
process. With 2059 items across 4 workers, memory accumulates until the
OOM killer strikes at ~86% completion (exit code 137).
Split core test types into 2 groups (API/Always/CLI and
Core/Other/Serialization), similar to how provider tests already use
_split_list with NUMBER_OF_LOW_DEP_SLICES. Each group gets ~1000 items,
well under the ~1770 threshold where OOM occurs.
Update selective_checks test expectations to reflect the 2-group split.
* Gracefully handle an already removed password file in fixture
The old code had a check-then-act race (if `os.path.exists` → `os.remove`), which fails when the file doesn't exist at removal time. `contextlib.suppress(FileNotFoundError)` handles this atomically — if the file is missing (never created in this xdist worker, or removed between check and delete), it's silently ignored.
* Fix OOM and flaky tests in test_process_utils
Replace multiprocessing.Process with subprocess.Popen running minimal
inline scripts. multiprocessing.Process uses fork(), which duplicates
the entire xdist worker memory. At 95% test completion the worker has
accumulated hundreds of MBs; forking it triggers the OOM killer
(exit code 137) on Python 3.14.
subprocess.Popen starts a fresh lightweight process (~10MB) without
copying the parent's memory, avoiding the OOM entirely.
Also replace the racy ps -ax process counting in
TestKillChildProcessesByPids with psutil.pid_exists() checks on the
specific PID — the old approach was non-deterministic because unrelated
processes could start/stop between measurements.
* Add prek hook to validate python_version markers for excluded providers
When a provider declares excluded-python-versions in provider.yaml,
every dependency string referencing that provider in pyproject.toml
must carry a matching python_version marker. Missing markers cause
excluded providers to be silently installed as transitive dependencies
(e.g. aiobotocore pulling in amazon on Python 3.14).
The new check-excluded-provider-markers hook reads exclusions from
provider.yaml and validates all dependency strings in pyproject.toml
at commit time, preventing regressions like the one fixed in the
previous commit.
* Update `uv.lock`
---------
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-19 23:03:46 +02:00
|
|
|
^pyproject\.toml$|
|
2025-12-02 18:00:42 +01:00
|
|
|
^generated/PYPI_README\.md$|
|
2025-10-25 22:26:32 +02:00
|
|
|
^airflow-core/docs/.*commits\.rst$|
|
|
|
|
|
^airflow-core/newsfragments/41368\.significant\.rst$|
|
|
|
|
|
^airflow-core/newsfragments/41761.significant\.rst$|
|
|
|
|
|
^airflow-core/newsfragments/43349\.significant\.rst$|
|
2026-02-04 09:45:09 +00:00
|
|
|
^airflow-core/newsfragments/60921\.significant\.rst$|
|
2025-10-25 22:26:32 +02:00
|
|
|
^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/pnpm-lock\.yaml$|
|
2026-02-04 09:45:09 +00:00
|
|
|
^airflow-core/src/airflow/api_fastapi/gunicorn_config\.py$|
|
|
|
|
|
^airflow-core/src/airflow/cli/commands/api_server_command\.py$|
|
|
|
|
|
^airflow-core/src/airflow/api_fastapi/gunicorn_monitor\.py$|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
^airflow-core/src/airflow/cli/commands/local_commands/fastapi_api_command\.py$|
|
2025-03-21 14:25:26 +01:00
|
|
|
^airflow-core/src/airflow/config_templates/|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
^airflow-core/src/airflow/models/baseoperator\.py$|
|
|
|
|
|
^airflow-core/src/airflow/operators/__init__\.py$|
|
2025-10-25 22:26:32 +02:00
|
|
|
^airflow-core/src/airflow/serialization/serialized_objects\.py$|
|
|
|
|
|
^airflow-core/src/airflow/ui/openapi-gen/|
|
|
|
|
|
^airflow-core/src/airflow/ui/pnpm-lock\.yaml$|
|
2026-03-11 07:18:54 +00:00
|
|
|
^providers/common/ai/src/airflow/providers/common/ai/plugins/www/pnpm-lock\.yaml$|
|
2025-10-25 22:26:32 +02:00
|
|
|
^airflow-core/src/airflow/ui/public/i18n/locales/de/README\.md$|
|
|
|
|
|
^airflow-core/src/airflow/ui/src/i18n/config\.ts$|
|
|
|
|
|
^airflow-core/src/airflow/utils/db\.py$|
|
|
|
|
|
^airflow-core/src/airflow/utils/trigger_rule\.py$|
|
|
|
|
|
^airflow-core/tests/|
|
2026-01-19 17:19:21 +05:30
|
|
|
^task-sdk/tests/|
|
2025-10-25 22:26:32 +02:00
|
|
|
^.*changelog\.(rst|txt)$|
|
|
|
|
|
^.*CHANGELOG\.(rst|txt)$|
|
|
|
|
|
^chart/values.schema\.json$|
|
|
|
|
|
^.*commits\.(rst|txt)$|
|
|
|
|
|
^.*/conf_constants\.py$|
|
|
|
|
|
^.*/conf\.py$|
|
|
|
|
|
^contributing-docs/03_contributors_quick_start\.rst$|
|
|
|
|
|
^dev/|
|
|
|
|
|
^devel-common/src/docs/README\.rst$|
|
|
|
|
|
^devel-common/src/sphinx_exts/removemarktransform\.py|
|
|
|
|
|
^devel-common/src/tests_common/test_utils/db\.py|
|
|
|
|
|
.*/dist/.*|
|
|
|
|
|
^docs/apache-airflow-providers-amazon/secrets-backends/aws-ssm-parameter-store\.rst$|
|
|
|
|
|
git|
|
|
|
|
|
^helm-tests/tests/chart_utils/helm_template_generator\.py$|
|
|
|
|
|
package-lock\.json$|
|
Add Apache Airflow Provider Registry (#62261)
Devlist Discussion: https://lists.apache.org/thread/7n4pklzcc4lxtxsy9g69ssffg9qbdyvb
A static-site provider registry for discovering and browsing Airflow providers and their modules. Deployed at `airflow.apache.org/registry/` alongside the existing docs infrastructure (S3 + CloudFront).
Staging preview: https://airflow.staged.apache.org/registry/
## Acknowledgments
Many of you know the [Astronomer Registry](https://registry.astronomer.io), which has been the go-to for discovering providers for years. Big thanks to **Astronomer** and @josh-fell for building and maintaining it. This new registry is designed to be a community-owned successor on `airflow.apache.org`, with the eventual goal of redirecting `registry.astronomer.io` traffic here once it's stable. Thanks also to @ashb for suggesting and prototyping the Eleventy-based approach.
## What it does
The registry indexes all 99 official providers and 840 modules (operators, hooks, sensors, triggers, transfers, bundles, notifiers, secrets backends, log handlers, executors) from the existing
`providers/*/provider.yaml` files and source code in this repo. No external data sources beyond PyPI download stats.
**Pages:**
- **Homepage** — search bar (Cmd+K), stats counters, featured and new providers
- **Providers listing** — filterable by lifecycle stage (stable/incubation/deprecated), category, and sort order (downloads, name, recently updated)
- **Provider detail** — module counts by type, install command with extras/version selection, dependency info, connection builder, and a tabbed module browser with category sidebar and per-module search
- **Explore by Category** — providers grouped into Cloud, Databases, Data Warehouses, Messaging, AI/ML, Data Processing, etc.
- **Statistics** — module type distribution, lifecycle breakdown, top providers by downloads and module count
- **JSON API** — `/api/providers.json`, `/api/modules.json`, per-provider endpoints for modules, parameters, and connections
**Connection Builder** — pick a connection type (e.g. `aws`, `redshift`), fill in the form fields with placeholders and sensitivity markers, and export as URI, JSON, or environment variable format. Fields are
extracted from provider.yaml connection metadata.
2026-03-07 04:01:20 +00:00
|
|
|
^.*\.(png|gif|jp[e]?g|svg|tgz|lock|woff2?)$|
|
2025-10-25 22:26:32 +02:00
|
|
|
^\.pre-commit-config\.yaml$|
|
|
|
|
|
^.*/provider_conf\.py$|
|
|
|
|
|
^providers/\.pre-commit-config\.yaml$|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
^providers/amazon/src/airflow/providers/amazon/aws/hooks/emr\.py$|
|
|
|
|
|
^providers/amazon/src/airflow/providers/amazon/aws/operators/emr\.py$|
|
2026-02-17 19:43:41 +05:30
|
|
|
^providers/.*/get_provider_info\.py$|
|
|
|
|
|
^providers/.*/provider\.yaml$|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
^providers/apache/cassandra/src/airflow/providers/apache/cassandra/hooks/cassandra\.py$|
|
2025-10-25 22:26:32 +02:00
|
|
|
^providers/apache/hdfs/docs/connections\.rst$|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
^providers/apache/hive/src/airflow/providers/apache/hive/operators/hive_stats\.py$|
|
|
|
|
|
^providers/apache/hive/src/airflow/providers/apache/hive/transfers/vertica_to_hive\.py$|
|
2025-10-25 22:26:32 +02:00
|
|
|
^providers/apache/kafka/docs/connections/kafka\.rst$|
|
|
|
|
|
^providers/apache/spark/docs/decorators/pyspark\.rst$|
|
2026-02-25 07:56:18 +01:00
|
|
|
^providers/apache/spark/docs/connections/spark-submit.rst$|
|
2025-01-27 17:20:24 +08:00
|
|
|
^providers/apache/spark/src/airflow/providers/apache/spark/decorators/|
|
|
|
|
|
^providers/apache/spark/src/airflow/providers/apache/spark/hooks/|
|
|
|
|
|
^providers/apache/spark/src/airflow/providers/apache/spark/operators/|
|
2025-10-25 22:26:32 +02:00
|
|
|
^providers/cncf/kubernetes/docs/operators\.rst$|
|
|
|
|
|
^providers/common/sql/tests/provider_tests/common/sql/operators/test_sql_execute\.py$|
|
|
|
|
|
^providers/edge3/src/airflow/providers/edge3/plugins/www/pnpm-lock.yaml$|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
^providers/exasol/src/airflow/providers/exasol/hooks/exasol\.py$|
|
2025-10-25 22:26:32 +02:00
|
|
|
^providers/fab/docs/auth-manager/webserver-authentication\.rst$|
|
2025-02-01 04:19:00 +08:00
|
|
|
^providers/fab/src/airflow/providers/fab/auth_manager/security_manager/|
|
2025-03-11 09:40:26 -04:00
|
|
|
^providers/fab/src/airflow/providers/fab/www/static/|
|
|
|
|
|
^providers/fab/src/airflow/providers/fab/www/templates/|
|
2025-10-25 22:26:32 +02:00
|
|
|
^providers/google/docs/operators/cloud/kubernetes_engine\.rst$|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
^providers/google/src/airflow/providers/google/cloud/hooks/bigquery\.py$|
|
|
|
|
|
^providers/google/src/airflow/providers/google/cloud/operators/cloud_build\.py$|
|
|
|
|
|
^providers/google/src/airflow/providers/google/cloud/operators/dataproc\.py$|
|
|
|
|
|
^providers/google/src/airflow/providers/google/cloud/operators/mlengine\.py$|
|
2026-01-06 18:48:17 +08:00
|
|
|
^providers/keycloak/src/airflow/providers/keycloak/cli/definition.py|
|
2025-10-25 22:26:32 +02:00
|
|
|
^providers/microsoft/azure/docs/connections/azure_cosmos\.rst$|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
^providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/cosmos\.py$|
|
|
|
|
|
^providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/hooks/winrm\.py$|
|
|
|
|
|
^providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/operators/winrm\.py$|
|
|
|
|
|
^providers/opsgenie/src/airflow/providers/opsgenie/hooks/opsgenie\.py$|
|
|
|
|
|
^providers/redis/src/airflow/providers/redis/provider\.yaml$|
|
2025-01-25 23:27:19 +01:00
|
|
|
^providers/.*/tests/|
|
2025-10-25 15:22:30 +02:00
|
|
|
.rat-excludes|
|
2025-10-25 22:26:32 +02:00
|
|
|
^.*RELEASE_NOTES\.rst$|
|
|
|
|
|
^scripts/ci/docker-compose/integration-keycloak\.yml$|
|
|
|
|
|
^scripts/ci/docker-compose/keycloak/keycloak-entrypoint\.sh$|
|
2025-11-13 00:26:32 +01:00
|
|
|
^scripts/ci/prek/upgrade_important_versions.py$|
|
2026-03-04 21:51:02 +01:00
|
|
|
^scripts/ci/prek/download_k8s_schemas\.py$|
|
2025-10-25 22:26:32 +02:00
|
|
|
^scripts/ci/prek/vendor_k8s_json_schema\.py$
|
2024-03-29 10:06:02 +08:00
|
|
|
- id: check-template-context-variable-in-sync
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Sync template context variable refs
|
2024-03-29 10:06:02 +08:00
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_template_context_variable_in_sync.py
|
2025-10-26 23:07:12 +01:00
|
|
|
files:
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-core/src/airflow/models/taskinstance\.py$|
|
|
|
|
|
^task-sdk/src/airflow/sdk/definitions/context\.py$|
|
|
|
|
|
^airflow-core/docs/templates-ref\.rst$
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-base-operator-usage
|
2019-11-26 23:19:45 +01:00
|
|
|
language: pygrep
|
2023-11-21 21:10:40 +04:00
|
|
|
name: Check BaseOperator core imports
|
|
|
|
|
description: Make sure BaseOperator is imported from airflow.models.baseoperator in core
|
|
|
|
|
entry: "from airflow\\.models import.* BaseOperator\\b"
|
|
|
|
|
files: \.py$
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
2025-03-21 14:25:26 +01:00
|
|
|
^airflow-core/src/airflow/decorators/.*$|
|
|
|
|
|
^airflow-core/src/airflow/hooks/.*$|
|
|
|
|
|
^airflow-core/src/airflow/operators/.*$|
|
|
|
|
|
^providers/.*$
|
2023-11-21 21:10:40 +04:00
|
|
|
- id: check-base-operator-usage
|
|
|
|
|
language: pygrep
|
|
|
|
|
name: Check BaseOperatorLink core imports
|
2025-03-07 13:54:43 +05:30
|
|
|
description: Make sure BaseOperatorLink is not imported from airflow.models in core
|
|
|
|
|
entry: "^\\s*from airflow\\.models\\.baseoperatorlink import BaseOperatorLink\\b"
|
2019-11-26 23:19:45 +01:00
|
|
|
files: \.py$
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
2025-03-21 14:25:26 +01:00
|
|
|
^airflow-core/src/airflow/decorators/.*$|
|
|
|
|
|
^airflow-core/src/airflow/hooks/.*$|
|
|
|
|
|
^airflow-core/src/airflow/operators/.*$|
|
2025-01-25 23:27:19 +01:00
|
|
|
^providers/.*/src/airflow/providers/.*$|
|
2025-03-21 14:25:26 +01:00
|
|
|
^providers/.*/src/airflow/providers/standard/sensors/.*$
|
2022-08-24 00:18:18 +03:00
|
|
|
- id: check-core-deprecation-classes
|
|
|
|
|
language: pygrep
|
2023-05-24 11:21:52 -07:00
|
|
|
name: Verify usage of Airflow deprecation classes in core
|
2022-08-24 00:18:18 +03:00
|
|
|
entry: category=DeprecationWarning|category=PendingDeprecationWarning
|
|
|
|
|
files: \.py$
|
Add tests for scripts and remove redundant sys.path.insert calls (#63598)
* Add tests for scripts and remove redundant sys.path.insert calls
- Remove 85 redundant `sys.path.insert(0, str(Path(__file__).parent.resolve()))`
calls from scripts in ci/prek/, cov/, and in_container/. Python already
adds the script's directory to sys.path when running a file directly,
making these calls unnecessary.
- Keep 6 cross-directory sys.path.insert calls that are genuinely needed
(AIRFLOW_CORE_SOURCES_PATH, AIRFLOW_ROOT, etc.).
- Add __init__.py files to scripts/ci/ and scripts/ci/prek/ to make them
proper Python packages.
- Add scripts/pyproject.toml with package discovery and pytest config.
- Add 176 tests covering: common_prek_utils (insert_documentation,
check_list_sorted, get_provider_id_from_path, ConsoleDiff, etc.),
new_session_in_provide_session, check_deprecations, unittest_testcase,
changelog_duplicates, newsfragments, checkout_no_credentials, and
check_order_dockerfile_extras.
- Add scripts tests to CI: new SCRIPTS_FILES file group in selective
checks, run-scripts-tests output, and tests-scripts job in
basic-tests.yml.
- Document scripts as a workspace distribution in CLAUDE.md.
* Add pytest as dev dependency for scripts distribution
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* Use devel-common instead of pytest for scripts dev dependencies
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* Fix xdist test collection order for newsfragment tests
Sort the VALID_CHANGE_TYPES set when passing to parametrize to ensure
deterministic test ordering across xdist workers.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* Update scripts/ci/prek/changelog_duplicates.py
Co-authored-by: Dev-iL <6509619+Dev-iL@users.noreply.github.com>
* Refactor scripts tests: convert setup methods to fixtures and extract constants
---------
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Dev-iL <6509619+Dev-iL@users.noreply.github.com>
2026-03-14 17:22:46 +01:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-core/src/airflow/configuration\.py$|
|
|
|
|
|
^airflow-core/tests/.*$|
|
|
|
|
|
^providers/.*/src/airflow/providers/|
|
|
|
|
|
^scripts/in_container/verify_providers\.py$|
|
|
|
|
|
^providers/.*/tests/.*$|
|
|
|
|
|
^scripts/tests/.*$|
|
|
|
|
|
^devel-common/
|
2022-08-24 00:18:18 +03:00
|
|
|
pass_filenames: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-provide-create-sessions-imports
|
2020-02-02 01:14:14 -08:00
|
|
|
language: pygrep
|
2024-11-27 09:15:32 +08:00
|
|
|
name: Check session util imports
|
|
|
|
|
description: NEW_SESSION, provide_session, and create_session should be imported from airflow.utils.session to avoid import cycles.
|
|
|
|
|
entry: "from airflow\\.utils\\.db import.* (NEW_SESSION|provide_session|create_session)"
|
2020-02-02 01:14:14 -08:00
|
|
|
files: \.py$
|
|
|
|
|
pass_filenames: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-incorrect-use-of-LoggingMixin
|
2020-02-02 20:48:03 +01:00
|
|
|
language: pygrep
|
|
|
|
|
name: Make sure LoggingMixin is not used alone
|
|
|
|
|
entry: "LoggingMixin\\(\\)"
|
|
|
|
|
files: \.py$
|
|
|
|
|
pass_filenames: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-start-date-not-used-in-defaults
|
2020-07-25 00:16:25 +01:00
|
|
|
language: pygrep
|
2024-09-20 13:29:48 -07:00
|
|
|
name: start_date not in default_args
|
2020-07-25 19:57:32 +01:00
|
|
|
entry: "default_args\\s*=\\s*{\\s*(\"|')start_date(\"|')|(\"|')start_date(\"|'):"
|
2021-05-06 12:36:32 +10:00
|
|
|
files: \.*example_dags.*\.py$
|
2020-07-25 00:16:25 +01:00
|
|
|
pass_filenames: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-apache-license-rat
|
2019-11-04 15:59:48 +01:00
|
|
|
name: Check if licenses are OK for Apache
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_license.py
|
2022-09-02 13:49:20 +02:00
|
|
|
language: python
|
2025-03-23 17:52:58 +01:00
|
|
|
files: ^LICENSE$
|
2019-11-04 15:59:48 +01:00
|
|
|
pass_filenames: false
|
Add a pre-commit script for checking code & metrics registry are synced (#63757)
This patch adds a script that checks the metrics in all provided files.
The script reads the metrics from the registry YAML file and checks if the code metrics exist. If they do, then it validates that the metric type in the code is the same as in the YAML.
In the code, there are a lot of dynamic metric names with variables in them. Some of them belong to legacy metrics but not all of them.
For the dynamic metric names, if there is a partial match until the 1st variable appears, then we consider the metric registered in the YAML. That's the best scenario without having to account for all possible names after the variable expansion.
It would be helpful to check if there are metrics in the YAML that don't appear in the code but it's not feasible when running the script against certain files and not every file in the project.
The next step after this PR, would be to remove the `DualStatsManager` entirely from the codebase. When that happens, the only change in this patch will be
```diff
- STATS_OBJECTS = {"Stats", "stats", "DualStatsManager"}
+ STATS_OBJECTS = {"Stats", "stats"}
```
For testing,
* I made a list for all the metrics that we should be catching with the script
* I asked Claude code to get me a list with all the metrics from the codebase
* I cross-referenced the lists
* Cross-referenced the lists with the metrics in the YAML file
* Cross-referenced all the discrepancies with the violations in the script output
2026-03-19 15:42:25 +02:00
|
|
|
- id: check-metrics-synced-with-registry
|
|
|
|
|
name: Check that metrics in the codebase are in sync with the metrics registry YAML file.
|
|
|
|
|
entry: ./scripts/ci/prek/check_metrics_synced_with_the_registry.py
|
|
|
|
|
language: python
|
|
|
|
|
files: \.py$
|
|
|
|
|
exclude: ^(tests/|.*/tests/)
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
additional_dependencies: ["PyYAML>=6.0", "rich>=13.6.0"]
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-boring-cyborg-configuration
|
2021-06-04 22:02:36 +02:00
|
|
|
name: Checks for Boring Cyborg configuration consistency
|
|
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/boring_cyborg.py
|
2021-06-04 22:02:36 +02:00
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: update-in-the-wild-to-be-sorted
|
2020-10-18 18:15:53 +01:00
|
|
|
name: Sort INTHEWILD.md alphabetically
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/sort_in_the_wild.py
|
2022-09-03 15:03:57 +02:00
|
|
|
language: python
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^\.pre-commit-config\.yaml$|^INTHEWILD\.md$
|
2022-09-03 15:03:57 +02:00
|
|
|
pass_filenames: false
|
2020-10-18 18:15:53 +01:00
|
|
|
require_serial: true
|
2023-01-30 02:37:50 +10:00
|
|
|
- id: update-installed-providers-to-be-sorted
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Sort and uniquify installed_providers.txt
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/sort_installed_providers.py
|
2023-01-30 02:37:50 +10:00
|
|
|
language: python
|
2024-02-26 20:28:27 +01:00
|
|
|
files: ^\.pre-commit-config\.yaml$|^.*_installed_providers\.txt$
|
2023-01-30 02:37:50 +10:00
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: update-spelling-wordlist-to-be-sorted
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Sort spelling_wordlist.txt
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/sort_spelling_wordlist.py
|
2022-09-04 11:15:07 +02:00
|
|
|
language: python
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^\.pre-commit-config\.yaml$|^docs/spelling_wordlist\.txt$
|
2021-03-01 02:26:13 +10:00
|
|
|
require_serial: true
|
2022-09-04 11:15:07 +02:00
|
|
|
pass_filenames: false
|
2023-04-13 00:03:58 +08:00
|
|
|
- id: shellcheck
|
2020-10-18 18:15:53 +01:00
|
|
|
name: Check Shell scripts syntax correctness
|
|
|
|
|
language: docker_image
|
2022-05-04 00:37:30 +02:00
|
|
|
entry: koalaman/shellcheck:v0.8.0 -x -a
|
2023-02-13 01:04:11 +10:00
|
|
|
files: \.(bash|sh)$|^hooks/build$|^hooks/push$
|
2022-04-03 20:52:48 +02:00
|
|
|
exclude: ^dev/breeze/autocomplete/.*$
|
2024-01-03 19:21:47 +01:00
|
|
|
- id: check-integrations-list-consistent
|
|
|
|
|
name: Sync integrations list with docs
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_integrations_list.py
|
2024-01-03 19:21:47 +01:00
|
|
|
language: python
|
2025-04-09 01:26:02 +10:00
|
|
|
files: ^scripts/ci/docker-compose/integration-.*\.yml$|^contributing-docs/testing/integration_tests\.rst$
|
2024-01-03 19:21:47 +01:00
|
|
|
require_serial: true
|
|
|
|
|
pass_filenames: false
|
2026-02-19 10:01:41 -05:00
|
|
|
- id: sync-translation-namespaces
|
|
|
|
|
name: Sync translation namespace file list
|
|
|
|
|
entry: ./scripts/ci/prek/sync_translation_namespaces.py
|
|
|
|
|
language: python
|
|
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-core/src/airflow/ui/public/i18n/locales/en/.*\.json$|
|
|
|
|
|
^\.github/skills/airflow-translations/SKILL\.md$
|
|
|
|
|
pass_filenames: false
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
- id: update-pyproject-toml
|
|
|
|
|
name: Update Airflow's meta-package pyproject.toml
|
2025-01-20 17:17:05 +01:00
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/update_airflow_pyproject_toml.py
|
Add Python 3.13 support for Airflow. (#46891)
* Breeze changes for Python 3.13
* Pyproject.toml provider changes for Python 3.13
* README.rst changes for providers for Python 3.13
* Add Python 3.13 support for Airflow.
Added Python 3.13 support across the codebase, including:
* CI/CD workflow updates to handle Python 3.13.
* Dockerfile and environment variable adjustments for compatibility.
* Documentation updates to list Python 3.13 as supported.
* Dependency and code changes for compatibility with new/changed
libraries (e.g., numpy 2.x, greenlet, pendulum).
* Test and plugin manager adjustments for Python 3.13-specific behaviors.
* Removal or skipping of tests and features not compatible with Python 3.13.
* Improved error handling and logging for unsupported plugins/providers
on Python 3.13.
* Updated all provider README.rst files to:
* List Python 3.13 as a supported version.
* Adjust dependency requirements for Python 3.13 (e.g., pandas,
pyarrow, ray, etc.).
* Add or update notes about excluded or conditionally supported
dependencies for Python 3.13.
* Document any known incompatibilities or workarounds for specific
providers.
* Updated all provider pyproject.toml files and automation to:
* Add Python 3.13 to the supported classifiers.
* Adjust requires-python and dependency constraints for Python 3.13.
* Exclude Python 3.13 for providers that are not yet compatible (e.g.,
those depending on Flask AppBuilder).
* Add conditional dependencies for new versions of libraries required
by Python 3.13.
* Updated Breeze and related scripts to:
* Build and test with Python 3.13.
* Update documentation and release scripts to include Python 3.13.
* Ensure all dev tooling and test environments work with Python 3.13.
* Removed remaining FAB related code from core and removed tests for
older FAB provider versions (<1.3.0)
* some of the tests in core still depend on FAB - but we make them
optional - both for parsing (we need to skip some imports) and
execution - because they test a legacy behaviour of embedding
Flask plugins that is still supported in core.
* k8s tests now run also with SimpleAuthManager and SimpleAuthManager
is used automatically when runnin tests in Python 3.13
* migration tests had to be excluded fo Python 3.13 because we cannot
migrate down on Python 3.13 without FAB tables created. The tests
will be brought back when FAB is supported or when we release 3.1
and start working on 3.2, we should be able to migrate down to 3.1
* Weaviate tests have been updated to handle case where weaviate-client
is > 2.10.0 - because it started to use httpx instead of requests,
and Python 3.13 actually unblocks using older version of weaviate
(it was previously blocked indirectly via Flask dependencies and
grpcio).
* Added doc change in our rules on what is the "default" Python version
for airflow images - handling the case where we cannot use "newest"
Python version as default where not all "regular" image providers are
supported in the latest version
* Connection is now flushed in `configure_git_connection_for_dag_bundle`
just before `clear_db_connections` - in order to handle Python 3.13
case where FAB app is not initialized by other fixtures - performing
implicit flush along the way
* Cleaning db connections was added to test_dag_run and test_dags test
in fast_api as the test implicitly rely on having connections
cleared (no test git connections added) - this will make the tests
side-effect free.
* Conditionally skipping some serialization tests involving yandex
and ray on Python 3.13 until they support Python 3.13
2025-07-17 16:53:49 +02:00
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
^.*/pyproject\.toml$|
|
2025-08-17 09:00:14 +02:00
|
|
|
^scripts/ci/prek/update_airflow_pyproject_toml\.py$|
|
Add Python 3.13 support for Airflow. (#46891)
* Breeze changes for Python 3.13
* Pyproject.toml provider changes for Python 3.13
* README.rst changes for providers for Python 3.13
* Add Python 3.13 support for Airflow.
Added Python 3.13 support across the codebase, including:
* CI/CD workflow updates to handle Python 3.13.
* Dockerfile and environment variable adjustments for compatibility.
* Documentation updates to list Python 3.13 as supported.
* Dependency and code changes for compatibility with new/changed
libraries (e.g., numpy 2.x, greenlet, pendulum).
* Test and plugin manager adjustments for Python 3.13-specific behaviors.
* Removal or skipping of tests and features not compatible with Python 3.13.
* Improved error handling and logging for unsupported plugins/providers
on Python 3.13.
* Updated all provider README.rst files to:
* List Python 3.13 as a supported version.
* Adjust dependency requirements for Python 3.13 (e.g., pandas,
pyarrow, ray, etc.).
* Add or update notes about excluded or conditionally supported
dependencies for Python 3.13.
* Document any known incompatibilities or workarounds for specific
providers.
* Updated all provider pyproject.toml files and automation to:
* Add Python 3.13 to the supported classifiers.
* Adjust requires-python and dependency constraints for Python 3.13.
* Exclude Python 3.13 for providers that are not yet compatible (e.g.,
those depending on Flask AppBuilder).
* Add conditional dependencies for new versions of libraries required
by Python 3.13.
* Updated Breeze and related scripts to:
* Build and test with Python 3.13.
* Update documentation and release scripts to include Python 3.13.
* Ensure all dev tooling and test environments work with Python 3.13.
* Removed remaining FAB related code from core and removed tests for
older FAB provider versions (<1.3.0)
* some of the tests in core still depend on FAB - but we make them
optional - both for parsing (we need to skip some imports) and
execution - because they test a legacy behaviour of embedding
Flask plugins that is still supported in core.
* k8s tests now run also with SimpleAuthManager and SimpleAuthManager
is used automatically when runnin tests in Python 3.13
* migration tests had to be excluded fo Python 3.13 because we cannot
migrate down on Python 3.13 without FAB tables created. The tests
will be brought back when FAB is supported or when we release 3.1
and start working on 3.2, we should be able to migrate down to 3.1
* Weaviate tests have been updated to handle case where weaviate-client
is > 2.10.0 - because it started to use httpx instead of requests,
and Python 3.13 actually unblocks using older version of weaviate
(it was previously blocked indirectly via Flask dependencies and
grpcio).
* Added doc change in our rules on what is the "default" Python version
for airflow images - handling the case where we cannot use "newest"
Python version as default where not all "regular" image providers are
supported in the latest version
* Connection is now flushed in `configure_git_connection_for_dag_bundle`
just before `clear_db_connections` - in order to handle Python 3.13
case where FAB app is not initialized by other fixtures - performing
implicit flush along the way
* Cleaning db connections was added to test_dag_run and test_dags test
in fast_api as the test implicitly rely on having connections
cleared (no test git connections added) - this will make the tests
side-effect free.
* Conditionally skipping some serialization tests involving yandex
and ray on Python 3.13 until they support Python 3.13
2025-07-17 16:53:49 +02:00
|
|
|
^providers/.*/pyproject\.toml$|
|
|
|
|
|
^providers/.*/provider\.yaml$
|
2025-01-20 17:17:05 +01:00
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
Add Python 3.14 Support (#63520)
* Update python version exclusion to 3.15
* Add 3.14 metadata version classifiers and related constants
* Regenerate Breeze command help screenshots
* Assorted workarounds to fix breeze image building
- constraints are skipped entirely
- greenlet pin updated
* Exclude cassandra
* Exclude amazon
* Exclude google
* CI: Only add pydantic extra to Airflow 2 migration tests
Before this fix there were two separate issues in the migration-test setup for Python 3.14:
1. The migration workflow always passes --airflow-extras pydantic.
2. For Python 3.14, the minimum Airflow version is resolved to 3.2.0 by get_min_airflow_version_for_python.py, and apache-airflow[pydantic]==3.2.0 is not a valid thing to install.
So when constraints installation fails, the fallback path tries to install an invalid spec.
* Disable DB migration tests for python 3.14
* Enforce werkzeug 3.x for python 3.14
* Increase K8s executor test timeout for Python 3.14
Python 3.14 changed the default multiprocessing start method from 'fork' to 'forkserver' on Linux. The forkserver start method is slower because each new process must import modules from scratch rather than copying the parent's address space. This makes `multiprocessing.Manager()` initialization take longer, causing the test to exceed its 10s timeout.
* Adapt LocalExecutor tests for Python 3.14 forkserver default
Python 3.14 changed the default multiprocessing start method from
'fork' to 'forkserver' on Linux. Like 'spawn', 'forkserver' doesn't
share the parent's address space, so mock patches applied in the test
process are invisible to worker subprocesses.
- Skip tests that mock across process boundaries on non-fork methods
- Add test_executor_lazy_worker_spawning to verify that non-fork start
methods defer worker creation and skip gc.freeze
- Make test_multiple_team_executors_isolation and
test_global_executor_without_team_name assert the correct worker
count for each start method instead of assuming pre-spawning
- Remove skip from test_clean_stop_on_signal (works on all methods)
and increase timeout from 5s to 30s for forkserver overhead
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* Bump dependencies to versions supporting 3.14
* Fix PROD image build failing on Python 3.14 due to excluded providers
The PROD image build installed all provider wheels regardless of Python
version compatibility. Providers like google and amazon that exclude
Python 3.14 were still passed to pip, causing resolution failures (e.g.
ray has no cp314 wheel on PyPI).
Two fixes:
- get_distribution_specs.py now reads each wheel's Requires-Python
metadata and skips incompatible wheels instead of passing them to pip.
- The requires-python specifier generation used !=3.14 which per PEP 440
only excludes 3.14.0, not 3.14.3. Changed to !=3.14.* wildcard.
* Split core test types into 2 matrix groups to avoid OOM on Python 3.14
Non-DB core tests use xdist which runs all test types in a single pytest
process. With 2059 items across 4 workers, memory accumulates until the
OOM killer strikes at ~86% completion (exit code 137).
Split core test types into 2 groups (API/Always/CLI and
Core/Other/Serialization), similar to how provider tests already use
_split_list with NUMBER_OF_LOW_DEP_SLICES. Each group gets ~1000 items,
well under the ~1770 threshold where OOM occurs.
Update selective_checks test expectations to reflect the 2-group split.
* Gracefully handle an already removed password file in fixture
The old code had a check-then-act race (if `os.path.exists` → `os.remove`), which fails when the file doesn't exist at removal time. `contextlib.suppress(FileNotFoundError)` handles this atomically — if the file is missing (never created in this xdist worker, or removed between check and delete), it's silently ignored.
* Fix OOM and flaky tests in test_process_utils
Replace multiprocessing.Process with subprocess.Popen running minimal
inline scripts. multiprocessing.Process uses fork(), which duplicates
the entire xdist worker memory. At 95% test completion the worker has
accumulated hundreds of MBs; forking it triggers the OOM killer
(exit code 137) on Python 3.14.
subprocess.Popen starts a fresh lightweight process (~10MB) without
copying the parent's memory, avoiding the OOM entirely.
Also replace the racy ps -ax process counting in
TestKillChildProcessesByPids with psutil.pid_exists() checks on the
specific PID — the old approach was non-deterministic because unrelated
processes could start/stop between measurements.
* Add prek hook to validate python_version markers for excluded providers
When a provider declares excluded-python-versions in provider.yaml,
every dependency string referencing that provider in pyproject.toml
must carry a matching python_version marker. Missing markers cause
excluded providers to be silently installed as transitive dependencies
(e.g. aiobotocore pulling in amazon on Python 3.14).
The new check-excluded-provider-markers hook reads exclusions from
provider.yaml and validates all dependency strings in pyproject.toml
at commit time, preventing regressions like the one fixed in the
previous commit.
* Update `uv.lock`
---------
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-19 23:03:46 +02:00
|
|
|
- id: check-excluded-provider-markers
|
|
|
|
|
name: Check excluded-provider python_version markers in pyproject.toml
|
|
|
|
|
language: python
|
|
|
|
|
entry: ./scripts/ci/prek/check_excluded_provider_markers.py
|
|
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
^pyproject\.toml$|
|
|
|
|
|
^providers/.*/provider\.yaml$
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
|
|
|
|
additional_dependencies: ['packaging>=25', 'pyyaml', 'tomli>=2.0.1', 'rich>=13.6.0']
|
2024-01-12 01:29:31 +01:00
|
|
|
- id: update-reproducible-source-date-epoch
|
|
|
|
|
name: Update Source Date Epoch for reproducible builds
|
|
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/update_source_date_epoch.py
|
2025-04-09 01:26:02 +10:00
|
|
|
files: ^RELEASE_NOTES\.rst$|^chart/RELEASE_NOTES\.rst$
|
2024-01-12 01:29:31 +01:00
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-breeze-top-dependencies-limited
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Check top-level breeze deps
|
|
|
|
|
description: Breeze should have small number of top-level dependencies
|
2022-04-10 20:47:29 +02:00
|
|
|
language: python
|
|
|
|
|
entry: ./scripts/tools/check_if_limited_dependencies.py
|
|
|
|
|
files: ^dev/breeze/.*$
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-system-tests-present
|
2022-03-25 14:46:14 +01:00
|
|
|
name: Check if system tests have required segments of code
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_system_tests.py
|
2022-03-25 14:46:14 +01:00
|
|
|
language: python
|
2025-03-21 14:25:26 +01:00
|
|
|
files: ^.*/tests/system/.*/example_[^/]*\.py$
|
2022-03-25 14:46:14 +01:00
|
|
|
pass_filenames: true
|
2023-09-01 18:59:57 +05:30
|
|
|
- id: generate-pypi-readme
|
|
|
|
|
name: Generate PyPI README
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/generate_pypi_readme.py
|
2023-09-01 18:59:57 +05:30
|
|
|
language: python
|
|
|
|
|
files: ^README\.md$
|
|
|
|
|
pass_filenames: false
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: lint-markdown
|
2020-11-14 12:23:23 +01:00
|
|
|
name: Run markdownlint
|
2021-05-06 12:36:32 +10:00
|
|
|
description: Checks the style of Markdown files.
|
2020-11-14 12:23:23 +01:00
|
|
|
entry: markdownlint
|
|
|
|
|
language: node
|
|
|
|
|
types: [markdown]
|
|
|
|
|
files: \.(md|mdown|markdown)$
|
2023-12-11 16:14:36 +01:00
|
|
|
additional_dependencies: ['markdownlint-cli@0.38.0']
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: lint-json-schema
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Lint JSON Schema files
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/lint_json_schema.py
|
2020-11-28 12:12:54 +01:00
|
|
|
args:
|
2024-10-14 15:26:05 +01:00
|
|
|
- --spec-file
|
2025-08-17 09:00:14 +02:00
|
|
|
- scripts/ci/prek/draft7_schema.json
|
2020-11-28 12:12:54 +01:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
2021-01-11 23:10:44 +01:00
|
|
|
files: .*\.schema\.json$
|
2020-11-28 12:12:54 +01:00
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: lint-json-schema
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Lint NodePort Service
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/lint_json_schema.py
|
2020-12-04 16:51:59 +01:00
|
|
|
args:
|
|
|
|
|
- --spec-url
|
2021-11-24 23:29:46 +01:00
|
|
|
- https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.20.2-standalone/service-v1.json
|
2020-12-04 16:51:59 +01:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
2022-03-08 09:00:46 +10:00
|
|
|
files: ^scripts/ci/kubernetes/nodeport\.yaml$
|
2020-12-04 16:51:59 +01:00
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: lint-json-schema
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Lint Docker compose files
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/lint_json_schema.py
|
2020-12-04 16:51:59 +01:00
|
|
|
args:
|
|
|
|
|
- --spec-url
|
2024-05-16 20:34:07 +04:00
|
|
|
- https://raw.githubusercontent.com/compose-spec/compose-spec/master/schema/compose-spec.json
|
2020-12-04 16:51:59 +01:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^scripts/ci/docker-compose/.+\.ya?ml$|docker-compose\.ya?ml$
|
2023-02-15 09:34:31 -08:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
|
|
|
|
^scripts/ci/docker-compose/grafana/.|
|
2025-04-24 11:19:09 +01:00
|
|
|
^scripts/ci/docker-compose/gremlin/.|
|
2025-04-09 01:26:02 +10:00
|
|
|
^scripts/ci/docker-compose/.+-config\.ya?ml$
|
2020-12-04 16:51:59 +01:00
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-persist-credentials-disabled-in-github-workflows
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Check persistent creds in workflow files
|
|
|
|
|
description: Check that workflow files have persist-credentials disabled
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/checkout_no_credentials.py
|
2021-12-20 22:22:54 +01:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
2022-03-08 09:00:46 +10:00
|
|
|
files: ^\.github/workflows/.*\.yml$
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-docstring-param-types
|
2022-02-08 15:07:54 -08:00
|
|
|
name: Check that docstrings do not specify param types
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/docstring_param_type.py
|
2022-02-08 15:07:54 -08:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
files: \.py$
|
2025-02-10 15:14:04 +08:00
|
|
|
- id: check-zip-file-is-not-committed
|
|
|
|
|
name: Check no zip files are committed
|
|
|
|
|
description: Zip files are not allowed in the repository
|
|
|
|
|
language: fail
|
|
|
|
|
entry: |
|
|
|
|
|
Zip files are not allowed in the repository as they are hard to
|
|
|
|
|
track and have security implications. Please remove the zip file from the repository.
|
|
|
|
|
files: \.zip$
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: update-inlined-dockerfile-scripts
|
2022-03-27 19:19:02 +02:00
|
|
|
name: Inline Dockerfile and Dockerfile.ci scripts
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/inline_scripts_in_docker.py
|
2022-03-27 19:19:02 +02:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: false
|
2023-01-30 05:26:45 +10:00
|
|
|
files: ^Dockerfile$|^Dockerfile\.ci$|^scripts/docker/.*$
|
2022-03-27 19:19:02 +02:00
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-changelog-has-no-duplicates
|
2021-11-23 12:35:17 -07:00
|
|
|
name: Check changelogs for duplicate entries
|
|
|
|
|
language: python
|
2025-02-09 09:17:18 +01:00
|
|
|
files: changelog\.(rst|txt)$
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/changelog_duplicates.py
|
2021-11-23 12:35:17 -07:00
|
|
|
pass_filenames: true
|
2024-05-04 15:56:04 +05:30
|
|
|
- id: check-changelog-format
|
|
|
|
|
name: Check changelog format
|
|
|
|
|
language: python
|
2025-02-09 09:17:18 +01:00
|
|
|
files: changelog\.(rst|txt)$
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_changelog_format.py
|
2024-05-04 15:56:04 +05:30
|
|
|
pass_filenames: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-newsfragments-are-valid
|
2022-04-20 09:30:08 -06:00
|
|
|
name: Check newsfragments are valid
|
|
|
|
|
language: python
|
2025-04-09 01:26:02 +10:00
|
|
|
files: newsfragments/.*\.rst$
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/newsfragments.py
|
2022-04-20 09:30:08 -06:00
|
|
|
pass_filenames: true
|
2022-10-03 09:47:36 -05:00
|
|
|
# We sometimes won't have newsfragments in the repo, so always run it so `check-hooks-apply` passes
|
|
|
|
|
# This is fast, so not too much downside
|
|
|
|
|
always_run: true
|
2022-09-18 12:47:38 +02:00
|
|
|
- id: update-breeze-cmd-output
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Update breeze docs
|
|
|
|
|
description: Update output of breeze commands in Breeze documentation
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/breeze_cmd_line.py
|
2022-09-18 12:47:38 +02:00
|
|
|
language: python
|
2023-04-04 23:00:08 +02:00
|
|
|
files: >
|
|
|
|
|
(?x)
|
2024-01-21 21:41:54 +01:00
|
|
|
^dev/breeze/.*$|
|
2023-04-04 23:00:08 +02:00
|
|
|
^\.pre-commit-config\.yaml$|
|
2025-08-17 09:00:14 +02:00
|
|
|
^scripts/ci/prek/breeze_cmd_line\.py$|
|
2025-04-09 01:26:02 +10:00
|
|
|
^generated/provider_dependencies\.json$
|
2022-10-03 09:47:36 -05:00
|
|
|
require_serial: true
|
2022-09-18 12:47:38 +02:00
|
|
|
pass_filenames: false
|
2022-06-09 00:56:47 +02:00
|
|
|
- id: check-example-dags-urls
|
|
|
|
|
name: Check that example dags url include provider versions
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/update_example_dags_paths.py
|
2022-06-09 00:56:47 +02:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
2025-10-26 23:07:12 +01:00
|
|
|
files:
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-core/docs/.*example-dags\.rst$|
|
|
|
|
|
^airflow-core/docs/.*index\.rst$|
|
|
|
|
|
^docs/.*index\.rst$
|
2022-06-13 19:38:20 +02:00
|
|
|
always_run: true
|
2022-07-08 13:02:28 +02:00
|
|
|
- id: check-lazy-logging
|
|
|
|
|
name: Check that all logging methods are lazy
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_lazy_logging.py
|
2022-07-08 13:02:28 +02:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
files: \.py$
|
2023-11-10 19:12:26 +01:00
|
|
|
- id: bandit
|
|
|
|
|
name: bandit
|
|
|
|
|
description: "Bandit is a tool for finding common security issues in Python code"
|
|
|
|
|
entry: bandit
|
|
|
|
|
language: python
|
|
|
|
|
language_version: python3
|
2025-04-09 01:26:02 +10:00
|
|
|
types: [python]
|
2023-12-11 08:38:18 +01:00
|
|
|
additional_dependencies: ['bandit==1.7.6']
|
2023-11-10 19:12:26 +01:00
|
|
|
require_serial: true
|
2025-10-26 23:07:12 +01:00
|
|
|
files: ^airflow-core/src/airflow/.* # TODO Expand this to more than just airflow-core
|
2023-11-10 19:12:26 +01:00
|
|
|
exclude:
|
|
|
|
|
airflow/example_dags/.*
|
|
|
|
|
args:
|
|
|
|
|
- "--skip"
|
2024-03-12 01:36:55 +04:00
|
|
|
- "B101,B301,B324,B403,B404,B603"
|
2023-11-10 19:12:26 +01:00
|
|
|
- "--severity-level"
|
|
|
|
|
- "high" # TODO: remove this line when we fix all the issues
|
2026-03-04 21:51:02 +01:00
|
|
|
- id: check-k8s-schemas-published
|
|
|
|
|
name: Check K8s schemas are published on airflow.apache.org
|
|
|
|
|
entry: ./scripts/ci/prek/check_k8s_schemas_published.py
|
|
|
|
|
language: python
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
files: ^dev/breeze/src/airflow_breeze/global_constants\.py$
|
|
|
|
|
require_serial: true
|
2026-03-17 03:11:14 +01:00
|
|
|
# This is a fast regular hook that runs when any pyproject.toml changes
|
|
|
|
|
# It runs locally and usually will not result in modifying the lock unnecessarily
|
|
|
|
|
# Unless there is a conflict and uv will determine that the lock needs to be updated to resolve it
|
|
|
|
|
- id: update-uv-lock
|
|
|
|
|
name: Update uv.lock
|
2026-03-17 17:30:12 +02:00
|
|
|
entry: uv lock
|
2026-03-17 03:11:14 +01:00
|
|
|
language: system
|
|
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
(^|/)pyproject\.toml$|
|
|
|
|
|
^uv\.lock$
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
2025-08-17 09:00:14 +02:00
|
|
|
## ADD MOST PREK HOOK ABOVE THAT LINE
|
|
|
|
|
# The below prek hooks are those requiring CI image to be built
|
2026-03-17 03:11:14 +01:00
|
|
|
## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE
|
2023-04-13 00:03:58 +08:00
|
|
|
- id: mypy-dev
|
2025-10-18 21:48:14 +02:00
|
|
|
stages: ['pre-push']
|
2022-04-20 00:15:46 +02:00
|
|
|
name: Run mypy for dev
|
2022-04-23 09:12:11 +02:00
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/mypy.py
|
2023-11-27 16:24:23 +01:00
|
|
|
files: ^dev/.*\.py$|^scripts/.*\.py$
|
2022-04-20 00:15:46 +02:00
|
|
|
require_serial: true
|
2024-01-13 17:44:28 +01:00
|
|
|
- id: mypy-dev
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
stages: ['manual']
|
2024-01-13 17:44:28 +01:00
|
|
|
name: Run mypy for dev (manual)
|
|
|
|
|
language: python
|
2026-01-21 22:06:43 +01:00
|
|
|
entry: ./scripts/ci/prek/mypy_folder.py dev scripts
|
2024-01-13 17:44:28 +01:00
|
|
|
pass_filenames: false
|
|
|
|
|
files: ^.*\.py$
|
|
|
|
|
require_serial: true
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
- id: mypy-devel-common
|
2025-10-18 21:48:14 +02:00
|
|
|
stages: ['pre-push']
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
name: Run mypy for devel-common
|
2024-10-20 00:49:52 +01:00
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/mypy.py
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
files: ^devel-common/.*\.py$
|
2024-10-20 00:49:52 +01:00
|
|
|
require_serial: true
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
- id: mypy-devel-common
|
2024-10-20 00:49:52 +01:00
|
|
|
stages: ['manual']
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
name: Run mypy for devel-common (manual)
|
2024-10-20 00:49:52 +01:00
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/mypy_folder.py devel-common
|
2024-10-20 00:49:52 +01:00
|
|
|
pass_filenames: false
|
|
|
|
|
files: ^.*\.py$
|
|
|
|
|
require_serial: true
|
2024-09-27 14:24:42 +01:00
|
|
|
- id: check-template-fields-valid
|
|
|
|
|
name: Check templated fields mapped in operators/sensors
|
|
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_template_fields.py
|
2025-03-22 00:28:55 +05:30
|
|
|
files: ^(providers/.*/)?airflow-core/.*/(sensors|operators)/.*\.py$
|
2024-09-27 14:24:42 +01:00
|
|
|
require_serial: true
|
2026-03-12 14:48:39 +08:00
|
|
|
- id: check-execution-api-versions
|
|
|
|
|
name: Check execution API datamodel changes have corresponding version updates
|
|
|
|
|
entry: ./scripts/ci/prek/check_execution_api_versions.py
|
|
|
|
|
language: python
|
2026-03-21 15:43:31 +05:30
|
|
|
pass_filenames: true
|
2026-03-12 14:48:39 +08:00
|
|
|
files: ^airflow-core/src/airflow/api_fastapi/execution_api/(datamodels|versions)/.*\.py$
|
|
|
|
|
require_serial: true
|
2025-02-25 15:01:33 +00:00
|
|
|
- id: generate-tasksdk-datamodels
|
|
|
|
|
name: Generate Datamodels for TaskSDK client
|
|
|
|
|
language: python
|
2025-08-10 18:34:44 +02:00
|
|
|
entry: uv run -p 3.12 --no-progress --active --group codegen --project apache-airflow-task-sdk --directory task-sdk -s dev/generate_task_sdk_models.py
|
2025-02-25 15:01:33 +00:00
|
|
|
pass_filenames: false
|
2025-03-21 14:25:26 +01:00
|
|
|
files: ^airflow-core/src/airflow/api_fastapi/execution_api/.*\.py$
|
2025-02-25 15:01:33 +00:00
|
|
|
require_serial: true
|
2025-04-21 20:01:05 +05:30
|
|
|
- id: generate-airflowctl-datamodels
|
|
|
|
|
name: Generate Datamodels for AirflowCTL
|
|
|
|
|
language: python
|
|
|
|
|
entry: >
|
|
|
|
|
bash -c '
|
|
|
|
|
uv run -p 3.12 --no-dev --no-progress --active --group codegen --project apache-airflow-ctl --directory airflow-ctl/ datamodel-codegen &&
|
2025-05-17 13:35:20 +05:30
|
|
|
uv run -p 3.12 --no-dev --no-progress --active --group codegen --project apache-airflow-ctl --directory airflow-ctl/ datamodel-codegen --input="../airflow-core/src/airflow/api_fastapi/auth/managers/simple/openapi/v2-simple-auth-manager-generated.yaml" --output="src/airflowctl/api/datamodels/auth_generated.py"'
|
2025-04-21 20:01:05 +05:30
|
|
|
pass_filenames: false
|
2025-10-26 23:07:12 +01:00
|
|
|
files:
|
2025-07-25 11:56:13 +05:30
|
|
|
(?x)
|
2025-10-26 23:07:12 +01:00
|
|
|
^airflow-core/src/airflow/api_fastapi/core_api/datamodels/.*\.py$|
|
|
|
|
|
^airflow-core/src/airflow/api_fastapi/auth/managers/simple/(datamodels|routes|services|openapi)/.*\.py$
|
2025-08-30 00:29:18 +01:00
|
|
|
require_serial: true
|
2025-09-23 04:42:46 +01:00
|
|
|
- id: check-contextmanager-class-decorators
|
|
|
|
|
name: Check for problematic context manager class decorators
|
|
|
|
|
entry: ./scripts/ci/prek/check_contextmanager_class_decorators.py
|
|
|
|
|
language: python
|
|
|
|
|
files: .*test.*\.py$
|
|
|
|
|
pass_filenames: true
|
2026-03-17 03:11:14 +01:00
|
|
|
# This is a manual hook, run by `breeze ci upgrade` - upgrading all dependencies inside the
|
|
|
|
|
# Breeze CI image - which allows checking all dependencies for all providers.
|
|
|
|
|
# ALWAYS keep it at the end so that it can take into account all the other hook's changes.
|
|
|
|
|
- id: update-uv-lock
|
|
|
|
|
stages: ['manual']
|
|
|
|
|
name: Update uv.lock (manual)
|
|
|
|
|
entry: breeze run uv lock --upgrade
|
|
|
|
|
language: system
|
|
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
(^|/)pyproject\.toml$|
|
|
|
|
|
^uv\.lock$
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
