2019-08-14 20:53:28 -04:00
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
|
|
|
# or more contributor license agreements. See the NOTICE file
|
|
|
|
|
# distributed with this work for additional information
|
|
|
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
|
|
|
# to you under the Apache License, Version 2.0 (the
|
|
|
|
|
# "License"); you may not use this file except in compliance
|
|
|
|
|
# with the License. You may obtain a copy of the License at
|
|
|
|
|
#
|
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
#
|
|
|
|
|
# Unless required by applicable law or agreed to in writing,
|
|
|
|
|
# software distributed under the License is distributed on an
|
|
|
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
|
# KIND, either express or implied. See the License for the
|
|
|
|
|
# specific language governing permissions and limitations
|
|
|
|
|
# under the License.
|
|
|
|
|
---
|
2024-10-09 21:31:26 +01:00
|
|
|
default_stages: [pre-commit, pre-push]
|
2025-12-14 09:36:08 +01:00
|
|
|
minimum_prek_version: '0.2.22'
|
2019-08-22 10:13:56 -04:00
|
|
|
default_language_version:
|
|
|
|
|
python: python3
|
2025-08-29 13:40:37 +08:00
|
|
|
node: 22.19.0
|
2025-11-05 11:35:28 +01:00
|
|
|
golang: 1.24.0
|
2025-01-02 20:29:53 +10:00
|
|
|
exclude: ^.*/.*_vendor/
|
2019-08-14 20:53:28 -04:00
|
|
|
repos:
|
2020-12-16 14:14:20 +01:00
|
|
|
- repo: meta
|
|
|
|
|
hooks:
|
|
|
|
|
- id: identity
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Print checked files
|
|
|
|
|
description: Print input to the static check hooks for troubleshooting
|
2020-12-16 14:14:20 +01:00
|
|
|
- id: check-hooks-apply
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Check if all hooks apply to the repository
|
2025-11-05 23:44:06 +01:00
|
|
|
- repo: https://github.com/eclipse-csi/octopin
|
|
|
|
|
# We need this commit because if supports two spaces before comments (yamllint compatibility)
|
2025-12-14 22:06:44 +01:00
|
|
|
rev: 74fd075c7b326c771cd95ca86c59cbe65f0dda37
|
2025-11-05 23:44:06 +01:00
|
|
|
hooks:
|
|
|
|
|
- id: pin-versions
|
|
|
|
|
name: Pin versions of dependencies in CI workflows (manual)
|
|
|
|
|
stages: ['manual']
|
|
|
|
|
language: python
|
|
|
|
|
language_version: python311
|
2022-02-15 23:52:50 +01:00
|
|
|
- repo: https://github.com/thlorenz/doctoc.git
|
2025-12-14 09:36:08 +01:00
|
|
|
rev: 68f070c98b9a053eabfa7f8899d1f42b9919f98c # frozen: v2.2.0
|
2022-02-15 23:52:50 +01:00
|
|
|
hooks:
|
|
|
|
|
- id: doctoc
|
2023-02-13 01:04:11 +10:00
|
|
|
name: Add TOC for Markdown and RST files
|
2022-02-15 23:52:50 +01:00
|
|
|
files:
|
2025-05-12 19:03:22 +02:00
|
|
|
(?x)
|
|
|
|
|
^README\.md$|
|
|
|
|
|
^UPDATING.*\.md$|
|
|
|
|
|
^chart/UPDATING.*\.md$|
|
|
|
|
|
^dev/.*\.md$|
|
|
|
|
|
^dev/.*\.rst$|
|
|
|
|
|
^docs/README\.md$|
|
|
|
|
|
^\.github/.*\.md$|
|
|
|
|
|
^airflow-core/tests/system/README\.md$
|
2022-02-15 23:52:50 +01:00
|
|
|
args:
|
|
|
|
|
- "--maxlevel"
|
|
|
|
|
- "2"
|
2019-09-04 15:56:03 +02:00
|
|
|
- repo: https://github.com/Lucas-C/pre-commit-hooks
|
2025-06-06 17:00:25 +05:30
|
|
|
# replace hash with version once PR #103 merged comes in a release
|
2025-12-05 19:24:18 +01:00
|
|
|
rev: f5cfd5fdaf0211dfd1027d9d8442b764a232c7ad
|
2019-08-15 19:59:35 -04:00
|
|
|
hooks:
|
|
|
|
|
- id: insert-license
|
2019-10-26 13:31:51 +02:00
|
|
|
name: Add license for all SQL files
|
2019-08-15 19:59:35 -04:00
|
|
|
files: \.sql$
|
2022-10-23 17:17:26 +01:00
|
|
|
exclude: |
|
|
|
|
|
(?x)
|
2025-01-02 20:29:53 +10:00
|
|
|
^\.github/
|
2019-08-15 19:59:35 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "/*||*/"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-16 00:09:26 -04:00
|
|
|
- id: insert-license
|
2023-02-13 01:04:11 +10:00
|
|
|
name: Add license for all RST files
|
2025-01-02 20:29:53 +10:00
|
|
|
exclude: ^\.github/.*$|newsfragments/.*\.rst$
|
2019-08-16 00:09:26 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "||"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.rst
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-16 00:09:26 -04:00
|
|
|
files: \.rst$
|
2019-08-15 20:06:18 -04:00
|
|
|
- id: insert-license
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Add license for CSS/JS/JSX/PUML/TS/TSX
|
2023-02-13 01:04:11 +10:00
|
|
|
files: \.(css|jsx?|puml|tsx?)$
|
2025-09-03 09:58:16 +02:00
|
|
|
exclude: ^\.github/.*$|ui/openapi-gen/|www/openapi-gen/|.*/dist/.*
|
2019-08-15 20:06:18 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
2020-10-16 03:59:09 -04:00
|
|
|
- "/*!| *| */"
|
2019-08-15 20:06:18 -04:00
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-20 00:07:39 -04:00
|
|
|
- id: insert-license
|
2023-02-13 01:04:11 +10:00
|
|
|
name: Add license for all Shell files
|
2025-01-02 20:29:53 +10:00
|
|
|
exclude: ^\.github/.*$|^dev/breeze/autocomplete/.*$
|
2022-09-02 23:20:45 +02:00
|
|
|
files: \.bash$|\.sh$
|
2019-08-20 00:07:39 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "|#|"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
Generate Python client in reproducible way (#36763)
Client source code and package generation was done using the code
generated and committed to `airflow-client-python` and while the
repository with such code is useful to have, it's just a convenience
repo, because all sources are (and should be) generated from the
API specification which is present in the Airflow repository.
This also made the reproducible builds and package generation not really
possible, because we never knew if the source generated in the
`airflow-client-python` repository has been generated and not tampered
with.
While implementing it, it turned out that there were some issues in
the past that nade our client generation somewhat broken..
* In 2.7.0 python client, we added the same code twice
(See https://github.com/apache/airflow-client-python/pull/93) on
top of "airflow_client.client" package, we also added copy of the
API client generated in "airflow_client.airflow_client" - that was
likely due to bad bash scripts and tools that were used to generate
it and errors during generation the clients.
* We used to generate the code for "client" package and then moved
the "client" package to "airflow_client.client" package, while
manually modifying imports with `sed` (!?). That was likely due to
limitations in some old version of the client generator. However the
client generator we use now is capable of generating code directly in
the "airflow_client.client" package.
* We also manually (via pre-commit) added Apache Licence to the
generated files. Whieh was completely unnecessary, because ASF rules
do not require licence headers to be added to code automatically
generated from a code that already has ASF licence.
* We also generated source tarball packages from such generated code,
which was completely unnecessary - because sdist packages are already
fulfilling all the reqirements of such source pacakges - the code
in the packages is enough to build the package from the sources and
it does not contain any binary code, moreover the code is generated
out of the API specificiation, which means that anyone can take
the code and genearate the pacakged software from just sources in
sdist. Similarly as in case of provider packages, we do not need
to produce separate -source.tar.gz files.
This PR fixes all of it.
First of all the source that lands in the source repository
`airflow-client-python` and sdist/wheel packages are generated directly
from the openapi specification.
They are generated using breeze release_management command from airflow
source tagged with specific tag in the Airflow repo (including the
source of reproducible build date that is updated together with airflow
release notes. This means that any PMC member can regenerate packages
(binary identical) straight from the Airflow repository - without
going through "airflow-client-python" repository.
No source tarball is generated - it is not needed, sdist is enough.
The `test_python_client.py` has been also moved over to Airflow repo
and updated with handling case when expose_config is not enabled and
it is used to automatically test the API client after it has been
generated.
2024-01-14 18:17:20 +01:00
|
|
|
- id: insert-license
|
|
|
|
|
name: Add license for all toml files
|
2025-01-02 20:29:53 +10:00
|
|
|
exclude: ^\.github/.*$|^dev/breeze/autocomplete/.*$
|
Generate Python client in reproducible way (#36763)
Client source code and package generation was done using the code
generated and committed to `airflow-client-python` and while the
repository with such code is useful to have, it's just a convenience
repo, because all sources are (and should be) generated from the
API specification which is present in the Airflow repository.
This also made the reproducible builds and package generation not really
possible, because we never knew if the source generated in the
`airflow-client-python` repository has been generated and not tampered
with.
While implementing it, it turned out that there were some issues in
the past that nade our client generation somewhat broken..
* In 2.7.0 python client, we added the same code twice
(See https://github.com/apache/airflow-client-python/pull/93) on
top of "airflow_client.client" package, we also added copy of the
API client generated in "airflow_client.airflow_client" - that was
likely due to bad bash scripts and tools that were used to generate
it and errors during generation the clients.
* We used to generate the code for "client" package and then moved
the "client" package to "airflow_client.client" package, while
manually modifying imports with `sed` (!?). That was likely due to
limitations in some old version of the client generator. However the
client generator we use now is capable of generating code directly in
the "airflow_client.client" package.
* We also manually (via pre-commit) added Apache Licence to the
generated files. Whieh was completely unnecessary, because ASF rules
do not require licence headers to be added to code automatically
generated from a code that already has ASF licence.
* We also generated source tarball packages from such generated code,
which was completely unnecessary - because sdist packages are already
fulfilling all the reqirements of such source pacakges - the code
in the packages is enough to build the package from the sources and
it does not contain any binary code, moreover the code is generated
out of the API specificiation, which means that anyone can take
the code and genearate the pacakged software from just sources in
sdist. Similarly as in case of provider packages, we do not need
to produce separate -source.tar.gz files.
This PR fixes all of it.
First of all the source that lands in the source repository
`airflow-client-python` and sdist/wheel packages are generated directly
from the openapi specification.
They are generated using breeze release_management command from airflow
source tagged with specific tag in the Airflow repo (including the
source of reproducible build date that is updated together with airflow
release notes. This means that any PMC member can regenerate packages
(binary identical) straight from the Airflow repository - without
going through "airflow-client-python" repository.
No source tarball is generated - it is not needed, sdist is enough.
The `test_python_client.py` has been also moved over to Airflow repo
and updated with handling case when expose_config is not enabled and
it is used to automatically test the API client after it has been
generated.
2024-01-14 18:17:20 +01:00
|
|
|
files: \.toml$
|
|
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "|#|"
|
|
|
|
|
- --license-filepath
|
|
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
|
|
|
|
- --fuzzy-match-generates-todo
|
2019-09-17 13:16:32 +02:00
|
|
|
- id: insert-license
|
2021-05-06 12:36:32 +10:00
|
|
|
name: Add license for all Python files
|
2025-04-21 20:01:05 +05:30
|
|
|
exclude: ^\.github/.*$|^.*/_vendor/.*$|^airflow-ctl/.*/.*generated\.py$
|
2021-12-27 16:46:22 +01:00
|
|
|
files: \.py$|\.pyi$
|
2019-09-17 13:16:32 +02:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "|#|"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2019-09-17 13:16:32 +02:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-20 10:29:34 -04:00
|
|
|
- id: insert-license
|
2019-10-26 13:31:51 +02:00
|
|
|
name: Add license for all XML files
|
2025-01-02 20:29:53 +10:00
|
|
|
exclude: ^\.github/.*$
|
2020-08-24 22:45:28 +02:00
|
|
|
files: \.xml$
|
2019-08-20 10:29:34 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "<!--||-->"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2023-04-22 23:22:29 +02:00
|
|
|
- id: insert-license
|
|
|
|
|
name: Add license for all YAML files except Helm templates
|
2025-05-20 09:49:44 -04:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
2025-10-26 15:30:54 +01:00
|
|
|
^\.github/.*$|
|
|
|
|
|
^chart/templates/.*|
|
2025-05-20 09:49:44 -04:00
|
|
|
.*reproducible_build\.yaml$|
|
|
|
|
|
^.*/v2.*\.yaml$|
|
|
|
|
|
^.*/openapi/_private_ui.*\.yaml$|
|
2025-05-20 12:48:21 -04:00
|
|
|
^.*/pnpm-lock\.yaml$|
|
2025-05-20 09:49:44 -04:00
|
|
|
.*-generated\.yaml$
|
2019-08-22 10:13:56 -04:00
|
|
|
types: [yaml]
|
2023-02-13 01:04:11 +10:00
|
|
|
files: \.ya?ml$
|
2019-08-22 10:13:56 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "|#|"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-21 23:27:54 -04:00
|
|
|
- id: insert-license
|
2023-02-13 01:04:11 +10:00
|
|
|
name: Add license for all Markdown files
|
2019-08-21 23:27:54 -04:00
|
|
|
files: \.md$
|
2025-01-02 20:29:53 +10:00
|
|
|
exclude: PROVIDER_CHANGES.*\.md$
|
2019-08-21 23:27:54 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "<!--|| -->"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: insert-license
|
|
|
|
|
name: Add license for all other files
|
2025-01-02 20:29:53 +10:00
|
|
|
exclude: ^\.github/.*$
|
2022-04-23 21:33:08 +02:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "|#|"
|
|
|
|
|
- --license-filepath
|
2023-08-19 01:54:31 -06:00
|
|
|
- scripts/ci/license-templates/LICENSE.txt
|
2022-04-23 21:33:08 +02:00
|
|
|
- --fuzzy-match-generates-todo
|
|
|
|
|
files: >
|
2025-04-09 01:26:02 +10:00
|
|
|
\.cfg$|\.conf$|\.ini$|\.ldif$|\.properties$|\.service$|\.tf$|Dockerfile.*$
|
2022-12-18 17:04:48 +01:00
|
|
|
- repo: local
|
|
|
|
|
hooks:
|
2024-10-23 14:59:50 +02:00
|
|
|
- id: check-min-python-version
|
|
|
|
|
name: Check minimum Python version
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_min_python_version.py
|
2024-10-23 14:59:50 +02:00
|
|
|
language: python
|
|
|
|
|
require_serial: true
|
2025-12-16 17:15:36 +01:00
|
|
|
- id: check-version-consistency
|
|
|
|
|
name: Check version consistency
|
|
|
|
|
entry: ./scripts/ci/prek/check_version_consistency.py
|
|
|
|
|
language: python
|
|
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-core/src/airflow/__init__\.py$|
|
|
|
|
|
^airflow-core/pyproject\.toml$|
|
|
|
|
|
^task-sdk/src/airflow/sdk/__init__\.py$|
|
|
|
|
|
^pyproject\.toml$
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
2025-08-17 09:00:14 +02:00
|
|
|
- id: upgrade-important-versions
|
|
|
|
|
name: Upgrade important versions (manual)
|
|
|
|
|
entry: ./scripts/ci/prek/upgrade_important_versions.py
|
2024-02-28 00:05:41 +01:00
|
|
|
stages: ['manual']
|
|
|
|
|
language: python
|
2025-06-23 18:33:23 +02:00
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
^\.pre-commit-config\.yaml$|
|
|
|
|
|
^\.github/\.pre-commit-config\.yaml$|
|
2025-08-17 09:00:14 +02:00
|
|
|
^scripts/ci/prek/update_installers_and_prek\.py$
|
2024-02-28 00:05:41 +01:00
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
2025-09-08 22:32:18 +01:00
|
|
|
- repo: https://github.com/adamchainz/blacken-docs
|
2025-12-14 09:36:08 +01:00
|
|
|
rev: fda77690955e9b63c6687d8806bafd56a526e45f # frozen: 1.20.0
|
2021-05-27 07:09:25 +02:00
|
|
|
hooks:
|
|
|
|
|
- id: blacken-docs
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Run black on docs
|
2022-07-01 20:12:15 +02:00
|
|
|
args:
|
|
|
|
|
- --line-length=110
|
|
|
|
|
- --target-version=py310
|
2024-10-23 01:28:50 +01:00
|
|
|
- --target-version=py311
|
|
|
|
|
- --target-version=py312
|
2025-08-19 14:26:08 +02:00
|
|
|
- --target-version=py313
|
2023-10-31 18:51:50 -04:00
|
|
|
alias: blacken-docs
|
2025-08-19 14:26:08 +02:00
|
|
|
additional_dependencies:
|
2025-09-19 16:38:35 +03:00
|
|
|
- 'black==25.9.0'
|
2019-08-16 10:35:34 -04:00
|
|
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
2025-08-11 23:23:42 +02:00
|
|
|
rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0
|
2019-08-16 10:35:34 -04:00
|
|
|
hooks:
|
|
|
|
|
- id: check-merge-conflict
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Check that merge conflicts are not being committed
|
2019-11-26 09:28:42 +01:00
|
|
|
- id: debug-statements
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Detect accidentally committed debug statements
|
2020-06-25 16:35:37 +01:00
|
|
|
- id: check-builtin-literals
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Require literal syntax when initializing builtins
|
2019-08-16 10:35:34 -04:00
|
|
|
- id: detect-private-key
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Detect if private key is added to the repository
|
2025-04-09 01:26:02 +10:00
|
|
|
exclude: ^providers/ssh/docs/connections/ssh\.rst$
|
2019-08-16 10:35:34 -04:00
|
|
|
- id: end-of-file-fixer
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Make sure that there is an empty line at the end
|
2025-06-15 14:13:19 +02:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-core/docs/img/.*\.dot|
|
|
|
|
|
^airflow-core/docs/img/.*\.sha256|
|
|
|
|
|
.*/dist/.*|
|
|
|
|
|
LICENSES-ui\.txt$|
|
|
|
|
|
.*/openapi-gen/.*
|
2019-08-16 10:35:34 -04:00
|
|
|
- id: mixed-line-ending
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Detect if mixed line ending is used (\r vs. \r\n)
|
2019-08-20 10:29:34 -04:00
|
|
|
- id: check-executables-have-shebangs
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Check that executables have shebang
|
2019-08-20 10:29:34 -04:00
|
|
|
- id: check-xml
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Check XML files with xmllint
|
2025-04-24 11:19:09 +01:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
|
|
|
|
^scripts/ci/docker-compose/gremlin/.
|
2019-12-28 19:34:35 +01:00
|
|
|
- id: trailing-whitespace
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Remove trailing whitespace at end of line
|
2025-06-15 14:13:19 +02:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-core/docs/img/.*\.dot$|
|
|
|
|
|
^dev/breeze/doc/images/output.*$|
|
2025-06-29 16:24:54 +02:00
|
|
|
^.*/openapi-gen/.*$|
|
|
|
|
|
^airflow-ctl/docs/images/.*\.svg$
|
2019-09-21 20:47:02 +01:00
|
|
|
- repo: https://github.com/pre-commit/pygrep-hooks
|
2025-08-11 23:23:42 +02:00
|
|
|
rev: 3a6eb0fadf60b3cccfd80bad9dbb6fae7e47b316 # frozen: v1.10.0
|
2019-09-21 20:47:02 +01:00
|
|
|
hooks:
|
|
|
|
|
- id: rst-backticks
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Check if RST files use double backticks for code
|
2019-09-22 08:55:39 +01:00
|
|
|
- id: python-no-log-warn
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Check if there are no deprecate log warn
|
2020-06-14 02:37:30 +02:00
|
|
|
- repo: https://github.com/adrienverge/yamllint
|
2025-08-11 23:23:42 +02:00
|
|
|
rev: 79a6b2b1392eaf49cdd32ac4f14be1a809bbd8f7 # frozen: v1.37.1
|
2019-08-14 20:53:28 -04:00
|
|
|
hooks:
|
2019-08-22 10:13:56 -04:00
|
|
|
- id: yamllint
|
2021-05-06 12:36:32 +10:00
|
|
|
name: Check YAML files with yamllint
|
2020-10-21 12:24:32 +02:00
|
|
|
entry: yamllint -c yamllint-config.yml --strict
|
2019-08-22 10:13:56 -04:00
|
|
|
types: [yaml]
|
2025-03-01 12:43:03 +01:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
|
|
|
|
^.*airflow\.template\.yaml$|
|
|
|
|
|
^.*init_git_sync\.template\.yaml$|
|
|
|
|
|
^chart/(?:templates|files)/.*\.yaml$|
|
2025-05-19 13:13:41 -04:00
|
|
|
^helm-tests/tests/chart_utils/keda.sh_scaledobjects\.yaml$|
|
2025-04-15 15:30:16 +02:00
|
|
|
.*/v1.*\.yaml$|
|
2025-03-01 12:43:03 +01:00
|
|
|
^.*openapi.*\.yaml$|
|
|
|
|
|
^\.pre-commit-config\.yaml$|
|
2025-04-09 01:26:02 +10:00
|
|
|
^.*reproducible_build\.yaml$|
|
2025-12-07 23:53:40 +01:00
|
|
|
^.*pnpm-lock\.yaml$|
|
|
|
|
|
^.*-generated\.yaml$
|
2021-09-10 11:54:15 -06:00
|
|
|
- repo: https://github.com/ikamensh/flynt
|
2025-10-14 16:17:52 +02:00
|
|
|
rev: 97be693bf18bc2f050667dd282d243e2824b81e2 # frozen: 1.0.6
|
2021-08-18 01:56:38 +02:00
|
|
|
hooks:
|
|
|
|
|
- id: flynt
|
2022-04-23 21:33:08 +02:00
|
|
|
name: Run flynt string format converter for Python
|
2021-10-17 18:34:06 +02:00
|
|
|
args:
|
|
|
|
|
# If flynt detects too long text it ignores it. So we set a very large limit to make it easy
|
|
|
|
|
# to split the text by hand. Too long lines are detected by flake8 (below),
|
|
|
|
|
# so the user is informed to take action.
|
|
|
|
|
- --line-length
|
|
|
|
|
- '99999'
|
2021-10-14 19:37:08 +01:00
|
|
|
- repo: https://github.com/codespell-project/codespell
|
2025-08-11 23:23:42 +02:00
|
|
|
rev: 63c8f8312b7559622c0d82815639671ae42132ac # frozen: v2.4.1
|
2021-10-14 19:37:08 +01:00
|
|
|
hooks:
|
|
|
|
|
- id: codespell
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Run codespell
|
|
|
|
|
description: Run codespell to check for common misspellings in files
|
2021-10-30 15:44:20 +05:30
|
|
|
entry: bash -c 'echo "If you think that this failure is an error, consider adding the word(s)
|
|
|
|
|
to the codespell dictionary at docs/spelling_wordlist.txt.
|
|
|
|
|
The word(s) should be in lowercase." && exec codespell "$@"' --
|
2021-10-14 19:37:08 +01:00
|
|
|
language: python
|
|
|
|
|
types: [text]
|
2025-05-23 23:01:25 +02:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
|
|
|
|
material-icons\.css$|
|
|
|
|
|
^images/.*$|
|
|
|
|
|
^RELEASE_NOTES\.txt$|
|
|
|
|
|
^.*package-lock\.json$|
|
|
|
|
|
^.*/kinglear\.txt$|
|
|
|
|
|
^.*pnpm-lock\.yaml$|
|
|
|
|
|
.*/dist/.*|
|
2025-06-20 15:43:23 +09:00
|
|
|
^airflow-core/src/airflow/ui/public/i18n/locales/(?!en/).+/
|
2021-10-14 19:37:08 +01:00
|
|
|
args:
|
|
|
|
|
- --ignore-words=docs/spelling_wordlist.txt
|
2025-05-08 14:50:47 +02:00
|
|
|
- --skip=providers/.*/src/airflow/providers/*/*.rst,providers/*/docs/changelog.rst,docs/*/commits.rst,providers/*/docs/commits.rst,providers/*/*/docs/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md,*.svg
|
2022-01-24 12:43:09 +00:00
|
|
|
- --exclude-file=.codespellignorelines
|
2025-01-05 15:52:38 +00:00
|
|
|
- repo: https://github.com/woodruffw/zizmor-pre-commit
|
2025-12-23 13:26:38 +05:30
|
|
|
rev: ea18690d7f8e44203c9efd7bc6229447d02e3951 # frozen: v1.19.0
|
2025-01-05 15:52:38 +00:00
|
|
|
hooks:
|
|
|
|
|
- id: zizmor
|
|
|
|
|
name: Run zizmor to check for github workflow syntax errors
|
|
|
|
|
types: [yaml]
|
2025-04-09 01:26:02 +10:00
|
|
|
files: ^\.github/workflows/.*$|^\.github/actions/.*$
|
2025-01-05 15:52:38 +00:00
|
|
|
require_serial: true
|
|
|
|
|
entry: zizmor
|
2020-06-14 02:37:30 +02:00
|
|
|
- repo: local
|
2023-04-21 17:53:48 +02:00
|
|
|
# Note that this is the 2nd "local" repo group in the .pre-commit-config.yaml file. This is because
|
2025-08-17 09:00:14 +02:00
|
|
|
# we try to minimize the number of passes that must happen to apply some of the changes
|
|
|
|
|
# done by prek-hooks. Some of the prek hooks not only check for errors but also fix them. This means
|
|
|
|
|
# that output from an earlier prek hook becomes input to another prek hook. Splitting the local
|
|
|
|
|
# scripts of our and adding some other non-local prek hook in-between allows us to handle such
|
2023-04-21 17:53:48 +02:00
|
|
|
# changes quickly - especially when we want the early modifications from the first local group
|
2025-08-17 09:00:14 +02:00
|
|
|
# to be applied before the non-local prek hooks are run
|
2020-06-14 02:37:30 +02:00
|
|
|
hooks:
|
2025-07-26 10:22:14 +02:00
|
|
|
- id: check-shared-distributions-structure
|
|
|
|
|
name: Check shared distributions structure
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_shared_distributions_structure.py
|
2025-07-25 16:26:22 +02:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: false
|
2025-07-27 07:51:39 +10:00
|
|
|
files: ^shared/.*$
|
2025-07-26 10:22:14 +02:00
|
|
|
- id: check-shared-distributions-usage
|
|
|
|
|
name: Check shared distributions usage
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_shared_distributions_usage.py
|
2025-07-26 10:22:14 +02:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: false
|
2025-07-27 07:51:39 +10:00
|
|
|
files: ^shared/.*$|^.*/pyproject.toml$|^.*/_shared/.*$
|
2026-01-09 15:45:58 +05:30
|
|
|
- id: check-secrets-search-path-sync
|
|
|
|
|
name: Check sync between sdk and core
|
|
|
|
|
entry: ./scripts/ci/prek/check_secrets_search_path_sync.py
|
|
|
|
|
language: python
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
files: ^airflow-core/src/airflow/secrets/base_secrets\.py$|^task-sdk/src/airflow/sdk/execution_time/secrets/__init__\.py$
|
2024-01-07 20:02:08 +01:00
|
|
|
- id: ruff
|
|
|
|
|
name: Run 'ruff' for extremely fast Python linting
|
|
|
|
|
description: "Run 'ruff' for extremely fast Python linting"
|
|
|
|
|
entry: ruff check --force-exclude
|
|
|
|
|
language: python
|
|
|
|
|
types_or: [python, pyi]
|
|
|
|
|
args: [--fix]
|
|
|
|
|
require_serial: true
|
2026-01-08 22:22:21 +01:00
|
|
|
additional_dependencies: ['ruff==0.14.11']
|
2025-04-09 01:26:02 +10:00
|
|
|
exclude: ^airflow-core/tests/unit/dags/test_imports\.py$|^performance/tests/test_.*\.py$
|
2024-01-07 20:02:08 +01:00
|
|
|
- id: ruff-format
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Run 'ruff format'
|
2024-01-07 20:02:08 +01:00
|
|
|
description: "Run 'ruff format' for extremely fast Python formatting"
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/ruff_format.py
|
2024-01-07 20:02:08 +01:00
|
|
|
language: python
|
|
|
|
|
types_or: [python, pyi]
|
|
|
|
|
args: []
|
|
|
|
|
require_serial: true
|
2025-04-09 01:26:02 +10:00
|
|
|
exclude: ^airflow-core/tests/unit/dags/test_imports\.py$
|
2022-07-05 17:08:52 +02:00
|
|
|
- id: replace-bad-characters
|
|
|
|
|
name: Replace bad characters
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/replace_bad_characters.py
|
2022-07-05 17:08:52 +02:00
|
|
|
language: python
|
|
|
|
|
types: [file, text]
|
2025-09-03 09:58:16 +02:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
|
|
|
|
^clients/gen/go\.sh$|
|
|
|
|
|
^\.gitmodules$|
|
|
|
|
|
^airflow-core/src/airflow/ui/openapi-gen/|
|
|
|
|
|
^providers/edge3/src/airflow/providers/edge3/plugins/www/openapi-gen/|
|
|
|
|
|
.*/dist/.*|
|
|
|
|
|
\.go$|
|
|
|
|
|
/go\.(mod|sum)$
|
2019-08-14 20:53:28 -04:00
|
|
|
- id: lint-dockerfile
|
2023-02-13 01:04:11 +10:00
|
|
|
name: Lint Dockerfile
|
2022-09-02 17:21:41 +02:00
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/lint_dockerfile.py
|
2020-09-06 18:06:05 +02:00
|
|
|
files: Dockerfile.*$
|
2019-09-18 13:43:30 +02:00
|
|
|
pass_filenames: true
|
2021-02-27 18:38:43 +01:00
|
|
|
require_serial: true
|
2023-12-12 16:15:48 +01:00
|
|
|
- id: check-airflow-providers-bug-report-template
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Sort airflow-bug-report provider list
|
2023-12-12 16:15:48 +01:00
|
|
|
language: python
|
2025-05-13 12:34:56 +05:30
|
|
|
files: ^\.github/ISSUE_TEMPLATE/3-airflow_providers_bug_report\.yml$
|
2023-12-12 16:15:48 +01:00
|
|
|
require_serial: true
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_airflow_bug_report_template.py
|
2020-03-07 18:12:28 +01:00
|
|
|
- id: update-local-yml-file
|
|
|
|
|
name: Update mounts in the local yml file
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/local_yml_mounts.py
|
2022-05-28 17:19:14 +02:00
|
|
|
language: python
|
|
|
|
|
files: ^dev/breeze/src/airflow_breeze/utils/docker_command_utils\.py$|^scripts/ci/docker_compose/local\.yml$
|
2020-03-07 18:12:28 +01:00
|
|
|
pass_filenames: false
|
2021-09-21 21:22:41 +02:00
|
|
|
- id: check-extras-order
|
|
|
|
|
name: Check order of extras in Dockerfile
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_order_dockerfile_extras.py
|
2021-09-21 21:22:41 +02:00
|
|
|
language: python
|
|
|
|
|
files: ^Dockerfile$
|
|
|
|
|
pass_filenames: false
|
2023-12-05 21:15:16 +01:00
|
|
|
- id: generate-airflow-diagrams
|
|
|
|
|
name: Generate airflow diagrams
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/generate_airflow_diagrams.py
|
2023-12-05 21:15:16 +01:00
|
|
|
language: python
|
2025-05-12 19:03:22 +02:00
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-core/docs/.*/diagram_[^/]*\.py$|
|
2025-09-16 20:46:44 +02:00
|
|
|
^docs/images/.*\.py$|
|
|
|
|
|
^airflow-ctl/docs/images/diagrams/.*\.py$
|
2023-12-20 19:44:33 +01:00
|
|
|
pass_filenames: true
|
2025-06-23 19:25:50 +05:30
|
|
|
- id: prevent-deprecated-sqlalchemy-usage
|
|
|
|
|
name: Prevent deprecated sqlalchemy usage
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/prevent_deprecated_sqlalchemy_usage.py
|
2025-06-23 19:25:50 +05:30
|
|
|
language: python
|
|
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-ctl.*\.py$|
|
2025-09-03 03:49:19 +05:30
|
|
|
^airflow-core/src/airflow/models/.*\.py$|
|
2026-01-07 01:18:48 -08:00
|
|
|
^airflow-core/src/airflow/migrations/versions/0015_2_9_0_update_trigger_kwargs_type.py$|
|
2025-12-22 10:48:56 -08:00
|
|
|
^airflow-core/src/airflow/api_fastapi/execution_api/routes/task_instances.py$|
|
2025-12-15 04:02:23 +08:00
|
|
|
^airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_assets.py$|
|
2026-01-03 17:43:36 -08:00
|
|
|
^airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_backfills.py$|
|
|
|
|
|
^airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_connections.py$|
|
|
|
|
|
^airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_pools.py$|
|
2026-01-07 01:18:48 -08:00
|
|
|
^airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_task_instances.py$|
|
2025-12-20 04:53:29 +08:00
|
|
|
^airflow-core/tests/unit/models/test_serialized_dag.py$|
|
|
|
|
|
^airflow-core/tests/unit/models/test_pool.py$|
|
|
|
|
|
^airflow-core/tests/unit/models/test_trigger.py$|
|
|
|
|
|
^airflow-core/tests/unit/models/test_callback.py$|
|
|
|
|
|
^airflow-core/tests/unit/models/test_cleartasks.py$|
|
|
|
|
|
^airflow-core/tests/unit/models/test_xcom.py$|
|
|
|
|
|
^airflow-core/tests/unit/models/test_dagrun.py$|
|
2025-12-27 14:40:19 +08:00
|
|
|
^airflow-core/tests/unit/models/test_dag\.py$|
|
|
|
|
|
^airflow-core/tests/unit/models/test_dagcode\.py$|
|
|
|
|
|
^airflow-core/tests/unit/models/test_mappedoperator\.py$|
|
|
|
|
|
^airflow-core/tests/unit/models/test_taskinstance\.py$|
|
|
|
|
|
^airflow-core/tests/unit/models/test_variable\.py$|
|
2025-12-19 10:20:06 -08:00
|
|
|
^airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_sources.py$|
|
|
|
|
|
^airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_hitl.py$|
|
|
|
|
|
^airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_assets.py$|
|
|
|
|
|
^airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_structure.py$|
|
2025-12-18 23:42:51 +08:00
|
|
|
^airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_variables.py$|
|
|
|
|
|
^airflow-core/tests/unit/cli/commands/test_task_command.py$|
|
2026-01-08 23:09:18 +08:00
|
|
|
^airflow-core/tests/unit/cli/commands/test_team_command.py$|
|
|
|
|
|
^airflow-core/tests/unit/cli/commands/test_pool_command.py$|
|
|
|
|
|
^airflow-core/tests/unit/cli/commands/test_connection_command.py$|
|
|
|
|
|
^airflow-core/tests/unit/cli/commands/test_dag_command.py$|
|
|
|
|
|
^airflow-core/tests/unit/cli/commands/test_rotate_fernet_key_command.py$|
|
2025-12-18 23:42:51 +08:00
|
|
|
^airflow-core/tests/unit/dag_processing/bundles/test_dag_bundle_manager.py$|
|
2025-12-22 10:48:56 -08:00
|
|
|
^airflow-core/tests/unit/api_fastapi/execution_api/versions/head/test_dag_runs.py$|
|
2025-12-17 20:50:01 +09:00
|
|
|
^airflow-core/tests/unit/api_fastapi/execution_api/versions/head/test_task_instances.py$|
|
2025-12-22 10:48:56 -08:00
|
|
|
^airflow-core/tests/unit/api_fastapi/execution_api/versions/head/test_variables.py$|
|
|
|
|
|
^airflow-core/tests/unit/api_fastapi/execution_api/versions/head/test_xcoms.py$|
|
2025-12-19 10:20:06 -08:00
|
|
|
^airflow-core/tests/unit/models/test_deadline.py$|
|
2025-12-17 20:50:01 +09:00
|
|
|
^airflow-core/tests/unit/models/test_renderedtifields.py$|
|
|
|
|
|
^airflow-core/tests/unit/models/test_timestamp.py$|
|
|
|
|
|
^airflow-core/tests/unit/utils/test_cli_util.py$|
|
|
|
|
|
^airflow-core/tests/unit/timetables/test_assets_timetable.py$|
|
|
|
|
|
^airflow-core/tests/unit/assets/test_manager.py$|
|
2025-12-28 14:48:00 +05:30
|
|
|
^airflow-core/tests/unit/jobs/test_scheduler_job.py$|
|
2025-12-17 20:50:01 +09:00
|
|
|
^airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py$|
|
|
|
|
|
^airflow-core/tests/unit/ti_deps/deps/test_runnable_exec_date_dep.py$|
|
|
|
|
|
^airflow-core/tests/unit/models/test_dagwarning.py$|
|
|
|
|
|
^airflow-core/tests/integration/otel/test_otel.py$|
|
2025-12-16 18:54:51 +08:00
|
|
|
^airflow-core/tests/unit/utils/test_db_cleanup.py$|
|
2025-12-20 04:53:29 +08:00
|
|
|
^airflow-core/tests/unit/utils/test_state.py$|
|
|
|
|
|
^airflow-core/tests/unit/utils/test_log_handlers.py$|
|
2026-01-03 16:09:47 -08:00
|
|
|
^airflow-core/tests/unit/ti_deps/deps/test_not_previously_skipped_dep.py$|
|
2025-12-24 14:50:09 +05:30
|
|
|
^airflow-core/tests/unit/utils/test_types.py$|
|
|
|
|
|
^airflow-core/tests/unit/dag_processing/test_manager.py$|
|
|
|
|
|
^airflow-core/tests/unit/dag_processing/test_processor.py$|
|
2025-12-30 17:49:24 +08:00
|
|
|
^airflow-core/tests/unit/dag_processing/test_collection\.py$|
|
2025-12-16 13:55:08 +09:00
|
|
|
^dev/airflow_perf/scheduler_dag_execution_timing.py$|
|
2025-12-17 20:42:15 +09:00
|
|
|
^providers/celery/.*\.py$|
|
|
|
|
|
^providers/cncf/kubernetes/.*\.py$|
|
|
|
|
|
^providers/databricks/.*\.py$|
|
2025-12-18 00:35:36 +01:00
|
|
|
^providers/edge3/.*\.py$|
|
2025-12-17 20:42:15 +09:00
|
|
|
^providers/mysql/.*\.py$|
|
2025-12-15 22:15:01 +09:00
|
|
|
^providers/openlineage/.*\.py$|
|
2026-01-01 02:53:40 +08:00
|
|
|
^providers/standard/tests/unit/standard/operators/test_latest_only_operator\.py$|
|
|
|
|
|
^providers/standard/tests/unit/standard/operators/test_trigger_dagrun\.py$|
|
|
|
|
|
^providers/standard/tests/unit/standard/operators/test_weekday\.py$|
|
|
|
|
|
^providers/standard/tests/unit/standard/operators/test_datetime\.py$|
|
|
|
|
|
^providers/standard/tests/unit/standard/operators/test_branch_operator\.py$|
|
|
|
|
|
^providers/standard/tests/unit/standard/sensors/test_external_task_sensor\.py$|
|
|
|
|
|
^providers/standard/tests/unit/standard/utils/test_skipmixin\.py$|
|
2025-12-27 17:48:21 +08:00
|
|
|
^task_sdk.*\.py$|
|
|
|
|
|
^devel-common/src/tests_common/pytest_plugin\.py$|
|
|
|
|
|
^devel-common/src/tests_common/test_utils/.*\.py$
|
|
|
|
|
|
2025-06-23 19:25:50 +05:30
|
|
|
pass_filenames: true
|
2021-12-30 22:13:10 +01:00
|
|
|
- id: update-supported-versions
|
|
|
|
|
name: Updates supported versions in documentation
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/supported_versions.py
|
2021-12-30 22:13:10 +01:00
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
files: ^airflow-core/docs/installation/supported-versions\.rst$|^scripts/ci/prek/supported_versions\.py$|^README\.md$
|
2021-12-30 22:13:10 +01:00
|
|
|
pass_filenames: false
|
2022-04-08 18:20:54 +01:00
|
|
|
- id: check-revision-heads-map
|
|
|
|
|
name: Check that the REVISION_HEADS_MAP is up-to-date
|
|
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_revision_heads_map.py
|
2022-04-08 18:20:54 +01:00
|
|
|
pass_filenames: false
|
2023-08-29 12:30:26 -07:00
|
|
|
files: >
|
|
|
|
|
(?x)
|
2025-08-17 09:00:14 +02:00
|
|
|
^scripts/ci/prek/version_heads_map\.py$|
|
2025-10-24 17:23:32 +02:00
|
|
|
^airflow-core/src/airflow/migrations/versions/.*$|
|
|
|
|
|
^airflow-core/src/airflow/migrations/versions|
|
|
|
|
|
^airflow-core/src/airflow/utils/db\.py$
|
2021-09-18 10:41:57 +02:00
|
|
|
- id: update-version
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Update versions in docs
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/update_versions.py
|
2021-09-18 10:41:57 +02:00
|
|
|
language: python
|
2025-04-27 17:13:14 +02:00
|
|
|
files: ^docs|^airflow-core/src/airflow/__init__\.py$|.*/pyproject\.toml$
|
2021-09-18 10:41:57 +02:00
|
|
|
pass_filenames: false
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-pydevd-left-in-code
|
2019-11-26 09:28:42 +01:00
|
|
|
language: pygrep
|
|
|
|
|
name: Check for pydevd debug statements accidentally left
|
|
|
|
|
entry: "pydevd.*settrace\\("
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
files: \.py$
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-safe-filter-usage-in-html
|
2020-06-08 22:27:02 +01:00
|
|
|
language: pygrep
|
|
|
|
|
name: Don't use safe in templates
|
|
|
|
|
description: the Safe filter is error-prone, use Markup() in code instead
|
|
|
|
|
entry: "\\|\\s*safe"
|
|
|
|
|
files: \.html$
|
|
|
|
|
pass_filenames: true
|
2023-02-20 10:06:12 +09:00
|
|
|
- id: check-urlparse-usage-in-code
|
|
|
|
|
language: pygrep
|
|
|
|
|
name: Don't use urlparse in code
|
|
|
|
|
description: urlparse is not recommended, use urlsplit() in code instead
|
|
|
|
|
entry: "^\\s*from urllib\\.parse import ((\\|, )(urlparse\\|urlunparse))+$"
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
files: \.py$
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-for-inclusive-language
|
Don't use the term "whitelist" - language matters (#9174)
It's fairly common to say whitelisting and blacklisting to describe
desirable and undesirable things in cyber security. However just because
it is common doesn't mean it's right.
However, there's an issue with the terminology. It only makes sense if
you equate white with 'good, permitted, safe' and black with 'bad,
dangerous, forbidden'. There are some obvious problems with this.
You may not see why this matters. If you're not adversely affected by
racial stereotyping yourself, then please count yourself lucky. For some
of your friends and colleagues (and potential future colleagues), this
really is a change worth making.
From now on, we will use 'allow list' and 'deny list' in place of
'whitelist' and 'blacklist' wherever possible. Which, in fact, is
clearer and less ambiguous. So as well as being more inclusive of all,
this is a net benefit to our understandability.
(Words mostly borrowed from
<https://www.ncsc.gov.uk/blog-post/terminology-its-not-black-and-white>)
Co-authored-by: Jarek Potiuk <jarek@potiuk.com>
2020-06-08 10:01:46 +01:00
|
|
|
language: pygrep
|
|
|
|
|
name: Check for language that we do not accept as community
|
2022-05-09 16:52:29 -05:00
|
|
|
description: Please use more appropriate words for community documentation.
|
|
|
|
|
entry: >
|
2023-02-22 09:45:03 -05:00
|
|
|
(?ix)
|
2022-05-09 16:52:29 -05:00
|
|
|
(black|white)[_-]?list|
|
|
|
|
|
\bshe\b|
|
|
|
|
|
\bhe\b|
|
|
|
|
|
\bher\b|
|
|
|
|
|
\bhis\b|
|
|
|
|
|
\bmaster\b|
|
|
|
|
|
\bslave\b|
|
|
|
|
|
\bsanity\b|
|
|
|
|
|
\bdummy\b
|
Don't use the term "whitelist" - language matters (#9174)
It's fairly common to say whitelisting and blacklisting to describe
desirable and undesirable things in cyber security. However just because
it is common doesn't mean it's right.
However, there's an issue with the terminology. It only makes sense if
you equate white with 'good, permitted, safe' and black with 'bad,
dangerous, forbidden'. There are some obvious problems with this.
You may not see why this matters. If you're not adversely affected by
racial stereotyping yourself, then please count yourself lucky. For some
of your friends and colleagues (and potential future colleagues), this
really is a change worth making.
From now on, we will use 'allow list' and 'deny list' in place of
'whitelist' and 'blacklist' wherever possible. Which, in fact, is
clearer and less ambiguous. So as well as being more inclusive of all,
this is a net benefit to our understandability.
(Words mostly borrowed from
<https://www.ncsc.gov.uk/blog-post/terminology-its-not-black-and-white>)
Co-authored-by: Jarek Potiuk <jarek@potiuk.com>
2020-06-08 10:01:46 +01:00
|
|
|
pass_filenames: true
|
|
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
2025-12-02 18:00:42 +01:00
|
|
|
^README\.md$|
|
|
|
|
|
^generated/PYPI_README\.md$|
|
2025-10-25 22:26:32 +02:00
|
|
|
^airflow-core/docs/.*commits\.rst$|
|
|
|
|
|
^airflow-core/newsfragments/41368\.significant\.rst$|
|
|
|
|
|
^airflow-core/newsfragments/41761.significant\.rst$|
|
|
|
|
|
^airflow-core/newsfragments/43349\.significant\.rst$|
|
|
|
|
|
^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/pnpm-lock\.yaml$|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
^airflow-core/src/airflow/cli/commands/local_commands/fastapi_api_command\.py$|
|
2025-03-21 14:25:26 +01:00
|
|
|
^airflow-core/src/airflow/config_templates/|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
^airflow-core/src/airflow/models/baseoperator\.py$|
|
|
|
|
|
^airflow-core/src/airflow/operators/__init__\.py$|
|
2025-10-25 22:26:32 +02:00
|
|
|
^airflow-core/src/airflow/serialization/serialized_objects\.py$|
|
|
|
|
|
^airflow-core/src/airflow/ui/openapi-gen/|
|
|
|
|
|
^airflow-core/src/airflow/ui/pnpm-lock\.yaml$|
|
|
|
|
|
^airflow-core/src/airflow/ui/public/i18n/locales/de/README\.md$|
|
|
|
|
|
^airflow-core/src/airflow/ui/src/i18n/config\.ts$|
|
|
|
|
|
^airflow-core/src/airflow/utils/db\.py$|
|
|
|
|
|
^airflow-core/src/airflow/utils/trigger_rule\.py$|
|
|
|
|
|
^airflow-core/tests/|
|
|
|
|
|
^.*changelog\.(rst|txt)$|
|
|
|
|
|
^.*CHANGELOG\.(rst|txt)$|
|
|
|
|
|
^chart/values.schema\.json$|
|
|
|
|
|
^.*commits\.(rst|txt)$|
|
|
|
|
|
^.*/conf_constants\.py$|
|
|
|
|
|
^.*/conf\.py$|
|
|
|
|
|
^contributing-docs/03_contributors_quick_start\.rst$|
|
|
|
|
|
^dev/|
|
|
|
|
|
^devel-common/src/docs/README\.rst$|
|
|
|
|
|
^devel-common/src/sphinx_exts/removemarktransform\.py|
|
|
|
|
|
^devel-common/src/tests_common/test_utils/db\.py|
|
|
|
|
|
.*/dist/.*|
|
|
|
|
|
^docs/apache-airflow-providers-amazon/secrets-backends/aws-ssm-parameter-store\.rst$|
|
|
|
|
|
git|
|
|
|
|
|
^helm-tests/tests/chart_utils/helm_template_generator\.py$|
|
|
|
|
|
package-lock\.json$|
|
|
|
|
|
^.*\.(png|gif|jp[e]?g|svg|tgz|lock)$|
|
|
|
|
|
^\.pre-commit-config\.yaml$|
|
|
|
|
|
^.*/provider_conf\.py$|
|
|
|
|
|
^providers/\.pre-commit-config\.yaml$|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
^providers/amazon/src/airflow/providers/amazon/aws/hooks/emr\.py$|
|
|
|
|
|
^providers/amazon/src/airflow/providers/amazon/aws/operators/emr\.py$|
|
|
|
|
|
^providers/apache/cassandra/src/airflow/providers/apache/cassandra/hooks/cassandra\.py$|
|
2025-10-25 22:26:32 +02:00
|
|
|
^providers/apache/hdfs/docs/connections\.rst$|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
^providers/apache/hive/src/airflow/providers/apache/hive/operators/hive_stats\.py$|
|
|
|
|
|
^providers/apache/hive/src/airflow/providers/apache/hive/transfers/vertica_to_hive\.py$|
|
2025-10-25 22:26:32 +02:00
|
|
|
^providers/apache/kafka/docs/connections/kafka\.rst$|
|
|
|
|
|
^providers/apache/spark/docs/decorators/pyspark\.rst$|
|
2025-01-27 17:20:24 +08:00
|
|
|
^providers/apache/spark/src/airflow/providers/apache/spark/decorators/|
|
|
|
|
|
^providers/apache/spark/src/airflow/providers/apache/spark/hooks/|
|
|
|
|
|
^providers/apache/spark/src/airflow/providers/apache/spark/operators/|
|
2025-10-25 22:26:32 +02:00
|
|
|
^providers/cncf/kubernetes/docs/operators\.rst$|
|
|
|
|
|
^providers/common/sql/tests/provider_tests/common/sql/operators/test_sql_execute\.py$|
|
|
|
|
|
^providers/edge3/src/airflow/providers/edge3/plugins/www/pnpm-lock.yaml$|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
^providers/exasol/src/airflow/providers/exasol/hooks/exasol\.py$|
|
2025-10-25 22:26:32 +02:00
|
|
|
^providers/fab/docs/auth-manager/webserver-authentication\.rst$|
|
2025-02-01 04:19:00 +08:00
|
|
|
^providers/fab/src/airflow/providers/fab/auth_manager/security_manager/|
|
2025-03-11 09:40:26 -04:00
|
|
|
^providers/fab/src/airflow/providers/fab/www/static/|
|
|
|
|
|
^providers/fab/src/airflow/providers/fab/www/templates/|
|
2025-10-25 22:26:32 +02:00
|
|
|
^providers/google/docs/operators/cloud/kubernetes_engine\.rst$|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
^providers/google/src/airflow/providers/google/cloud/hooks/bigquery\.py$|
|
|
|
|
|
^providers/google/src/airflow/providers/google/cloud/operators/cloud_build\.py$|
|
|
|
|
|
^providers/google/src/airflow/providers/google/cloud/operators/dataproc\.py$|
|
|
|
|
|
^providers/google/src/airflow/providers/google/cloud/operators/mlengine\.py$|
|
2026-01-06 18:48:17 +08:00
|
|
|
^providers/keycloak/src/airflow/providers/keycloak/cli/definition.py|
|
2025-10-25 22:26:32 +02:00
|
|
|
^providers/microsoft/azure/docs/connections/azure_cosmos\.rst$|
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
^providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/cosmos\.py$|
|
|
|
|
|
^providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/hooks/winrm\.py$|
|
|
|
|
|
^providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/operators/winrm\.py$|
|
|
|
|
|
^providers/opsgenie/src/airflow/providers/opsgenie/hooks/opsgenie\.py$|
|
|
|
|
|
^providers/redis/src/airflow/providers/redis/provider\.yaml$|
|
2025-01-25 23:27:19 +01:00
|
|
|
^providers/.*/tests/|
|
2025-10-25 15:22:30 +02:00
|
|
|
.rat-excludes|
|
2025-10-25 22:26:32 +02:00
|
|
|
^.*RELEASE_NOTES\.rst$|
|
|
|
|
|
^scripts/ci/docker-compose/integration-keycloak\.yml$|
|
|
|
|
|
^scripts/ci/docker-compose/keycloak/keycloak-entrypoint\.sh$|
|
2025-11-13 00:26:32 +01:00
|
|
|
^scripts/ci/prek/upgrade_important_versions.py$|
|
2025-10-25 22:26:32 +02:00
|
|
|
^scripts/ci/prek/vendor_k8s_json_schema\.py$
|
2024-03-29 10:06:02 +08:00
|
|
|
- id: check-template-context-variable-in-sync
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Sync template context variable refs
|
2024-03-29 10:06:02 +08:00
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_template_context_variable_in_sync.py
|
2025-10-26 23:07:12 +01:00
|
|
|
files:
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-core/src/airflow/models/taskinstance\.py$|
|
|
|
|
|
^task-sdk/src/airflow/sdk/definitions/context\.py$|
|
|
|
|
|
^airflow-core/docs/templates-ref\.rst$
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-base-operator-usage
|
2019-11-26 23:19:45 +01:00
|
|
|
language: pygrep
|
2023-11-21 21:10:40 +04:00
|
|
|
name: Check BaseOperator core imports
|
|
|
|
|
description: Make sure BaseOperator is imported from airflow.models.baseoperator in core
|
|
|
|
|
entry: "from airflow\\.models import.* BaseOperator\\b"
|
|
|
|
|
files: \.py$
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
2025-03-21 14:25:26 +01:00
|
|
|
^airflow-core/src/airflow/decorators/.*$|
|
|
|
|
|
^airflow-core/src/airflow/hooks/.*$|
|
|
|
|
|
^airflow-core/src/airflow/operators/.*$|
|
|
|
|
|
^providers/.*$
|
2023-11-21 21:10:40 +04:00
|
|
|
- id: check-base-operator-usage
|
|
|
|
|
language: pygrep
|
|
|
|
|
name: Check BaseOperatorLink core imports
|
2025-03-07 13:54:43 +05:30
|
|
|
description: Make sure BaseOperatorLink is not imported from airflow.models in core
|
|
|
|
|
entry: "^\\s*from airflow\\.models\\.baseoperatorlink import BaseOperatorLink\\b"
|
2019-11-26 23:19:45 +01:00
|
|
|
files: \.py$
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
2025-03-21 14:25:26 +01:00
|
|
|
^airflow-core/src/airflow/decorators/.*$|
|
|
|
|
|
^airflow-core/src/airflow/hooks/.*$|
|
|
|
|
|
^airflow-core/src/airflow/operators/.*$|
|
2025-01-25 23:27:19 +01:00
|
|
|
^providers/.*/src/airflow/providers/.*$|
|
2025-03-21 14:25:26 +01:00
|
|
|
^providers/.*/src/airflow/providers/standard/sensors/.*$
|
2022-08-24 00:18:18 +03:00
|
|
|
- id: check-core-deprecation-classes
|
|
|
|
|
language: pygrep
|
2023-05-24 11:21:52 -07:00
|
|
|
name: Verify usage of Airflow deprecation classes in core
|
2022-08-24 00:18:18 +03:00
|
|
|
entry: category=DeprecationWarning|category=PendingDeprecationWarning
|
|
|
|
|
files: \.py$
|
2025-07-22 08:50:56 +10:00
|
|
|
exclude: ^airflow-core/src/airflow/configuration\.py$|^airflow-core/tests/.*$|^providers/.*/src/airflow/providers/|^scripts/in_container/verify_providers\.py$|^providers/.*/tests/.*$|^devel-common/
|
2022-08-24 00:18:18 +03:00
|
|
|
pass_filenames: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-provide-create-sessions-imports
|
2020-02-02 01:14:14 -08:00
|
|
|
language: pygrep
|
2024-11-27 09:15:32 +08:00
|
|
|
name: Check session util imports
|
|
|
|
|
description: NEW_SESSION, provide_session, and create_session should be imported from airflow.utils.session to avoid import cycles.
|
|
|
|
|
entry: "from airflow\\.utils\\.db import.* (NEW_SESSION|provide_session|create_session)"
|
2020-02-02 01:14:14 -08:00
|
|
|
files: \.py$
|
|
|
|
|
pass_filenames: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-incorrect-use-of-LoggingMixin
|
2020-02-02 20:48:03 +01:00
|
|
|
language: pygrep
|
|
|
|
|
name: Make sure LoggingMixin is not used alone
|
|
|
|
|
entry: "LoggingMixin\\(\\)"
|
|
|
|
|
files: \.py$
|
|
|
|
|
pass_filenames: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-start-date-not-used-in-defaults
|
2020-07-25 00:16:25 +01:00
|
|
|
language: pygrep
|
2024-09-20 13:29:48 -07:00
|
|
|
name: start_date not in default_args
|
2020-07-25 19:57:32 +01:00
|
|
|
entry: "default_args\\s*=\\s*{\\s*(\"|')start_date(\"|')|(\"|')start_date(\"|'):"
|
2021-05-06 12:36:32 +10:00
|
|
|
files: \.*example_dags.*\.py$
|
2020-07-25 00:16:25 +01:00
|
|
|
pass_filenames: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-apache-license-rat
|
2019-11-04 15:59:48 +01:00
|
|
|
name: Check if licenses are OK for Apache
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_license.py
|
2022-09-02 13:49:20 +02:00
|
|
|
language: python
|
2025-03-23 17:52:58 +01:00
|
|
|
files: ^LICENSE$
|
2019-11-04 15:59:48 +01:00
|
|
|
pass_filenames: false
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-boring-cyborg-configuration
|
2021-06-04 22:02:36 +02:00
|
|
|
name: Checks for Boring Cyborg configuration consistency
|
|
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/boring_cyborg.py
|
2021-06-04 22:02:36 +02:00
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: update-in-the-wild-to-be-sorted
|
2020-10-18 18:15:53 +01:00
|
|
|
name: Sort INTHEWILD.md alphabetically
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/sort_in_the_wild.py
|
2022-09-03 15:03:57 +02:00
|
|
|
language: python
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^\.pre-commit-config\.yaml$|^INTHEWILD\.md$
|
2022-09-03 15:03:57 +02:00
|
|
|
pass_filenames: false
|
2020-10-18 18:15:53 +01:00
|
|
|
require_serial: true
|
2023-01-30 02:37:50 +10:00
|
|
|
- id: update-installed-providers-to-be-sorted
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Sort and uniquify installed_providers.txt
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/sort_installed_providers.py
|
2023-01-30 02:37:50 +10:00
|
|
|
language: python
|
2024-02-26 20:28:27 +01:00
|
|
|
files: ^\.pre-commit-config\.yaml$|^.*_installed_providers\.txt$
|
2023-01-30 02:37:50 +10:00
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: update-spelling-wordlist-to-be-sorted
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Sort spelling_wordlist.txt
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/sort_spelling_wordlist.py
|
2022-09-04 11:15:07 +02:00
|
|
|
language: python
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^\.pre-commit-config\.yaml$|^docs/spelling_wordlist\.txt$
|
2021-03-01 02:26:13 +10:00
|
|
|
require_serial: true
|
2022-09-04 11:15:07 +02:00
|
|
|
pass_filenames: false
|
2023-04-13 00:03:58 +08:00
|
|
|
- id: shellcheck
|
2020-10-18 18:15:53 +01:00
|
|
|
name: Check Shell scripts syntax correctness
|
|
|
|
|
language: docker_image
|
2022-05-04 00:37:30 +02:00
|
|
|
entry: koalaman/shellcheck:v0.8.0 -x -a
|
2023-02-13 01:04:11 +10:00
|
|
|
files: \.(bash|sh)$|^hooks/build$|^hooks/push$
|
2022-04-03 20:52:48 +02:00
|
|
|
exclude: ^dev/breeze/autocomplete/.*$
|
2024-01-03 19:21:47 +01:00
|
|
|
- id: check-integrations-list-consistent
|
|
|
|
|
name: Sync integrations list with docs
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_integrations_list.py
|
2024-01-03 19:21:47 +01:00
|
|
|
language: python
|
2025-04-09 01:26:02 +10:00
|
|
|
files: ^scripts/ci/docker-compose/integration-.*\.yml$|^contributing-docs/testing/integration_tests\.rst$
|
2024-01-03 19:21:47 +01:00
|
|
|
require_serial: true
|
|
|
|
|
pass_filenames: false
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
- id: update-pyproject-toml
|
|
|
|
|
name: Update Airflow's meta-package pyproject.toml
|
2025-01-20 17:17:05 +01:00
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/update_airflow_pyproject_toml.py
|
Add Python 3.13 support for Airflow. (#46891)
* Breeze changes for Python 3.13
* Pyproject.toml provider changes for Python 3.13
* README.rst changes for providers for Python 3.13
* Add Python 3.13 support for Airflow.
Added Python 3.13 support across the codebase, including:
* CI/CD workflow updates to handle Python 3.13.
* Dockerfile and environment variable adjustments for compatibility.
* Documentation updates to list Python 3.13 as supported.
* Dependency and code changes for compatibility with new/changed
libraries (e.g., numpy 2.x, greenlet, pendulum).
* Test and plugin manager adjustments for Python 3.13-specific behaviors.
* Removal or skipping of tests and features not compatible with Python 3.13.
* Improved error handling and logging for unsupported plugins/providers
on Python 3.13.
* Updated all provider README.rst files to:
* List Python 3.13 as a supported version.
* Adjust dependency requirements for Python 3.13 (e.g., pandas,
pyarrow, ray, etc.).
* Add or update notes about excluded or conditionally supported
dependencies for Python 3.13.
* Document any known incompatibilities or workarounds for specific
providers.
* Updated all provider pyproject.toml files and automation to:
* Add Python 3.13 to the supported classifiers.
* Adjust requires-python and dependency constraints for Python 3.13.
* Exclude Python 3.13 for providers that are not yet compatible (e.g.,
those depending on Flask AppBuilder).
* Add conditional dependencies for new versions of libraries required
by Python 3.13.
* Updated Breeze and related scripts to:
* Build and test with Python 3.13.
* Update documentation and release scripts to include Python 3.13.
* Ensure all dev tooling and test environments work with Python 3.13.
* Removed remaining FAB related code from core and removed tests for
older FAB provider versions (<1.3.0)
* some of the tests in core still depend on FAB - but we make them
optional - both for parsing (we need to skip some imports) and
execution - because they test a legacy behaviour of embedding
Flask plugins that is still supported in core.
* k8s tests now run also with SimpleAuthManager and SimpleAuthManager
is used automatically when runnin tests in Python 3.13
* migration tests had to be excluded fo Python 3.13 because we cannot
migrate down on Python 3.13 without FAB tables created. The tests
will be brought back when FAB is supported or when we release 3.1
and start working on 3.2, we should be able to migrate down to 3.1
* Weaviate tests have been updated to handle case where weaviate-client
is > 2.10.0 - because it started to use httpx instead of requests,
and Python 3.13 actually unblocks using older version of weaviate
(it was previously blocked indirectly via Flask dependencies and
grpcio).
* Added doc change in our rules on what is the "default" Python version
for airflow images - handling the case where we cannot use "newest"
Python version as default where not all "regular" image providers are
supported in the latest version
* Connection is now flushed in `configure_git_connection_for_dag_bundle`
just before `clear_db_connections` - in order to handle Python 3.13
case where FAB app is not initialized by other fixtures - performing
implicit flush along the way
* Cleaning db connections was added to test_dag_run and test_dags test
in fast_api as the test implicitly rely on having connections
cleared (no test git connections added) - this will make the tests
side-effect free.
* Conditionally skipping some serialization tests involving yandex
and ray on Python 3.13 until they support Python 3.13
2025-07-17 16:53:49 +02:00
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
^.*/pyproject\.toml$|
|
2025-08-17 09:00:14 +02:00
|
|
|
^scripts/ci/prek/update_airflow_pyproject_toml\.py$|
|
Add Python 3.13 support for Airflow. (#46891)
* Breeze changes for Python 3.13
* Pyproject.toml provider changes for Python 3.13
* README.rst changes for providers for Python 3.13
* Add Python 3.13 support for Airflow.
Added Python 3.13 support across the codebase, including:
* CI/CD workflow updates to handle Python 3.13.
* Dockerfile and environment variable adjustments for compatibility.
* Documentation updates to list Python 3.13 as supported.
* Dependency and code changes for compatibility with new/changed
libraries (e.g., numpy 2.x, greenlet, pendulum).
* Test and plugin manager adjustments for Python 3.13-specific behaviors.
* Removal or skipping of tests and features not compatible with Python 3.13.
* Improved error handling and logging for unsupported plugins/providers
on Python 3.13.
* Updated all provider README.rst files to:
* List Python 3.13 as a supported version.
* Adjust dependency requirements for Python 3.13 (e.g., pandas,
pyarrow, ray, etc.).
* Add or update notes about excluded or conditionally supported
dependencies for Python 3.13.
* Document any known incompatibilities or workarounds for specific
providers.
* Updated all provider pyproject.toml files and automation to:
* Add Python 3.13 to the supported classifiers.
* Adjust requires-python and dependency constraints for Python 3.13.
* Exclude Python 3.13 for providers that are not yet compatible (e.g.,
those depending on Flask AppBuilder).
* Add conditional dependencies for new versions of libraries required
by Python 3.13.
* Updated Breeze and related scripts to:
* Build and test with Python 3.13.
* Update documentation and release scripts to include Python 3.13.
* Ensure all dev tooling and test environments work with Python 3.13.
* Removed remaining FAB related code from core and removed tests for
older FAB provider versions (<1.3.0)
* some of the tests in core still depend on FAB - but we make them
optional - both for parsing (we need to skip some imports) and
execution - because they test a legacy behaviour of embedding
Flask plugins that is still supported in core.
* k8s tests now run also with SimpleAuthManager and SimpleAuthManager
is used automatically when runnin tests in Python 3.13
* migration tests had to be excluded fo Python 3.13 because we cannot
migrate down on Python 3.13 without FAB tables created. The tests
will be brought back when FAB is supported or when we release 3.1
and start working on 3.2, we should be able to migrate down to 3.1
* Weaviate tests have been updated to handle case where weaviate-client
is > 2.10.0 - because it started to use httpx instead of requests,
and Python 3.13 actually unblocks using older version of weaviate
(it was previously blocked indirectly via Flask dependencies and
grpcio).
* Added doc change in our rules on what is the "default" Python version
for airflow images - handling the case where we cannot use "newest"
Python version as default where not all "regular" image providers are
supported in the latest version
* Connection is now flushed in `configure_git_connection_for_dag_bundle`
just before `clear_db_connections` - in order to handle Python 3.13
case where FAB app is not initialized by other fixtures - performing
implicit flush along the way
* Cleaning db connections was added to test_dag_run and test_dags test
in fast_api as the test implicitly rely on having connections
cleared (no test git connections added) - this will make the tests
side-effect free.
* Conditionally skipping some serialization tests involving yandex
and ray on Python 3.13 until they support Python 3.13
2025-07-17 16:53:49 +02:00
|
|
|
^providers/.*/pyproject\.toml$|
|
|
|
|
|
^providers/.*/provider\.yaml$
|
2025-01-20 17:17:05 +01:00
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
2024-01-12 01:29:31 +01:00
|
|
|
- id: update-reproducible-source-date-epoch
|
|
|
|
|
name: Update Source Date Epoch for reproducible builds
|
|
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/update_source_date_epoch.py
|
2025-04-09 01:26:02 +10:00
|
|
|
files: ^RELEASE_NOTES\.rst$|^chart/RELEASE_NOTES\.rst$
|
2024-01-12 01:29:31 +01:00
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-breeze-top-dependencies-limited
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Check top-level breeze deps
|
|
|
|
|
description: Breeze should have small number of top-level dependencies
|
2022-04-10 20:47:29 +02:00
|
|
|
language: python
|
|
|
|
|
entry: ./scripts/tools/check_if_limited_dependencies.py
|
|
|
|
|
files: ^dev/breeze/.*$
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-system-tests-present
|
2022-03-25 14:46:14 +01:00
|
|
|
name: Check if system tests have required segments of code
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_system_tests.py
|
2022-03-25 14:46:14 +01:00
|
|
|
language: python
|
2025-03-21 14:25:26 +01:00
|
|
|
files: ^.*/tests/system/.*/example_[^/]*\.py$
|
2022-03-25 14:46:14 +01:00
|
|
|
pass_filenames: true
|
2023-09-01 18:59:57 +05:30
|
|
|
- id: generate-pypi-readme
|
|
|
|
|
name: Generate PyPI README
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/generate_pypi_readme.py
|
2023-09-01 18:59:57 +05:30
|
|
|
language: python
|
|
|
|
|
files: ^README\.md$
|
|
|
|
|
pass_filenames: false
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: lint-markdown
|
2020-11-14 12:23:23 +01:00
|
|
|
name: Run markdownlint
|
2021-05-06 12:36:32 +10:00
|
|
|
description: Checks the style of Markdown files.
|
2020-11-14 12:23:23 +01:00
|
|
|
entry: markdownlint
|
|
|
|
|
language: node
|
|
|
|
|
types: [markdown]
|
|
|
|
|
files: \.(md|mdown|markdown)$
|
2023-12-11 16:14:36 +01:00
|
|
|
additional_dependencies: ['markdownlint-cli@0.38.0']
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: lint-json-schema
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Lint JSON Schema files
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/lint_json_schema.py
|
2020-11-28 12:12:54 +01:00
|
|
|
args:
|
2024-10-14 15:26:05 +01:00
|
|
|
- --spec-file
|
2025-08-17 09:00:14 +02:00
|
|
|
- scripts/ci/prek/draft7_schema.json
|
2020-11-28 12:12:54 +01:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
2021-01-11 23:10:44 +01:00
|
|
|
files: .*\.schema\.json$
|
2020-11-28 12:12:54 +01:00
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: lint-json-schema
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Lint NodePort Service
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/lint_json_schema.py
|
2020-12-04 16:51:59 +01:00
|
|
|
args:
|
|
|
|
|
- --spec-url
|
2021-11-24 23:29:46 +01:00
|
|
|
- https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.20.2-standalone/service-v1.json
|
2020-12-04 16:51:59 +01:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
2022-03-08 09:00:46 +10:00
|
|
|
files: ^scripts/ci/kubernetes/nodeport\.yaml$
|
2020-12-04 16:51:59 +01:00
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: lint-json-schema
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Lint Docker compose files
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/lint_json_schema.py
|
2020-12-04 16:51:59 +01:00
|
|
|
args:
|
|
|
|
|
- --spec-url
|
2024-05-16 20:34:07 +04:00
|
|
|
- https://raw.githubusercontent.com/compose-spec/compose-spec/master/schema/compose-spec.json
|
2020-12-04 16:51:59 +01:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^scripts/ci/docker-compose/.+\.ya?ml$|docker-compose\.ya?ml$
|
2023-02-15 09:34:31 -08:00
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
|
|
|
|
^scripts/ci/docker-compose/grafana/.|
|
2025-04-24 11:19:09 +01:00
|
|
|
^scripts/ci/docker-compose/gremlin/.|
|
2025-04-09 01:26:02 +10:00
|
|
|
^scripts/ci/docker-compose/.+-config\.ya?ml$
|
2020-12-04 16:51:59 +01:00
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-persist-credentials-disabled-in-github-workflows
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Check persistent creds in workflow files
|
|
|
|
|
description: Check that workflow files have persist-credentials disabled
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/checkout_no_credentials.py
|
2021-12-20 22:22:54 +01:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
2022-03-08 09:00:46 +10:00
|
|
|
files: ^\.github/workflows/.*\.yml$
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-docstring-param-types
|
2022-02-08 15:07:54 -08:00
|
|
|
name: Check that docstrings do not specify param types
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/docstring_param_type.py
|
2022-02-08 15:07:54 -08:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
files: \.py$
|
2025-02-10 15:14:04 +08:00
|
|
|
- id: check-zip-file-is-not-committed
|
|
|
|
|
name: Check no zip files are committed
|
|
|
|
|
description: Zip files are not allowed in the repository
|
|
|
|
|
language: fail
|
|
|
|
|
entry: |
|
|
|
|
|
Zip files are not allowed in the repository as they are hard to
|
|
|
|
|
track and have security implications. Please remove the zip file from the repository.
|
|
|
|
|
files: \.zip$
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: update-inlined-dockerfile-scripts
|
2022-03-27 19:19:02 +02:00
|
|
|
name: Inline Dockerfile and Dockerfile.ci scripts
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/inline_scripts_in_docker.py
|
2022-03-27 19:19:02 +02:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: false
|
2023-01-30 05:26:45 +10:00
|
|
|
files: ^Dockerfile$|^Dockerfile\.ci$|^scripts/docker/.*$
|
2022-03-27 19:19:02 +02:00
|
|
|
require_serial: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-changelog-has-no-duplicates
|
2021-11-23 12:35:17 -07:00
|
|
|
name: Check changelogs for duplicate entries
|
|
|
|
|
language: python
|
2025-02-09 09:17:18 +01:00
|
|
|
files: changelog\.(rst|txt)$
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/changelog_duplicates.py
|
2021-11-23 12:35:17 -07:00
|
|
|
pass_filenames: true
|
2024-05-04 15:56:04 +05:30
|
|
|
- id: check-changelog-format
|
|
|
|
|
name: Check changelog format
|
|
|
|
|
language: python
|
2025-02-09 09:17:18 +01:00
|
|
|
files: changelog\.(rst|txt)$
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_changelog_format.py
|
2024-05-04 15:56:04 +05:30
|
|
|
pass_filenames: true
|
2022-04-23 21:33:08 +02:00
|
|
|
- id: check-newsfragments-are-valid
|
2022-04-20 09:30:08 -06:00
|
|
|
name: Check newsfragments are valid
|
|
|
|
|
language: python
|
2025-04-09 01:26:02 +10:00
|
|
|
files: newsfragments/.*\.rst$
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/newsfragments.py
|
2022-04-20 09:30:08 -06:00
|
|
|
pass_filenames: true
|
2022-10-03 09:47:36 -05:00
|
|
|
# We sometimes won't have newsfragments in the repo, so always run it so `check-hooks-apply` passes
|
|
|
|
|
# This is fast, so not too much downside
|
|
|
|
|
always_run: true
|
2022-09-18 12:47:38 +02:00
|
|
|
- id: update-breeze-cmd-output
|
2024-09-20 13:29:48 -07:00
|
|
|
name: Update breeze docs
|
|
|
|
|
description: Update output of breeze commands in Breeze documentation
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/breeze_cmd_line.py
|
2022-09-18 12:47:38 +02:00
|
|
|
language: python
|
2023-04-04 23:00:08 +02:00
|
|
|
files: >
|
|
|
|
|
(?x)
|
2024-01-21 21:41:54 +01:00
|
|
|
^dev/breeze/.*$|
|
2023-04-04 23:00:08 +02:00
|
|
|
^\.pre-commit-config\.yaml$|
|
2025-08-17 09:00:14 +02:00
|
|
|
^scripts/ci/prek/breeze_cmd_line\.py$|
|
2025-04-09 01:26:02 +10:00
|
|
|
^generated/provider_dependencies\.json$
|
2022-10-03 09:47:36 -05:00
|
|
|
require_serial: true
|
2022-09-18 12:47:38 +02:00
|
|
|
pass_filenames: false
|
2022-06-09 00:56:47 +02:00
|
|
|
- id: check-example-dags-urls
|
|
|
|
|
name: Check that example dags url include provider versions
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/update_example_dags_paths.py
|
2022-06-09 00:56:47 +02:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
2025-10-26 23:07:12 +01:00
|
|
|
files:
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow-core/docs/.*example-dags\.rst$|
|
|
|
|
|
^airflow-core/docs/.*index\.rst$|
|
|
|
|
|
^docs/.*index\.rst$
|
2022-06-13 19:38:20 +02:00
|
|
|
always_run: true
|
2022-07-08 13:02:28 +02:00
|
|
|
- id: check-lazy-logging
|
|
|
|
|
name: Check that all logging methods are lazy
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_lazy_logging.py
|
2022-07-08 13:02:28 +02:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
files: \.py$
|
2023-11-10 19:12:26 +01:00
|
|
|
- id: bandit
|
|
|
|
|
name: bandit
|
|
|
|
|
description: "Bandit is a tool for finding common security issues in Python code"
|
|
|
|
|
entry: bandit
|
|
|
|
|
language: python
|
|
|
|
|
language_version: python3
|
2025-04-09 01:26:02 +10:00
|
|
|
types: [python]
|
2023-12-11 08:38:18 +01:00
|
|
|
additional_dependencies: ['bandit==1.7.6']
|
2023-11-10 19:12:26 +01:00
|
|
|
require_serial: true
|
2025-10-26 23:07:12 +01:00
|
|
|
files: ^airflow-core/src/airflow/.* # TODO Expand this to more than just airflow-core
|
2023-11-10 19:12:26 +01:00
|
|
|
exclude:
|
|
|
|
|
airflow/example_dags/.*
|
|
|
|
|
args:
|
|
|
|
|
- "--skip"
|
2024-03-12 01:36:55 +04:00
|
|
|
- "B101,B301,B324,B403,B404,B603"
|
2023-11-10 19:12:26 +01:00
|
|
|
- "--severity-level"
|
|
|
|
|
- "high" # TODO: remove this line when we fix all the issues
|
2025-08-17 09:00:14 +02:00
|
|
|
## ADD MOST PREK HOOK ABOVE THAT LINE
|
|
|
|
|
# The below prek hooks are those requiring CI image to be built
|
2023-04-13 00:03:58 +08:00
|
|
|
- id: mypy-dev
|
2025-10-18 21:48:14 +02:00
|
|
|
stages: ['pre-push']
|
2022-04-20 00:15:46 +02:00
|
|
|
name: Run mypy for dev
|
2022-04-23 09:12:11 +02:00
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/mypy.py
|
2023-11-27 16:24:23 +01:00
|
|
|
files: ^dev/.*\.py$|^scripts/.*\.py$
|
2022-04-20 00:15:46 +02:00
|
|
|
require_serial: true
|
2024-01-13 17:44:28 +01:00
|
|
|
- id: mypy-dev
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
stages: ['manual']
|
2024-01-13 17:44:28 +01:00
|
|
|
name: Run mypy for dev (manual)
|
|
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/mypy_folder.py dev
|
2024-01-13 17:44:28 +01:00
|
|
|
pass_filenames: false
|
|
|
|
|
files: ^.*\.py$
|
|
|
|
|
require_serial: true
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
- id: mypy-devel-common
|
2025-10-18 21:48:14 +02:00
|
|
|
stages: ['pre-push']
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
name: Run mypy for devel-common
|
2024-10-20 00:49:52 +01:00
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/mypy.py
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
files: ^devel-common/.*\.py$
|
2024-10-20 00:49:52 +01:00
|
|
|
require_serial: true
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
- id: mypy-devel-common
|
2024-10-20 00:49:52 +01:00
|
|
|
stages: ['manual']
|
Simplify tooling by switching completely to uv (#48223)
The lazy consensus decision has been made at the devlist to switch
entirely to `uv` as development tool:
link: https://lists.apache.org/thread/6xxdon9lmjx3xh8zw09xc5k9jxb2n256
This PR implements that decision and removes a lot of baggage connected
to using `pip` additionally to uv to install and sync the environment.
It also introduces more consistency in the way how distribution
packages are used in airflow sources - basicaly switching all internal
distributions to use `pyproject.toml` approach and linking them all
together via `uv`'s workspace feature.
This enables much more streamlined development workflows, where any
part of airflow development is manageable using `uv sync` in the right
distribution - opening the way to moving more of the "sub-worfklows"
from the CI image to local virtualenv environment.
Unfortunately, such change cannot be done incrementally, really, because
any change in the project layout drags with itself a lot of changes
in the test/CI/management scripts, so we have to implement one big
PR covering the move.
This PR is "safe" in terms of the airflow and provider's code - it
does not **really** (except occasional imports and type hint changes
resulting from better isolation of packages) change Airflow code nor
it should not affect any airflow or provider code, because it does
not move any of the folder where airflow or provider's code is modified.
It does move the test code - in a number of "auxiliary" distributions
we have. It also moves the `docs` generation code to `devel-common`
and introduces separate conf.py files for every doc package.
What is still NOT done after that move and will be covered in the
follow-up changes:
* isolating docs-building to have separate configuraiton for docs
building per distribution - allowing to run doc build locally
with it's own conf.py file
* moving some of the tests and checks out from breeze container
image up to the local environment (for example mypy checks) and
likely isolating them per-provider
* Constraints are still generated using `pip freeze` and automatically
managed by our custom scripts in `canary` builds - this will be
replaced later by switching to `uv.lock` mechanism.
* potentially, we could merge `devel-common` and `dev` - to be
considered as a follow-up.
* PROD image is stil build with `pip` by default when using
`PyPI` or distribution packages - but we do not support building
the source image with `pip` - when building from sources, uv
is forced internally to install packages. Currently we have
no plans to change default PROD building to use `uv`.
This is the detailed list of changes implemented in this PR:
* uv is now mandatory to install as pre-requisite in order to
develop airflow. We do not support installing airflow for
development with `pip` - there will be a lot of cases where
it will not work for development - including development
dependencies and installing several distributions together.
* removed meta-package `hatch_build.py' and replacing it with
pre-commit automatically modifying declarative pyproject.toml
* stripped down `hatch_build_airflow_core.py` to only cover custom
git and asset build hooks (and renaming the file to `hatch_build.py`
and moving all airflow dependencies to `pyproject.toml`
* converted "loose" packages in airflow repo into distributions:
* docker-tests
* kubernetes-tests
* helm-tests
* dev (here we do not have `src` subfolder - sources are directly
in the distribution, which is for-now inconsistent with other
distributions).
The names of the `_tests` distribution folders have been renamed to
the `-tests` convention to make sure the imports are always
referring to base of each distribution and are not used from the
content root.
* Each eof the distributions (on top of already existing airflow-core,
task-sdk, devel-common and 90+providers has it's own set of
dependencies, and the top-level meta-package workspace root brings
those distributions together allowing to install them all tegether
with a simple `uv sync --all-packages` command and come up with
consistent set of dependencies that are good for all those
packages (yay!). This is used to build CI image with single
common environment to run the tests (with some quirks due to
constraints use where we have to manually list all distributions
until we switch to `uv.lock` mechanism)
* `doc` code is moved to `devel-common` distribution. The `doc` folder
only keeps README informing where the other doc code is, the
spelling_wordlist.txt and start_docs_server.sh. The documentation is
generated in `generated/generated-docs/` folder which is entirely
.gitignored.
* the documentation is now fully moved to:
* `airflow-core/docs` - documentation for Airflow Core
* `providers/**/docs` - documentation for Providers
* `chart/docs` - documentation for Helm Chart
* `task-sdk/docs` - documentation for Task SDK (new format not yet published)
* `docker-stack-docs` - documentation for Docker Stack'
* `providers-summary-docs` - documentation for provider summary page
* `versions` are not dynamically retrieved from `__init__.py` all
of them are synchronized directly to pyproject.toml files - this
way - except the custom build hook - we have no dynamic components
in our `pyproject.toml` properties.
* references to extras were removed from INSTALL and other places,
the only references to extras remains in the user documentation - we
stop using extras for local development, we switch to using
dependency groups.
* backtracking command was removed from breeze - we did not need it
since we started using `uv`
* internal commands (except constraint generation) have been moved to
`uv` from `pip`
* breeze requires `uv` to be installed and expects to be installed by
`uv tool install -e ./dev/breeze`
* pyproject.tomls are dynamically modified when we add a version
suffix dynamically (`--version-suffix-for-pypi`) - only for the
time of building the versions with updated suffix
* `mypy` checks are now consistently used across all the different
distributions and for consistency (and to fix some of the issues
with namespace packages) rather than using "folder" approach
when running mypy checks, even if we run mypy for whole
distribution, we run check on individual files rather than on
a folder. That adds consistency in execution of mypy heursistics.
Rather than using in-container mypy script all the logic of
selection and parameters passed to mypy are in pre-commit code.
For now we are still using CI image to run mypy because mypy is
very sensitive to version of dependencies installed, we should
be able to switch to running mypy locally once we have the
`uv.lock` mechanism incorporated in our workflows.
* lower bounds for dependencies have been set consistently across
all the distributions. With `uv sync` and dependabot, those
should be generally kept consistently for the future
* the `devel-common` dependencies have been groupped together in
`devel-common` extras - including `basic`, `doc`, `doc-gen`, and
`all` which will make it easier to install them for some OS-es
(basic is used as default set of dependencies to cover most
common set of development dependencies to be used for development)
* generated/provider_dependencies.json are not committed to the
repository any longer. They are .gitignored and geberated
on-the-flight as needed (breeze will generate them automatically
when empty and pre-commit will always regenerate them to be
consistent with provider's pyproject.toml files.
* `chart-utils` have been noved to `helm-tests` from `devel-common`
as they were only used there.
* for k8s tests we are using the `uv` main `.venv` environment
rather than creating our own `.build` environment and we use
`uv sync` to keep it in sync
* Updated `uv` version to 0.6.10
* We are using `uv sync` to perform "upgrade to newer depencies"
in `canary` builds and locally
* leveldb has been turned into "dependency group" and removed from
apache-airflow and apache-airflow-core extras, it is now only
available by google provider's leveldb optional extra to install
with `pip`
2025-04-02 13:11:13 +02:00
|
|
|
name: Run mypy for devel-common (manual)
|
2024-10-20 00:49:52 +01:00
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/mypy_folder.py devel-common
|
2024-10-20 00:49:52 +01:00
|
|
|
pass_filenames: false
|
|
|
|
|
files: ^.*\.py$
|
|
|
|
|
require_serial: true
|
2024-09-27 14:24:42 +01:00
|
|
|
- id: check-template-fields-valid
|
|
|
|
|
name: Check templated fields mapped in operators/sensors
|
|
|
|
|
language: python
|
2025-08-17 09:00:14 +02:00
|
|
|
entry: ./scripts/ci/prek/check_template_fields.py
|
2025-03-22 00:28:55 +05:30
|
|
|
files: ^(providers/.*/)?airflow-core/.*/(sensors|operators)/.*\.py$
|
2024-09-27 14:24:42 +01:00
|
|
|
require_serial: true
|
2025-02-25 15:01:33 +00:00
|
|
|
- id: generate-tasksdk-datamodels
|
|
|
|
|
name: Generate Datamodels for TaskSDK client
|
|
|
|
|
language: python
|
2025-08-10 18:34:44 +02:00
|
|
|
entry: uv run -p 3.12 --no-progress --active --group codegen --project apache-airflow-task-sdk --directory task-sdk -s dev/generate_task_sdk_models.py
|
2025-02-25 15:01:33 +00:00
|
|
|
pass_filenames: false
|
2025-03-21 14:25:26 +01:00
|
|
|
files: ^airflow-core/src/airflow/api_fastapi/execution_api/.*\.py$
|
2025-02-25 15:01:33 +00:00
|
|
|
require_serial: true
|
2025-04-21 20:01:05 +05:30
|
|
|
- id: generate-airflowctl-datamodels
|
|
|
|
|
name: Generate Datamodels for AirflowCTL
|
|
|
|
|
language: python
|
|
|
|
|
entry: >
|
|
|
|
|
bash -c '
|
|
|
|
|
uv run -p 3.12 --no-dev --no-progress --active --group codegen --project apache-airflow-ctl --directory airflow-ctl/ datamodel-codegen &&
|
2025-05-17 13:35:20 +05:30
|
|
|
uv run -p 3.12 --no-dev --no-progress --active --group codegen --project apache-airflow-ctl --directory airflow-ctl/ datamodel-codegen --input="../airflow-core/src/airflow/api_fastapi/auth/managers/simple/openapi/v2-simple-auth-manager-generated.yaml" --output="src/airflowctl/api/datamodels/auth_generated.py"'
|
2025-04-21 20:01:05 +05:30
|
|
|
pass_filenames: false
|
2025-10-26 23:07:12 +01:00
|
|
|
files:
|
2025-07-25 11:56:13 +05:30
|
|
|
(?x)
|
2025-10-26 23:07:12 +01:00
|
|
|
^airflow-core/src/airflow/api_fastapi/core_api/datamodels/.*\.py$|
|
|
|
|
|
^airflow-core/src/airflow/api_fastapi/auth/managers/simple/(datamodels|routes|services|openapi)/.*\.py$
|
2025-08-30 00:29:18 +01:00
|
|
|
require_serial: true
|
2025-10-26 23:07:12 +01:00
|
|
|
## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE
|
2025-09-23 04:42:46 +01:00
|
|
|
- id: check-contextmanager-class-decorators
|
|
|
|
|
name: Check for problematic context manager class decorators
|
|
|
|
|
entry: ./scripts/ci/prek/check_contextmanager_class_decorators.py
|
|
|
|
|
language: python
|
|
|
|
|
files: .*test.*\.py$
|
|
|
|
|
pass_filenames: true
|
