2019-08-14 20:53:28 -04:00
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
|
|
|
# or more contributor license agreements. See the NOTICE file
|
|
|
|
|
# distributed with this work for additional information
|
|
|
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
|
|
|
# to you under the Apache License, Version 2.0 (the
|
|
|
|
|
# "License"); you may not use this file except in compliance
|
|
|
|
|
# with the License. You may obtain a copy of the License at
|
|
|
|
|
#
|
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
#
|
|
|
|
|
# Unless required by applicable law or agreed to in writing,
|
|
|
|
|
# software distributed under the License is distributed on an
|
|
|
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
|
# KIND, either express or implied. See the License for the
|
|
|
|
|
# specific language governing permissions and limitations
|
|
|
|
|
# under the License.
|
|
|
|
|
---
|
|
|
|
|
default_stages: [commit, push]
|
2019-08-22 10:13:56 -04:00
|
|
|
default_language_version:
|
2021-05-06 23:34:14 +02:00
|
|
|
# force all unspecified python hooks to run python3
|
2019-08-22 10:13:56 -04:00
|
|
|
python: python3
|
2019-11-04 09:50:06 +01:00
|
|
|
minimum_pre_commit_version: "1.20.0"
|
2019-08-14 20:53:28 -04:00
|
|
|
repos:
|
2020-12-16 14:14:20 +01:00
|
|
|
- repo: meta
|
|
|
|
|
hooks:
|
|
|
|
|
- id: identity
|
|
|
|
|
- id: check-hooks-apply
|
2019-09-04 15:56:03 +02:00
|
|
|
- repo: https://github.com/Lucas-C/pre-commit-hooks
|
2021-04-29 09:08:38 +02:00
|
|
|
rev: v1.1.10
|
2019-08-15 19:59:35 -04:00
|
|
|
hooks:
|
2019-08-16 10:35:34 -04:00
|
|
|
- id: forbid-tabs
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^docs/Makefile$|^clients/gen/go.sh|\.gitmodules$|^airflow/_vendor/
|
2019-08-15 19:59:35 -04:00
|
|
|
- id: insert-license
|
2019-10-26 13:31:51 +02:00
|
|
|
name: Add license for all SQL files
|
2019-08-15 19:59:35 -04:00
|
|
|
files: \.sql$
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^\.github/.*$|^airflow/_vendor/
|
2019-08-15 19:59:35 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "/*||*/"
|
|
|
|
|
- --license-filepath
|
|
|
|
|
- license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-15 20:02:53 -04:00
|
|
|
- id: insert-license
|
2019-10-26 13:31:51 +02:00
|
|
|
name: Add license for all other files
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^\.github/.*$|^airflow/_vendor/
|
2019-08-15 20:02:53 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "|#|"
|
|
|
|
|
- --license-filepath
|
|
|
|
|
- license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-15 20:02:53 -04:00
|
|
|
files: >
|
2020-11-25 00:58:01 +01:00
|
|
|
\.properties$|\.cfg$|\.conf$|\.ini$|\.ldif$|\.readthedocs$|\.service$|\.tf$|Dockerfile.*$
|
2019-08-16 00:09:26 -04:00
|
|
|
- id: insert-license
|
2019-10-26 13:31:51 +02:00
|
|
|
name: Add license for all rst files
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^\.github/.*$|^airflow/_vendor/
|
2019-08-16 00:09:26 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "||"
|
|
|
|
|
- --license-filepath
|
|
|
|
|
- license-templates/LICENSE.rst
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-16 00:09:26 -04:00
|
|
|
files: \.rst$
|
2019-08-15 20:06:18 -04:00
|
|
|
- id: insert-license
|
2021-03-18 07:02:29 -04:00
|
|
|
name: Add license for all JS/TS/TSX/CSS/PUML files
|
|
|
|
|
files: \.(js|ts|tsx|css|puml)$
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^\.github/.*$|^airflow/_vendor/
|
2019-08-15 20:06:18 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
2020-10-16 03:59:09 -04:00
|
|
|
- "/*!| *| */"
|
2019-08-15 20:06:18 -04:00
|
|
|
- --license-filepath
|
|
|
|
|
- license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
|
|
|
|
- id: insert-license
|
2019-10-26 13:31:51 +02:00
|
|
|
name: Add license for all JINJA template files
|
2021-05-06 23:34:14 +02:00
|
|
|
files: "^airflow/www/templates/.*\\.html$|^docs/templates/.*\\.html$.*\\.jinja2"
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^\.github/.*$^airflow/_vendor/
|
2019-08-27 14:06:41 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "{#||#}"
|
|
|
|
|
- --license-filepath
|
|
|
|
|
- license-templates/LICENSE.txt
|
|
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-20 00:07:39 -04:00
|
|
|
- id: insert-license
|
2019-10-26 13:31:51 +02:00
|
|
|
name: Add license for all shell files
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^\.github/.*$|^airflow/_vendor/
|
2022-02-01 12:21:09 +01:00
|
|
|
files: ^breeze$|^breeze-complete$|\.sh$|\.bash$
|
2019-08-20 00:07:39 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "|#|"
|
|
|
|
|
- --license-filepath
|
|
|
|
|
- license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-09-17 13:16:32 +02:00
|
|
|
- id: insert-license
|
2021-05-06 12:36:32 +10:00
|
|
|
name: Add license for all Python files
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^\.github/.*$|^airflow/_vendor/
|
2021-12-27 16:46:22 +01:00
|
|
|
files: \.py$|\.pyi$
|
2019-09-17 13:16:32 +02:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "|#|"
|
|
|
|
|
- --license-filepath
|
|
|
|
|
- license-templates/LICENSE.txt
|
|
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-20 10:29:34 -04:00
|
|
|
- id: insert-license
|
2019-10-26 13:31:51 +02:00
|
|
|
name: Add license for all XML files
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^\.github/.*$|^airflow/_vendor/
|
2020-08-24 22:45:28 +02:00
|
|
|
files: \.xml$
|
2019-08-20 10:29:34 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "<!--||-->"
|
|
|
|
|
- --license-filepath
|
|
|
|
|
- license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-22 10:13:56 -04:00
|
|
|
- id: insert-license
|
2021-05-06 12:36:32 +10:00
|
|
|
name: Add license for all YAML files
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^\.github/.*$|^airflow/_vendor/
|
2019-08-22 10:13:56 -04:00
|
|
|
types: [yaml]
|
2021-05-06 23:34:14 +02:00
|
|
|
files: \.yml$|\.yaml$
|
2019-08-22 10:13:56 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "|#|"
|
|
|
|
|
- --license-filepath
|
|
|
|
|
- license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2019-08-21 23:27:54 -04:00
|
|
|
- id: insert-license
|
2019-10-26 13:31:51 +02:00
|
|
|
name: Add license for all md files
|
2019-08-21 23:27:54 -04:00
|
|
|
files: \.md$
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^\.github/.*$|PROVIDER_CHANGES.*\.md|^airflow/_vendor/
|
2019-08-21 23:27:54 -04:00
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "<!--|| -->"
|
|
|
|
|
- --license-filepath
|
|
|
|
|
- license-templates/LICENSE.txt
|
2019-08-27 14:06:41 -04:00
|
|
|
- --fuzzy-match-generates-todo
|
2020-08-24 22:45:28 +02:00
|
|
|
- id: insert-license
|
|
|
|
|
name: Add license for all mermaid files
|
|
|
|
|
args:
|
|
|
|
|
- --comment-style
|
|
|
|
|
- "|%%|"
|
|
|
|
|
- --license-filepath
|
|
|
|
|
- license-templates/LICENSE.txt
|
|
|
|
|
- --fuzzy-match-generates-todo
|
|
|
|
|
files: \.mermaid$
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2019-08-21 23:27:54 -04:00
|
|
|
- repo: https://github.com/thlorenz/doctoc.git
|
2022-01-02 12:54:52 +01:00
|
|
|
rev: v2.1.0
|
2019-08-21 23:27:54 -04:00
|
|
|
hooks:
|
|
|
|
|
- id: doctoc
|
|
|
|
|
name: Add TOC for md files
|
2021-05-20 09:12:24 -06:00
|
|
|
files:
|
|
|
|
|
^README\.md$|^CONTRIBUTING\.md$|^UPDATING.*\.md$|^chart/UPDATING.*\.md$|^dev/.*\.md$|^dev/.*\.rst$
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2019-08-21 23:27:54 -04:00
|
|
|
args:
|
|
|
|
|
- "--maxlevel"
|
|
|
|
|
- "2"
|
2020-08-25 12:10:20 +01:00
|
|
|
- repo: https://github.com/psf/black
|
2022-01-02 12:54:52 +01:00
|
|
|
rev: 21.12b0
|
2020-08-25 12:10:20 +01:00
|
|
|
hooks:
|
|
|
|
|
- id: black
|
|
|
|
|
args: [--config=./pyproject.toml]
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2021-05-27 07:09:25 +02:00
|
|
|
- repo: https://github.com/asottile/blacken-docs
|
2022-01-02 12:54:52 +01:00
|
|
|
rev: v1.12.0
|
2021-05-27 07:09:25 +02:00
|
|
|
hooks:
|
|
|
|
|
- id: blacken-docs
|
|
|
|
|
alias: black
|
2022-01-02 12:54:52 +01:00
|
|
|
additional_dependencies: [black==21.12b0]
|
2019-08-16 10:35:34 -04:00
|
|
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
2022-01-02 12:54:52 +01:00
|
|
|
rev: v4.1.0
|
2019-08-16 10:35:34 -04:00
|
|
|
hooks:
|
|
|
|
|
- id: check-merge-conflict
|
2019-11-26 09:28:42 +01:00
|
|
|
- id: debug-statements
|
2020-06-25 16:35:37 +01:00
|
|
|
- id: check-builtin-literals
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2019-08-16 10:35:34 -04:00
|
|
|
- id: detect-private-key
|
|
|
|
|
- id: end-of-file-fixer
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2019-08-16 10:35:34 -04:00
|
|
|
- id: mixed-line-ending
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2019-08-20 10:29:34 -04:00
|
|
|
- id: check-executables-have-shebangs
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2019-08-20 10:29:34 -04:00
|
|
|
- id: check-xml
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2019-12-28 19:34:35 +01:00
|
|
|
- id: trailing-whitespace
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2020-02-04 09:44:35 +01:00
|
|
|
- id: fix-encoding-pragma
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2020-02-04 09:44:35 +01:00
|
|
|
args:
|
|
|
|
|
- --remove
|
2021-05-17 12:23:45 -06:00
|
|
|
- id: pretty-format-json
|
|
|
|
|
args:
|
|
|
|
|
- --autofix
|
|
|
|
|
- --no-sort-keys
|
|
|
|
|
- --indent
|
|
|
|
|
- "4"
|
|
|
|
|
files: ^chart/values\.schema\.json$|^chart/values_schema\.schema\.json$
|
|
|
|
|
pass_filenames: true
|
2022-01-10 11:14:22 -05:00
|
|
|
# TODO: Bump to Python 3.7 when support for Python 3.6 is dropped in Airflow 2.3.
|
2020-11-03 21:53:59 +00:00
|
|
|
- repo: https://github.com/asottile/pyupgrade
|
2022-01-02 12:54:52 +01:00
|
|
|
rev: v2.31.0
|
2020-11-03 21:53:59 +00:00
|
|
|
hooks:
|
|
|
|
|
- id: pyupgrade
|
2022-01-10 11:14:22 -05:00
|
|
|
args: ["--py36-plus"]
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2019-09-21 20:47:02 +01:00
|
|
|
- repo: https://github.com/pre-commit/pygrep-hooks
|
2021-07-14 21:22:59 +01:00
|
|
|
rev: v1.9.0
|
2019-09-21 20:47:02 +01:00
|
|
|
hooks:
|
|
|
|
|
- id: rst-backticks
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2019-09-22 08:55:39 +01:00
|
|
|
- id: python-no-log-warn
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2020-06-14 02:37:30 +02:00
|
|
|
- repo: https://github.com/adrienverge/yamllint
|
2021-09-11 23:47:06 +01:00
|
|
|
rev: v1.26.3
|
2019-08-14 20:53:28 -04:00
|
|
|
hooks:
|
2019-08-22 10:13:56 -04:00
|
|
|
- id: yamllint
|
2021-05-06 12:36:32 +10:00
|
|
|
name: Check YAML files with yamllint
|
2020-10-21 12:24:32 +02:00
|
|
|
entry: yamllint -c yamllint-config.yml --strict
|
2019-08-22 10:13:56 -04:00
|
|
|
types: [yaml]
|
AIP-40: Add Deferrable "Async" Tasks (#15389)
This is the implementation of AIP-40, Deferrable "Async" Tasks (https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=177050929).
The main changes are:
- A new concept of a Trigger is introduced, as a small piece of asyncio code that can fire off events
- There is a BaseTrigger and some time-related triggers under a new `airflow.triggers` package
- There is a new Trigger database model and associated `trigger` table
- `Async` versions of the various date/time sensors have been added which defer rather than poke.
- There is a new persistent process (Job) called `triggerer`
- It only runs on Python 3.7+
- It handles polling the database for which triggers need running, running them, and re-scheduling task instances whose triggers have fired events
- If a trigger throws an exception or exits without firing an event, it logs why and marks dependent task instances as failed
- It monitors the asyncio event loop with a watchdog task and alerts the user if anything is overrunning (i.e. not using `await`) and blocking the loop.
- It is designed to run in parallel with itself in a highly-available manner, and also has built-in consistent-hash based partitioning (sharding) support
- Task Instances have a new `deferred` state which indicates they are waiting on a trigger to run
- The trigger they are waiting for is stored in a new `trigger_id` column, and a failure timeout is in a `trigger_timeout` column
- The scheduler takes care of timing out task instances into the `failed` state
- Deferral is triggered by raising the `TaskDeferred` exception, or calling `self.defer` on the TaskInstance which does the same thing.
- A `next_method` and `next_kwargs` column are added to specify what a task instance/operator's execution entry point should be if it's not the default of `execute()`. They are currently only used by deferral, but have been written to be independent in case they are useful elsewhere.
- Two new dependencies are added
- `jump-consistent-hash` is a small MIT licensed library that implements a fast, consistent hash algorithm
- `pytest-asyncio` is an Apache 2 licensed library that enables async tests to be written easily
Changes that are deliberately not in here and will be in a future PR for them specifically:
- UI warning when the triggerer is not running and you have deferred task instances
- Updating Breeze to include `triggerer` in what it runs
- Updating the Docker Compose files to include `triggerer`
- Updating the Helm Chart to include `triggerer`
- Some way of detecting/preventing DB access within triggers
2021-08-11 13:33:38 -06:00
|
|
|
exclude: ^.*init_git_sync\.template\.yaml$|^.*airflow\.template\.yaml$|^chart/(?:templates|files)/.*\.yaml$|openapi/.*\.yaml|pre-commit-config|^airflow/_vendor/
|
2022-01-02 12:54:52 +01:00
|
|
|
- repo: https://github.com/PyCQA/isort
|
|
|
|
|
rev: 5.10.1
|
2020-06-14 02:37:30 +02:00
|
|
|
hooks:
|
|
|
|
|
- id: isort
|
2021-05-06 12:36:32 +10:00
|
|
|
name: Run isort to sort imports in Python files
|
2021-12-29 23:27:14 +01:00
|
|
|
files: \.py|\.pyi$
|
2020-06-14 02:37:30 +02:00
|
|
|
# To keep consistent with the global isort skip config defined in setup.cfg
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^build/.*$|^.tox/.*$|^venv/.*$|^airflow/_vendor/
|
2020-06-21 09:34:41 +01:00
|
|
|
- repo: https://github.com/pycqa/pydocstyle
|
2021-05-31 02:51:31 +01:00
|
|
|
rev: 6.1.1
|
2020-06-21 09:34:41 +01:00
|
|
|
hooks:
|
|
|
|
|
- id: pydocstyle
|
2020-06-21 09:51:57 +01:00
|
|
|
name: Run pydocstyle
|
2020-06-21 09:34:41 +01:00
|
|
|
args:
|
2020-06-21 09:51:57 +01:00
|
|
|
- --convention=pep257
|
2022-02-09 13:21:39 -08:00
|
|
|
- --add-ignore=D100,D102,D103,D104,D105,D107,D202,D205,D400,D401
|
2020-10-27 18:29:47 -07:00
|
|
|
exclude: |
|
|
|
|
|
(?x)
|
|
|
|
|
^tests/.*\.py$|
|
|
|
|
|
^scripts/.*\.py$|
|
|
|
|
|
^dev|
|
|
|
|
|
^provider_packages|
|
2021-11-24 17:59:47 +01:00
|
|
|
^docker_tests|
|
2020-10-27 18:29:47 -07:00
|
|
|
^kubernetes_tests|
|
|
|
|
|
.*example_dags/.*|
|
2021-05-17 12:10:54 +02:00
|
|
|
^chart/.*\.py$|
|
|
|
|
|
^airflow/_vendor/
|
2021-06-28 19:26:16 +02:00
|
|
|
additional_dependencies: ['toml']
|
|
|
|
|
- repo: https://github.com/asottile/yesqa
|
2022-01-02 12:54:52 +01:00
|
|
|
rev: v1.3.0
|
2021-06-28 19:26:16 +02:00
|
|
|
hooks:
|
|
|
|
|
- id: yesqa
|
|
|
|
|
exclude: |
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow/_vendor/
|
2021-10-11 23:22:46 +05:30
|
|
|
additional_dependencies: ['flake8>=4.0.1']
|
2021-09-10 11:54:15 -06:00
|
|
|
- repo: https://github.com/ikamensh/flynt
|
2021-09-11 23:47:06 +01:00
|
|
|
rev: '0.69'
|
2021-08-18 01:56:38 +02:00
|
|
|
hooks:
|
|
|
|
|
- id: flynt
|
|
|
|
|
exclude: |
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow/_vendor/
|
2021-10-17 18:34:06 +02:00
|
|
|
args:
|
|
|
|
|
# If flynt detects too long text it ignores it. So we set a very large limit to make it easy
|
|
|
|
|
# to split the text by hand. Too long lines are detected by flake8 (below),
|
|
|
|
|
# so the user is informed to take action.
|
|
|
|
|
- --line-length
|
|
|
|
|
- '99999'
|
2021-10-14 19:37:08 +01:00
|
|
|
- repo: https://github.com/codespell-project/codespell
|
|
|
|
|
rev: v2.1.0
|
|
|
|
|
hooks:
|
|
|
|
|
- id: codespell
|
|
|
|
|
name: Run codespell to check for common misspellings in files
|
2021-10-30 15:44:20 +05:30
|
|
|
entry: bash -c 'echo "If you think that this failure is an error, consider adding the word(s)
|
|
|
|
|
to the codespell dictionary at docs/spelling_wordlist.txt.
|
|
|
|
|
The word(s) should be in lowercase." && exec codespell "$@"' --
|
2021-10-14 19:37:08 +01:00
|
|
|
language: python
|
|
|
|
|
types: [text]
|
|
|
|
|
exclude: ^airflow/_vendor/|^CHANGELOG.txt|^airflow/www/static/css/material-icons.css
|
|
|
|
|
args:
|
|
|
|
|
- --ignore-words=docs/spelling_wordlist.txt
|
2021-10-14 18:22:10 -04:00
|
|
|
- --skip=docs/*/commits.rst,airflow/providers/*/*.rst,*.lock,INTHEWILD.md,*.min.js,docs/apache-airflow/pipeline_example.csv
|
2022-01-24 12:43:09 +00:00
|
|
|
- --exclude-file=.codespellignorelines
|
2020-06-14 02:37:30 +02:00
|
|
|
- repo: local
|
|
|
|
|
hooks:
|
2021-12-23 13:38:57 +01:00
|
|
|
- id: autoflake
|
|
|
|
|
name: Remove all unused code
|
|
|
|
|
entry: autoflake --remove-all-unused-imports --ignore-init-module-imports --in-place
|
|
|
|
|
language: python
|
|
|
|
|
additional_dependencies: ['autoflake']
|
2020-06-01 21:46:15 +02:00
|
|
|
- id: lint-openapi
|
2020-09-03 17:06:23 +01:00
|
|
|
name: Lint OpenAPI using spectral
|
2020-06-01 21:46:15 +02:00
|
|
|
language: docker_image
|
2020-09-03 17:06:23 +01:00
|
|
|
entry: stoplight/spectral lint -r ./scripts/ci/spectral_rules/connexion.yml
|
2020-06-02 18:45:37 +02:00
|
|
|
files: ^airflow/api_connexion/openapi/
|
2020-06-01 21:46:15 +02:00
|
|
|
- id: lint-openapi
|
|
|
|
|
name: Lint OpenAPI using openapi-spec-validator
|
2020-06-02 18:45:37 +02:00
|
|
|
entry: openapi-spec-validator --schema 3.0.0
|
2020-06-01 21:46:15 +02:00
|
|
|
language: python
|
|
|
|
|
additional_dependencies: ['openapi-spec-validator']
|
2020-06-02 18:45:37 +02:00
|
|
|
files: ^airflow/api_connexion/openapi/
|
2019-08-14 20:53:28 -04:00
|
|
|
- id: lint-dockerfile
|
|
|
|
|
name: Lint dockerfile
|
|
|
|
|
language: system
|
2021-05-06 12:36:32 +10:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh
|
2020-09-06 18:06:05 +02:00
|
|
|
files: Dockerfile.*$
|
2019-09-18 13:43:30 +02:00
|
|
|
pass_filenames: true
|
2021-02-27 18:38:43 +01:00
|
|
|
require_serial: true
|
2019-11-30 17:05:14 +08:00
|
|
|
- id: setup-order
|
2021-05-06 12:36:32 +10:00
|
|
|
name: Check order of dependencies in setup.cfg and setup.py
|
2019-11-30 17:05:14 +08:00
|
|
|
language: python
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^setup\.cfg$|^setup\.py$
|
2019-11-30 17:05:14 +08:00
|
|
|
pass_filenames: false
|
2020-07-16 18:05:35 +02:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py
|
2020-12-28 19:10:27 +01:00
|
|
|
additional_dependencies: ['rich']
|
2020-11-13 09:38:54 +01:00
|
|
|
- id: setup-extra-packages
|
2020-12-26 02:24:27 -08:00
|
|
|
name: Checks setup extra packages
|
|
|
|
|
description: Checks if all the libraries in setup.py are listed in extra-packages-ref.rst file
|
2020-11-02 14:17:41 +01:00
|
|
|
language: python
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^setup\.py$|^docs/apache-airflow/extra-packages-ref\.rst$
|
2020-11-02 14:17:41 +01:00
|
|
|
pass_filenames: false
|
2020-11-13 09:38:54 +01:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
|
2020-11-10 00:54:47 +00:00
|
|
|
additional_dependencies: ['rich==9.2.0']
|
2019-11-04 10:37:31 +01:00
|
|
|
- id: update-breeze-file
|
|
|
|
|
name: Update output of breeze command in BREEZE.rst
|
2021-05-06 12:36:32 +10:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
|
2019-11-04 10:37:31 +01:00
|
|
|
language: system
|
2021-07-20 03:47:25 +02:00
|
|
|
files: ^BREEZE\.rst$|^breeze$|^breeze-complete|^Dockerfile$
|
2019-11-04 10:37:31 +01:00
|
|
|
pass_filenames: false
|
2020-03-07 18:12:28 +01:00
|
|
|
- id: update-local-yml-file
|
|
|
|
|
name: Update mounts in the local yml file
|
2021-05-06 12:36:32 +10:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
|
2020-03-07 18:12:28 +01:00
|
|
|
language: system
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^scripts/ci/libraries/_local_mounts\.sh$|^scripts/ci/docker_compose/local\.yml$
|
2020-03-07 18:12:28 +01:00
|
|
|
pass_filenames: false
|
2020-02-24 10:29:38 +01:00
|
|
|
- id: update-setup-cfg-file
|
|
|
|
|
name: Update setup.cfg file with all licenses
|
2021-05-06 12:36:32 +10:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh
|
2020-02-24 10:29:38 +01:00
|
|
|
language: system
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^setup\.cfg$
|
2020-02-24 10:29:38 +01:00
|
|
|
pass_filenames: false
|
2020-03-07 20:55:27 +01:00
|
|
|
- id: build-providers-dependencies
|
|
|
|
|
name: Build cross-dependencies for providers packages
|
2021-05-06 12:36:32 +10:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh
|
2021-06-20 14:32:11 +02:00
|
|
|
language: python
|
2020-03-07 20:55:27 +01:00
|
|
|
files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$
|
|
|
|
|
pass_filenames: false
|
2021-06-20 14:32:11 +02:00
|
|
|
additional_dependencies: ['setuptools']
|
2020-03-22 13:34:48 +01:00
|
|
|
- id: update-extras
|
|
|
|
|
name: Update extras in documentation
|
2020-07-16 18:05:35 +02:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_insert_extras.py
|
|
|
|
|
language: python
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^setup\.py$|^INSTALL$|^CONTRIBUTING\.rst$
|
2020-03-22 13:34:48 +01:00
|
|
|
pass_filenames: false
|
2021-09-21 21:22:41 +02:00
|
|
|
- id: check-extras-order
|
|
|
|
|
name: Check order of extras in Dockerfile
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_check_order_dockerfile_extras.py
|
|
|
|
|
language: python
|
|
|
|
|
files: ^Dockerfile$
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
additional_dependencies: ['rich']
|
2021-12-30 22:13:10 +01:00
|
|
|
- id: update-supported-versions
|
|
|
|
|
name: Updates supported versions in documentation
|
|
|
|
|
entry: ./scripts/ci/pre_commit/supported_versions.py
|
|
|
|
|
language: python
|
|
|
|
|
files: ^scripts/ci/pre_commit/supported_versions.py$|^README.md$|^docs/apache-airflow/supported-versions.rst$
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
additional_dependencies: ['tabulate']
|
2021-09-18 10:41:57 +02:00
|
|
|
- id: update-version
|
|
|
|
|
name: Update version to the latest version in the documentation
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_update_versions.py
|
|
|
|
|
language: python
|
|
|
|
|
files: ^docs
|
|
|
|
|
pass_filenames: false
|
2019-11-26 09:28:42 +01:00
|
|
|
- id: pydevd
|
|
|
|
|
language: pygrep
|
|
|
|
|
name: Check for pydevd debug statements accidentally left
|
|
|
|
|
entry: "pydevd.*settrace\\("
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
files: \.py$
|
2020-06-08 22:27:02 +01:00
|
|
|
- id: dont-use-safe-filter
|
|
|
|
|
language: pygrep
|
|
|
|
|
name: Don't use safe in templates
|
|
|
|
|
description: the Safe filter is error-prone, use Markup() in code instead
|
|
|
|
|
entry: "\\|\\s*safe"
|
|
|
|
|
files: \.html$
|
|
|
|
|
pass_filenames: true
|
2020-11-10 22:49:08 +01:00
|
|
|
- id: no-providers-in-core-examples
|
|
|
|
|
language: pygrep
|
|
|
|
|
name: No providers imports in core example DAGs
|
|
|
|
|
description: The core example DAGs have no dependencies other than core Airflow
|
|
|
|
|
entry: "^\\s*from airflow\\.providers.*"
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
files: ^airflow/example_dags/.*\.py$
|
2020-09-10 18:07:50 +01:00
|
|
|
- id: no-relative-imports
|
|
|
|
|
language: pygrep
|
|
|
|
|
name: No relative imports
|
|
|
|
|
description: Airflow style is to use absolute imports only
|
|
|
|
|
entry: "^\\s*from\\s+\\."
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
files: \.py$
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^tests/|^airflow/_vendor/
|
Don't use the term "whitelist" - language matters (#9174)
It's fairly common to say whitelisting and blacklisting to describe
desirable and undesirable things in cyber security. However just because
it is common doesn't mean it's right.
However, there's an issue with the terminology. It only makes sense if
you equate white with 'good, permitted, safe' and black with 'bad,
dangerous, forbidden'. There are some obvious problems with this.
You may not see why this matters. If you're not adversely affected by
racial stereotyping yourself, then please count yourself lucky. For some
of your friends and colleagues (and potential future colleagues), this
really is a change worth making.
From now on, we will use 'allow list' and 'deny list' in place of
'whitelist' and 'blacklist' wherever possible. Which, in fact, is
clearer and less ambiguous. So as well as being more inclusive of all,
this is a net benefit to our understandability.
(Words mostly borrowed from
<https://www.ncsc.gov.uk/blog-post/terminology-its-not-black-and-white>)
Co-authored-by: Jarek Potiuk <jarek@potiuk.com>
2020-06-08 10:01:46 +01:00
|
|
|
- id: language-matters
|
|
|
|
|
language: pygrep
|
|
|
|
|
name: Check for language that we do not accept as community
|
|
|
|
|
description: Please use "deny_list" or "allow_list" instead.
|
|
|
|
|
entry: "(?i)(black|white)[_-]?list"
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
2021-09-28 14:29:06 -07:00
|
|
|
^airflow/www/fab_security/manager\.py$|
|
2021-05-06 12:36:32 +10:00
|
|
|
^airflow/providers/apache/cassandra/hooks/cassandra\.py$|
|
|
|
|
|
^airflow/providers/apache/hive/operators/hive_stats\.py$|
|
2020-10-25 19:45:50 +01:00
|
|
|
^airflow/providers/apache/hive/.*PROVIDER_CHANGES_*|
|
2021-05-06 12:36:32 +10:00
|
|
|
^airflow/providers/apache/hive/.*README\.md$|
|
|
|
|
|
^tests/providers/apache/cassandra/hooks/test_cassandra\.py$|
|
|
|
|
|
^docs/apache-airflow-providers-apache-cassandra/connections/cassandra\.rst$|
|
|
|
|
|
^docs/apache-airflow-providers-apache-hive/commits\.rst$|
|
2021-03-29 14:18:57 +01:00
|
|
|
git|
|
2021-05-06 12:36:32 +10:00
|
|
|
^CHANGELOG\.txt$
|
2019-11-26 23:19:45 +01:00
|
|
|
- id: base-operator
|
|
|
|
|
language: pygrep
|
2020-12-26 02:24:27 -08:00
|
|
|
name: Check BaseOperator[Link] core imports
|
|
|
|
|
description: Make sure BaseOperator[Link] is imported from airflow.models.baseoperator in core
|
2019-12-01 11:14:52 +01:00
|
|
|
entry: "from airflow\\.models import.* BaseOperator"
|
2019-11-26 23:19:45 +01:00
|
|
|
files: \.py$
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
exclude: >
|
|
|
|
|
(?x)
|
2021-03-12 13:43:32 -08:00
|
|
|
^airflow/decorators/.*$|
|
2019-11-26 23:19:45 +01:00
|
|
|
^airflow/gcp/.*$|
|
|
|
|
|
^airflow/hooks/.*$|
|
|
|
|
|
^airflow/operators/.*$|
|
|
|
|
|
^airflow/sensors/.*$|
|
|
|
|
|
^airflow/providers/.*$|
|
2020-11-09 13:27:10 +01:00
|
|
|
^dev/provider_packages/.*$
|
2019-11-26 23:19:45 +01:00
|
|
|
- id: base-operator
|
|
|
|
|
language: pygrep
|
2020-12-26 02:24:27 -08:00
|
|
|
name: Check BaseOperator[Link] other imports
|
|
|
|
|
description: Make sure BaseOperator[Link] is imported from airflow.models outside of core
|
2019-12-01 11:14:52 +01:00
|
|
|
entry: "from airflow\\.models\\.baseoperator import.* BaseOperator"
|
2019-11-26 23:19:45 +01:00
|
|
|
pass_filenames: true
|
|
|
|
|
files: >
|
|
|
|
|
(?x)
|
|
|
|
|
^airflow/gcp/.*$|
|
|
|
|
|
^airflow/hooks/.*$|
|
|
|
|
|
^airflow/operators/.*$|
|
|
|
|
|
^airflow/sensors/.*$|
|
2020-06-29 08:08:58 +02:00
|
|
|
^airflow/providers/.*\.py$
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2020-02-02 01:14:14 -08:00
|
|
|
- id: provide-create-sessions
|
|
|
|
|
language: pygrep
|
2020-12-26 02:24:27 -08:00
|
|
|
name: Check provide_session and create_session imports
|
|
|
|
|
description: provide_session and create_session should be imported from airflow.utils.session
|
|
|
|
|
to avoid import cycles.
|
2020-02-02 01:14:14 -08:00
|
|
|
entry: "from airflow\\.utils\\.db import.* (provide_session|create_session)"
|
|
|
|
|
files: \.py$
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2020-02-02 01:14:14 -08:00
|
|
|
pass_filenames: true
|
2020-02-02 20:48:03 +01:00
|
|
|
- id: incorrect-use-of-LoggingMixin
|
|
|
|
|
language: pygrep
|
|
|
|
|
name: Make sure LoggingMixin is not used alone
|
|
|
|
|
entry: "LoggingMixin\\(\\)"
|
|
|
|
|
files: \.py$
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2020-02-02 20:48:03 +01:00
|
|
|
pass_filenames: true
|
2020-02-03 03:25:49 +05:30
|
|
|
- id: daysago-import-check
|
|
|
|
|
language: pygrep
|
|
|
|
|
name: Make sure days_ago is imported from airflow.utils.dates
|
|
|
|
|
entry: "(airflow\\.){0,1}utils\\.dates\\.days_ago"
|
|
|
|
|
files: \.py$
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2020-02-03 03:25:49 +05:30
|
|
|
pass_filenames: true
|
2020-09-08 14:06:42 +02:00
|
|
|
- id: restrict-start_date
|
2020-07-25 00:16:25 +01:00
|
|
|
language: pygrep
|
|
|
|
|
name: "'start_date' should not be defined in default_args in example_dags"
|
2020-07-25 19:57:32 +01:00
|
|
|
entry: "default_args\\s*=\\s*{\\s*(\"|')start_date(\"|')|(\"|')start_date(\"|'):"
|
2021-05-06 12:36:32 +10:00
|
|
|
files: \.*example_dags.*\.py$
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2020-07-25 00:16:25 +01:00
|
|
|
pass_filenames: true
|
2020-04-22 14:38:56 +02:00
|
|
|
- id: check-integrations
|
|
|
|
|
name: Check if integration list is aligned
|
2020-07-16 18:05:35 +02:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh
|
2020-04-22 14:38:56 +02:00
|
|
|
language: system
|
|
|
|
|
pass_filenames: false
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^common/_common_values\.sh$|^breeze-complete$
|
2019-11-04 15:59:48 +01:00
|
|
|
- id: check-apache-license
|
|
|
|
|
name: Check if licenses are OK for Apache
|
2021-05-06 12:36:32 +10:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_check_license.sh
|
2019-11-04 15:59:48 +01:00
|
|
|
language: system
|
|
|
|
|
files: ^.*LICENSE.*$|^.*LICENCE.*$
|
|
|
|
|
pass_filenames: false
|
2020-01-08 21:48:22 +00:00
|
|
|
- id: airflow-config-yaml
|
|
|
|
|
name: Checks for consistency between config.yml and default_config.cfg
|
|
|
|
|
language: python
|
2020-07-16 18:05:35 +02:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py
|
2021-05-06 12:36:32 +10:00
|
|
|
files: config\.yml$|default_airflow\.cfg$|default\.cfg$
|
2020-01-08 21:48:22 +00:00
|
|
|
pass_filenames: false
|
2020-07-16 18:05:35 +02:00
|
|
|
require_serial: true
|
2020-01-08 21:48:22 +00:00
|
|
|
additional_dependencies: ['pyyaml']
|
2021-06-03 18:20:26 +01:00
|
|
|
- id: verify-db-migrations-documented
|
|
|
|
|
name: Verify that DB Migrations have been documented
|
|
|
|
|
language: python
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_migration_documented.py
|
|
|
|
|
files: ^airflow/migrations/versions/.*\.(py)$
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
2021-06-04 22:02:36 +02:00
|
|
|
- id: boring-cyborg
|
|
|
|
|
name: Checks for Boring Cyborg configuration consistency
|
|
|
|
|
language: python
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_boring_cyborg.py
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
|
|
|
|
additional_dependencies: ['pyyaml', 'termcolor==1.1.0', 'wcmatch==8.2']
|
2020-10-18 18:15:53 +01:00
|
|
|
- id: sort-in-the-wild
|
|
|
|
|
name: Sort INTHEWILD.md alphabetically
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_sort_in_the_wild.sh
|
|
|
|
|
language: system
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^\.pre-commit-config\.yaml$|^INTHEWILD\.md$
|
2020-10-18 18:15:53 +01:00
|
|
|
require_serial: true
|
2021-03-01 02:26:13 +10:00
|
|
|
- id: sort-spelling-wordlist
|
|
|
|
|
name: Sort alphabetically and uniquify spelling_wordlist.txt
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_sort_spelling_wordlist.sh
|
|
|
|
|
language: system
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^\.pre-commit-config\.yaml$|^docs/spelling_wordlist\.txt$
|
2021-03-01 02:26:13 +10:00
|
|
|
require_serial: true
|
2020-10-18 18:15:53 +01:00
|
|
|
- id: helm-lint
|
|
|
|
|
name: Lint Helm Chart
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_helm_lint.sh
|
|
|
|
|
language: system
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
files: ^chart
|
|
|
|
|
require_serial: true
|
|
|
|
|
- id: shellcheck
|
|
|
|
|
name: Check Shell scripts syntax correctness
|
|
|
|
|
language: docker_image
|
2021-04-20 12:00:16 +02:00
|
|
|
entry: koalaman/shellcheck:v0.7.2 -x -a
|
2022-02-01 12:21:09 +01:00
|
|
|
files: ^breeze$|^breeze-complete$|\.sh$|^hooks/build$|^hooks/push$|\.bash$
|
2020-10-20 09:21:28 +02:00
|
|
|
- id: stylelint
|
|
|
|
|
name: stylelint
|
|
|
|
|
entry: "stylelint"
|
|
|
|
|
language: node
|
|
|
|
|
files: ^airflow/www/.*\.(css|scss|sass)$
|
|
|
|
|
# Keep dependency versions in sync w/ airflow/www/package.json
|
|
|
|
|
additional_dependencies: ['stylelint@13.3.1', 'stylelint-config-standard@20.0.0']
|
2020-11-09 13:27:10 +01:00
|
|
|
- id: providers-init-file
|
2021-09-10 16:08:32 +02:00
|
|
|
name: Provider init file is missing
|
2020-11-09 13:27:10 +01:00
|
|
|
pass_filenames: false
|
|
|
|
|
always_run: true
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_check_providers_init.sh
|
|
|
|
|
language: system
|
2021-09-10 16:08:32 +02:00
|
|
|
- id: providers-subpackages-init-file
|
|
|
|
|
name: Provider subpackage init files are there
|
2021-09-08 04:50:15 -04:00
|
|
|
pass_filenames: false
|
|
|
|
|
always_run: true
|
2021-09-10 16:08:32 +02:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_check_providers_subpackages_all_have_init.py
|
2021-09-08 04:50:15 -04:00
|
|
|
language: python
|
|
|
|
|
require_serial: true
|
2020-11-17 15:44:46 +01:00
|
|
|
- id: provider-yamls
|
|
|
|
|
name: Validate providers.yaml files
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
|
|
|
|
|
language: python
|
|
|
|
|
require_serial: true
|
2021-05-06 12:36:32 +10:00
|
|
|
files: provider\.yaml$|scripts/ci/pre_commit/pre_commit_check_provider_yaml_files\.py$|^docs/
|
2021-08-27 22:33:31 +02:00
|
|
|
additional_dependencies:
|
|
|
|
|
- 'PyYAML==5.3.1'
|
|
|
|
|
- 'jsonschema==3.2.0'
|
2021-10-21 15:56:51 -07:00
|
|
|
- 'tabulate==0.8.8'
|
2021-08-27 22:33:31 +02:00
|
|
|
- 'jsonpath-ng==1.5.3'
|
2021-09-12 21:59:19 +02:00
|
|
|
- 'rich==10.9.0'
|
2020-10-20 09:21:28 +02:00
|
|
|
- id: mermaid
|
|
|
|
|
name: Generate mermaid images
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_mermaid.sh
|
|
|
|
|
language: system
|
|
|
|
|
files: \.mermaid$
|
|
|
|
|
require_serial: true
|
|
|
|
|
- id: pre-commit-descriptions
|
|
|
|
|
name: Check if pre-commits are described
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commits.sh
|
|
|
|
|
language: system
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^\.pre-commit-config\.yaml$|^STATIC_CODE_CHECKS\.rst|^breeze-complete$
|
2020-10-20 09:21:28 +02:00
|
|
|
require_serial: true
|
2020-12-26 02:24:27 -08:00
|
|
|
- id: pre-commit-hook-names
|
|
|
|
|
name: Ensure hook ids are not overly long
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commit_hook_names.py
|
|
|
|
|
args:
|
|
|
|
|
- --max-length=70
|
|
|
|
|
language: python
|
2022-01-27 23:04:44 +05:30
|
|
|
files: ^\.pre-commit-config\.yaml$|^scripts/ci/pre_commit/pre_commit_check_pre_commit_hook_names.py$
|
|
|
|
|
additional_dependencies: ['pyyaml', 'jinja2', 'black']
|
2020-12-26 02:24:27 -08:00
|
|
|
require_serial: true
|
|
|
|
|
pass_filenames: false
|
2020-11-11 17:13:57 +01:00
|
|
|
- id: airflow-providers-available
|
2020-12-26 02:24:27 -08:00
|
|
|
name: Checks providers available when declared by extras in setup.py
|
2020-11-11 17:13:57 +01:00
|
|
|
language: python
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py
|
2021-05-06 23:34:14 +02:00
|
|
|
files: setup\.py|^airflow/providers/.*\.py
|
2020-11-11 17:13:57 +01:00
|
|
|
pass_filenames: false
|
|
|
|
|
require_serial: true
|
2020-12-28 19:10:27 +01:00
|
|
|
additional_dependencies: ['rich']
|
2020-11-14 12:23:23 +01:00
|
|
|
- id: markdownlint
|
|
|
|
|
name: Run markdownlint
|
2021-05-06 12:36:32 +10:00
|
|
|
description: Checks the style of Markdown files.
|
2020-11-14 12:23:23 +01:00
|
|
|
entry: markdownlint
|
|
|
|
|
language: node
|
|
|
|
|
types: [markdown]
|
|
|
|
|
files: \.(md|mdown|markdown)$
|
|
|
|
|
additional_dependencies: ['markdownlint-cli']
|
2020-11-28 12:12:54 +01:00
|
|
|
- id: json-schema
|
|
|
|
|
name: Lint JSON Schema files with JSON Schema
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
|
|
|
|
|
args:
|
|
|
|
|
- --spec-url
|
|
|
|
|
- https://json-schema.org/draft-07/schema
|
|
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
2021-01-11 23:10:44 +01:00
|
|
|
files: .*\.schema\.json$
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2020-11-28 12:12:54 +01:00
|
|
|
require_serial: true
|
|
|
|
|
additional_dependencies: ['jsonschema==3.2.0', 'PyYAML==5.3.1', 'requests==2.25.0']
|
2020-12-04 16:51:59 +01:00
|
|
|
- id: json-schema
|
2021-05-17 16:25:05 -06:00
|
|
|
name: Lint NodePort Service with JSON Schema
|
2020-12-04 16:51:59 +01:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
|
|
|
|
|
args:
|
|
|
|
|
- --spec-url
|
2021-11-24 23:29:46 +01:00
|
|
|
- https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.20.2-standalone/service-v1.json
|
2020-12-04 16:51:59 +01:00
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
2021-05-06 23:34:14 +02:00
|
|
|
files: scripts/ci/kubernetes/nodeport.yaml
|
2020-12-04 16:51:59 +01:00
|
|
|
require_serial: true
|
|
|
|
|
additional_dependencies: ['jsonschema==3.2.0', 'PyYAML==5.3.1', 'requests==2.25.0']
|
|
|
|
|
- id: json-schema
|
|
|
|
|
name: Lint Docker compose files with JSON Schema
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
|
|
|
|
|
args:
|
|
|
|
|
- --spec-url
|
|
|
|
|
- https://raw.githubusercontent.com/compose-spec/compose-spec/master/schema/compose-spec.json
|
|
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
2021-05-06 12:36:32 +10:00
|
|
|
files: ^scripts/ci/docker-compose/.+\.ya?ml$|docker-compose\.ya?ml$
|
2020-12-04 16:51:59 +01:00
|
|
|
require_serial: true
|
|
|
|
|
additional_dependencies: ['jsonschema==3.2.0', 'PyYAML==5.3.1', 'requests==2.25.0']
|
2021-05-13 14:58:04 -06:00
|
|
|
- id: json-schema
|
|
|
|
|
name: Lint chart/values.schema.json file with JSON Schema
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
|
|
|
|
|
args:
|
|
|
|
|
- --spec-file
|
|
|
|
|
- chart/values_schema.schema.json
|
2021-05-17 15:44:17 -06:00
|
|
|
- chart/values.schema.json
|
2021-05-13 14:58:04 -06:00
|
|
|
language: python
|
2021-05-17 15:44:17 -06:00
|
|
|
pass_filenames: false
|
|
|
|
|
files: ^chart/values\.schema\.json$|^chart/values_schema\.schema\.json$
|
2021-05-13 14:58:04 -06:00
|
|
|
require_serial: true
|
|
|
|
|
additional_dependencies: ['jsonschema==3.2.0', 'PyYAML==5.3.1', 'requests==2.25.0']
|
2021-12-29 12:29:07 -07:00
|
|
|
- id: vendor-k8s-json-schema
|
|
|
|
|
name: Vendor k8s definitions into values.schema.json
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_vendor_k8s_json_schema.py
|
|
|
|
|
language: python
|
|
|
|
|
files: ^chart/values\.schema\.json$
|
|
|
|
|
additional_dependencies: ['requests==2.25.0']
|
2020-12-04 16:51:59 +01:00
|
|
|
- id: json-schema
|
|
|
|
|
name: Lint chart/values.yaml file with JSON Schema
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
|
|
|
|
|
args:
|
2021-05-13 14:58:04 -06:00
|
|
|
- --enforce-defaults
|
2020-12-04 16:51:59 +01:00
|
|
|
- --spec-file
|
|
|
|
|
- chart/values.schema.json
|
2021-05-17 15:44:17 -06:00
|
|
|
- chart/values.yaml
|
2020-12-04 16:51:59 +01:00
|
|
|
language: python
|
2021-05-17 15:44:17 -06:00
|
|
|
pass_filenames: false
|
|
|
|
|
files: ^chart/values\.yaml$|^chart/values\.schema\.json$
|
2020-12-04 16:51:59 +01:00
|
|
|
require_serial: true
|
|
|
|
|
additional_dependencies: ['jsonschema==3.2.0', 'PyYAML==5.3.1', 'requests==2.25.0']
|
2020-12-15 20:49:06 -05:00
|
|
|
- id: json-schema
|
|
|
|
|
name: Lint airflow/config_templates/config.yml file with JSON Schema
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
|
|
|
|
|
args:
|
|
|
|
|
- --spec-file
|
|
|
|
|
- airflow/config_templates/config.yml.schema.json
|
|
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
2021-05-06 23:34:14 +02:00
|
|
|
files: airflow/config_templates/config\.yml$
|
2020-12-15 20:49:06 -05:00
|
|
|
require_serial: true
|
|
|
|
|
additional_dependencies: ['jsonschema==3.2.0', 'PyYAML==5.3.1', 'requests==2.25.0']
|
2021-12-20 22:22:54 +01:00
|
|
|
- id: persist-credentials-disabled
|
|
|
|
|
name: Check that workflow files have persist-credentials disabled
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py
|
|
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
files: \.github/workflows/.*\.yml$
|
|
|
|
|
additional_dependencies: ['PyYAML', 'rich']
|
2022-02-08 15:07:54 -08:00
|
|
|
- id: docstring-params
|
|
|
|
|
name: Check that docstrings do not specify param types
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_docstring_param_type.py
|
|
|
|
|
language: python
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
files: \.py$
|
|
|
|
|
pass_filenames: true
|
|
|
|
|
exclude: ^airflow/_vendor/
|
|
|
|
|
additional_dependencies: ['rich']
|
2021-12-13 22:49:12 +01:00
|
|
|
- id: chart-schema-lint
|
|
|
|
|
name: Lint chart/values.schema.json file
|
|
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_chart_schema.py
|
|
|
|
|
language: python
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
files: ^chart/values\.schema\.json$
|
|
|
|
|
require_serial: true
|
2021-06-20 14:32:11 +02:00
|
|
|
- id: ui-lint
|
|
|
|
|
name: ESLint against airflow/ui
|
|
|
|
|
language: node
|
|
|
|
|
'types_or': [javascript, tsx, ts]
|
|
|
|
|
files: ^airflow/ui/
|
|
|
|
|
entry: scripts/ci/static_checks/ui_lint.sh
|
|
|
|
|
pass_filenames: false
|
|
|
|
|
- id: www-lint
|
|
|
|
|
name: ESLint against current UI js files
|
|
|
|
|
language: node
|
|
|
|
|
'types_or': [javascript]
|
|
|
|
|
files: ^airflow/www/static/js/
|
|
|
|
|
entry: scripts/ci/static_checks/www_lint.sh
|
|
|
|
|
pass_filenames: false
|
2021-11-23 12:35:17 -07:00
|
|
|
- id: changelog-duplicates
|
|
|
|
|
name: Check changelogs for duplicate entries
|
|
|
|
|
language: python
|
|
|
|
|
files: CHANGELOG\.txt$|CHANGELOG\.rst$
|
|
|
|
|
entry: scripts/ci/pre_commit/pre_commit_changelog_duplicates.py
|
|
|
|
|
pass_filenames: true
|
2020-11-14 12:23:23 +01:00
|
|
|
## ADD MOST PRE-COMMITS ABOVE THAT LINE
|
|
|
|
|
# The below pre-commits are those requiring CI image to be built
|
2019-08-14 20:53:28 -04:00
|
|
|
- id: mypy
|
|
|
|
|
name: Run mypy
|
|
|
|
|
language: system
|
2021-12-01 00:23:50 +01:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_mypy.sh --namespace-packages
|
2019-08-14 20:53:28 -04:00
|
|
|
files: \.py$
|
2021-12-01 00:23:50 +01:00
|
|
|
exclude: ^provider_packages|^chart|^docs|^airflow/_vendor/
|
|
|
|
|
require_serial: true
|
2020-10-27 18:29:47 -07:00
|
|
|
- id: mypy
|
|
|
|
|
name: Run mypy for helm chart tests
|
|
|
|
|
language: system
|
2021-05-06 12:36:32 +10:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_mypy.sh
|
2020-10-27 18:29:47 -07:00
|
|
|
files: ^chart/.*\.py$
|
2021-12-01 00:23:50 +01:00
|
|
|
require_serial: true
|
2020-11-15 00:50:30 +01:00
|
|
|
- id: mypy
|
|
|
|
|
name: Run mypy for /docs/ folder
|
|
|
|
|
language: system
|
2021-05-06 12:36:32 +10:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_mypy.sh
|
2020-11-15 00:50:30 +01:00
|
|
|
files: ^docs/.*\.py$
|
2020-11-23 09:52:24 +01:00
|
|
|
exclude: rtd-deprecation
|
2021-12-01 00:23:50 +01:00
|
|
|
require_serial: true
|
2019-08-14 20:53:28 -04:00
|
|
|
- id: flake8
|
|
|
|
|
name: Run flake8
|
|
|
|
|
language: system
|
2021-05-06 12:36:32 +10:00
|
|
|
entry: ./scripts/ci/pre_commit/pre_commit_flake8.sh
|
2019-08-14 20:53:28 -04:00
|
|
|
files: \.py$
|
2019-09-18 13:43:30 +02:00
|
|
|
pass_filenames: true
|
2021-05-17 12:10:54 +02:00
|
|
|
exclude: ^airflow/_vendor/
|
2020-11-14 12:23:23 +01:00
|
|
|
## ONLY ADD PRE-COMMITS HERE THAT REQUIRE CI IMAGE
|
