├── .readthedocs.yml ├── docker ├── entrypoint.sh ├── hooks │ └── build └── ci.Dockerfile ├── images ├── pylint-airflow.png └── pylint-airflow.psd ├── .gitignore ├── .dockerignore ├── tests ├── pylint_airflow │ ├── integration │ │ ├── scripts │ │ │ ├── test_no_dag.py │ │ │ ├── test_dagid_filename.py │ │ │ ├── test_dagcontextmanager_filename.py │ │ │ ├── test_multi_dag_cm.py │ │ │ ├── test_modelsdag_filename.py │ │ │ ├── test_modelsdagcontextmanager_filename.py │ │ │ ├── test_nested_dag_cm.py │ │ │ ├── test_dag_nokwarg.py │ │ │ ├── test_airflowmodelsdag_filename.py │ │ │ ├── test_airflowmodelsdagcontextmanager_filename.py │ │ │ └── test_dag_mixed_assignment.py │ │ └── test_integration.py │ └── checkers │ │ ├── test_dag.py │ │ ├── test_xcom.py │ │ └── test_operator.py └── conftest.py ├── requirements.txt ├── src └── pylint_airflow │ ├── __pkginfo__.py │ ├── __init__.py │ └── checkers │ ├── __init__.py │ ├── xcom.py │ ├── operator.py │ └── dag.py ├── docs ├── index.rst ├── Makefile ├── make.bat └── conf.py ├── .circleci └── config.yml ├── LICENSE ├── setup.py ├── Makefile ├── scripts ├── ci_validate_msg_ids.py └── generate_codes_table.py ├── CONTRIBUTING.rst ├── README.rst └── .pylintrc /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sphinx: 4 | configuration: docs/conf.py 5 | -------------------------------------------------------------------------------- /docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | pip install . 4 | make ci 5 | -------------------------------------------------------------------------------- /images/pylint-airflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BasPH/pylint-airflow/HEAD/images/pylint-airflow.png -------------------------------------------------------------------------------- /images/pylint-airflow.psd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BasPH/pylint-airflow/HEAD/images/pylint-airflow.psd -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info/ 2 | .cache/ 3 | .DS_Store 4 | .idea/ 5 | .pylint.d/ 6 | .pytest_cache/ 7 | __pycache__/ 8 | build/ 9 | dist/ 10 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .cache/ 2 | .circleci/ 3 | .git/ 4 | .gitignore 5 | build/ 6 | CONTRIBUTING.rst 7 | dist/ 8 | docker/ 9 | docs/ 10 | images/ 11 | LICENSE 12 | -------------------------------------------------------------------------------- /tests/pylint_airflow/integration/scripts/test_no_dag.py: -------------------------------------------------------------------------------- 1 | """Verify if no failures when no DAG object is found.""" 2 | # pylint: disable=invalid-name 3 | 4 | foobar = "bla" 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | apache-airflow~=1.10.2 2 | black>=18.9b0 3 | pytest-helpers-namespace==2019.1.8 4 | pytest-mock==1.10.1 5 | pytest~=4.3 6 | sphinx-rtd-theme~=0.4.3 7 | Sphinx~=2.0.0 8 | -------------------------------------------------------------------------------- /src/pylint_airflow/__pkginfo__.py: -------------------------------------------------------------------------------- 1 | """pkginfo, following the PyLint repository style.""" 2 | 3 | # All base ids: https://github.com/PyCQA/pylint/blob/master/pylint/checkers/__init__.py 4 | BASE_ID = 83 5 | -------------------------------------------------------------------------------- /docker/hooks/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note: current dir is the dir containing Dockerfile (configured in Docker Hub) 4 | docker build --build-arg PYTHON_VERSION=${DOCKER_TAG} -f ci.Dockerfile -t $IMAGE_NAME .. 5 | -------------------------------------------------------------------------------- /src/pylint_airflow/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Pylint looks for a "register" attribute here. 3 | https://github.com/PyCQA/pylint/blob/pylint-2.3.1/pylint/utils.py#L1311-L1337 4 | """ 5 | from pylint_airflow.checkers import register_checkers as register 6 | -------------------------------------------------------------------------------- /tests/pylint_airflow/integration/scripts/test_dagid_filename.py: -------------------------------------------------------------------------------- 1 | # [match-dagid-filename] 2 | """Explicit DAG assignment, checking DAG()""" 3 | # pylint: disable=invalid-name 4 | 5 | from airflow.models import DAG 6 | 7 | dag = DAG(dag_id="foobar") 8 | -------------------------------------------------------------------------------- /tests/pylint_airflow/integration/scripts/test_dagcontextmanager_filename.py: -------------------------------------------------------------------------------- 1 | # [match-dagid-filename] 2 | """DAG assignment via context manager, checking DAG()""" 3 | 4 | from airflow.models import DAG 5 | 6 | with DAG(dag_id="foobar") as dag: 7 | pass 8 | -------------------------------------------------------------------------------- /tests/pylint_airflow/integration/scripts/test_multi_dag_cm.py: -------------------------------------------------------------------------------- 1 | """Multiple DAG assignment via context manager, checking DAG()""" 2 | 3 | from airflow.models import DAG 4 | 5 | with DAG(dag_id="foobar") as dag1, DAG(dag_id="foobar2") as dag2: 6 | pass 7 | -------------------------------------------------------------------------------- /tests/pylint_airflow/integration/scripts/test_modelsdag_filename.py: -------------------------------------------------------------------------------- 1 | # [match-dagid-filename] 2 | """Explicit DAG assignment, checking models.DAG()""" 3 | # pylint: disable=invalid-name 4 | 5 | from airflow import models 6 | 7 | dag = models.DAG(dag_id="foobar") 8 | -------------------------------------------------------------------------------- /tests/pylint_airflow/integration/scripts/test_modelsdagcontextmanager_filename.py: -------------------------------------------------------------------------------- 1 | # [match-dagid-filename] 2 | """DAG assignment via context manager, checking models.DAG()""" 3 | 4 | from airflow import models 5 | 6 | with models.DAG(dag_id="foobar") as dag: 7 | pass 8 | -------------------------------------------------------------------------------- /tests/pylint_airflow/integration/scripts/test_nested_dag_cm.py: -------------------------------------------------------------------------------- 1 | """Nested DAG assignment via context manager, checking DAG()""" 2 | 3 | from airflow.models import DAG 4 | 5 | with DAG(dag_id="foobar") as dag1: 6 | with DAG(dag_id="foobar2") as dag2: 7 | pass 8 | -------------------------------------------------------------------------------- /tests/pylint_airflow/integration/scripts/test_dag_nokwarg.py: -------------------------------------------------------------------------------- 1 | # [match-dagid-filename] 2 | # pylint: disable=invalid-name 3 | """Explicit DAG assignment, checking DAG(), passing dag_id via non-keyword arg.""" 4 | 5 | from airflow.models import DAG 6 | 7 | dag = DAG("foobar") 8 | -------------------------------------------------------------------------------- /tests/pylint_airflow/integration/scripts/test_airflowmodelsdag_filename.py: -------------------------------------------------------------------------------- 1 | # [match-dagid-filename] 2 | """Explicit DAG assignment, checking airflow.models.DAG()""" 3 | # pylint: disable=invalid-name 4 | 5 | import airflow.models 6 | 7 | dag = airflow.models.DAG(dag_id="foobar") 8 | -------------------------------------------------------------------------------- /tests/pylint_airflow/integration/scripts/test_airflowmodelsdagcontextmanager_filename.py: -------------------------------------------------------------------------------- 1 | # [match-dagid-filename] 2 | """DAG assignment via context manager, checking airflow.models.DAG()""" 3 | 4 | import airflow.models 5 | 6 | with airflow.models.DAG(dag_id="foobar") as dag: 7 | pass 8 | -------------------------------------------------------------------------------- /tests/pylint_airflow/integration/scripts/test_dag_mixed_assignment.py: -------------------------------------------------------------------------------- 1 | """Mixed DAG assignment, explicit and via context manager, checking DAG()""" 2 | # pylint: disable=invalid-name 3 | 4 | from airflow.models import DAG 5 | 6 | dag1 = DAG(dag_id="foobar") 7 | with DAG(dag_id="foobar2") as dag2: 8 | pass 9 | -------------------------------------------------------------------------------- /src/pylint_airflow/checkers/__init__.py: -------------------------------------------------------------------------------- 1 | """Checkers""" 2 | 3 | from pylint_airflow.checkers.dag import DagChecker 4 | from pylint_airflow.checkers.operator import OperatorChecker 5 | from pylint_airflow.checkers.xcom import XComChecker 6 | 7 | 8 | def register_checkers(linter): 9 | """Register checkers.""" 10 | linter.register_checker(DagChecker(linter)) 11 | linter.register_checker(OperatorChecker(linter)) 12 | linter.register_checker(XComChecker(linter)) 13 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. pylint-airflow documentation master file, created by 2 | sphinx-quickstart on Fri Apr 5 14:57:12 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to pylint-airflow's documentation! 7 | ========================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | 14 | 15 | Indices and tables 16 | ================== 17 | 18 | * :ref:`genindex` 19 | * :ref:`modindex` 20 | * :ref:`search` 21 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = . 8 | BUILDDIR = _build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docker/ci.Dockerfile: -------------------------------------------------------------------------------- 1 | ARG PYTHON_VERSION 2 | FROM python:${PYTHON_VERSION} 3 | 4 | WORKDIR /pylint-airflow 5 | # Note: only copy requirements.txt which contains development dependencies. The repository itself 6 | # should be mounted to the container at runtime. 7 | COPY requirements.txt requirements.txt 8 | 9 | # As long as Airflow requires this GPL dependency we have to install with SLUGIFY_USES_TEXT_UNIDECODE=yes 10 | # https://github.com/apache/airflow/pull/4513 11 | RUN apt-get update && \ 12 | apt-get install -y gcc g++ make --no-install-recommends && \ 13 | SLUGIFY_USES_TEXT_UNIDECODE=yes pip install -r requirements.txt && \ 14 | apt-get remove -y --purge gcc g++ && \ 15 | apt-get autoremove -y && \ 16 | apt-get clean -y && \ 17 | rm -rf /var/lib/apt/lists/* 18 | -------------------------------------------------------------------------------- /tests/pylint_airflow/integration/test_integration.py: -------------------------------------------------------------------------------- 1 | """Run pylint-airflow on all *.py files found in scripts/""" 2 | 3 | import os 4 | import pathlib 5 | 6 | import pytest 7 | 8 | match_dagid_filename_testfilesdir = os.path.join(pytest.helpers.file_abspath(__file__), "scripts") 9 | match_dagid_filename_testfiles = list( 10 | pathlib.Path(match_dagid_filename_testfilesdir).glob("**/*.py") 11 | ) 12 | 13 | 14 | @pytest.mark.parametrize( 15 | "test_filepath", 16 | match_dagid_filename_testfiles, 17 | ids=[ 18 | str(p.relative_to(match_dagid_filename_testfilesdir)) 19 | for p in match_dagid_filename_testfiles 20 | ], 21 | ) 22 | def test_match_dagid_filename(test_filepath): 23 | """Run pylint-airflow on file, given filepath.""" 24 | pytest.helpers.functional_test(test_filepath) 25 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | # These steps are executed for all Python versions 4 | _steps: &steps 5 | steps: 6 | - checkout 7 | - run: 8 | name: Update requirements 9 | command: pip install -r requirements.txt 10 | - run: 11 | name: Pip install Pylint-Airflow package 12 | command: pip install . 13 | - run: 14 | name: Check Black formatting 15 | command: make black 16 | - run: 17 | name: Check Pylint 18 | command: make pylint 19 | - run: 20 | name: Validate Pylint-Airflow message IDs 21 | command: make validate_message_ids 22 | - run: 23 | name: Run tests 24 | command: make pytest 25 | 26 | jobs: 27 | python36: 28 | docker: 29 | - image: basph/pylint-airflow-ci:3.6-slim 30 | <<: *steps 31 | python37: 32 | docker: 33 | - image: basph/pylint-airflow-ci:3.7-slim 34 | <<: *steps 35 | 36 | workflows: 37 | version: 2 38 | validate: 39 | jobs: 40 | - python36 41 | - python37 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Bas Harenslak 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Setup of pylint-airflow package""" 2 | from pathlib import Path 3 | 4 | from setuptools import setup, find_packages 5 | 6 | requirements = ["pylint"] 7 | 8 | readme = Path(__file__).resolve().parent / "README.rst" 9 | with open(readme) as f: 10 | long_description = f.read() 11 | 12 | setup( 13 | name="pylint-airflow", 14 | url="https://github.com/BasPH/pylint-airflow", 15 | description="A Pylint plugin to lint Apache Airflow code.", 16 | long_description=long_description, 17 | long_description_content_type="text/x-rst", 18 | version="0.1.0-alpha.1", 19 | packages=find_packages(where="src"), 20 | package_dir={"": "src"}, 21 | install_requires=requirements, 22 | keywords=["pylint", "airflow", "plugin"], 23 | classifiers=[ 24 | "Development Status :: 3 - Alpha", 25 | "Intended Audience :: Developers", 26 | "License :: OSI Approved :: MIT License", 27 | "Natural Language :: English", 28 | "Programming Language :: Python :: 3", 29 | "Programming Language :: Python :: 3.6", 30 | "Programming Language :: Python :: 3.7", 31 | ], 32 | ) 33 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: black 2 | black: 3 | find . -name '*.py' | xargs black --check --line-length=100 4 | 5 | .PHONY: pylint 6 | pylint: 7 | find . -name '*.py' | xargs pylint --output-format=colorized 8 | 9 | .PHONY: validate_message_ids 10 | validate_message_ids: 11 | python scripts/ci_validate_msg_ids.py 12 | 13 | .PHONY: pytest 14 | pytest: 15 | pytest tests/ -W ignore::DeprecationWarning 16 | 17 | .PHONY: clean-compiled 18 | clean-compiled: # Remove Python artifacts 19 | find . -name '*.pyc' -exec rm -f {} + 20 | find . -name '__pycache__' -exec rm -rf {} + 21 | 22 | .PHONY: ci 23 | ci: | clean-compiled black pylint validate_message_ids pytest 24 | 25 | .PHONY: ci-docker 26 | ci-docker: build_ci_image 27 | docker run -ti -v `pwd`:/pylint-airflow -w /pylint-airflow basph/pylint-airflow-ci:3.6-slim pip install .; make ci 28 | docker run -ti -v `pwd`:/pylint-airflow -w /pylint-airflow basph/pylint-airflow-ci:3.7-slim pip install .; make ci 29 | 30 | .PHONY: build_ci_image 31 | build_ci_image: 32 | docker build --file docker/ci.Dockerfile --build-arg PYTHON_VERSION=3.6-slim --tag basph/pylint-airflow-ci:3.6-slim . 33 | docker build --file docker/ci.Dockerfile --build-arg PYTHON_VERSION=3.7-slim --tag basph/pylint-airflow-ci:3.7-slim . 34 | 35 | .PHONY: upload-to-pypi 36 | upload-to-pypi: 37 | rm -rf dist/ 38 | python setup.py sdist bdist_wheel 39 | twine check dist/* 40 | twine upload dist/* 41 | -------------------------------------------------------------------------------- /tests/pylint_airflow/checkers/test_dag.py: -------------------------------------------------------------------------------- 1 | """Tests for the DAG checker.""" 2 | 3 | import astroid 4 | from pylint.testutils import CheckerTestCase, Message 5 | 6 | import pylint_airflow 7 | 8 | 9 | class TestDagChecker(CheckerTestCase): 10 | """Tests for the DAG checker.""" 11 | 12 | CHECKER_CLASS = pylint_airflow.checkers.dag.DagChecker 13 | 14 | def test_duplicate_dag(self): 15 | """Test for multiple DAG instances with identical names.""" 16 | testcase = """ 17 | from airflow import models 18 | from airflow.models import DAG 19 | 20 | dagname = "test" 21 | 22 | dag = models.DAG(dag_id="lintme") 23 | dag2 = DAG(dag_id="lintme") 24 | dag3 = DAG(dag_id=dagname) # test dag_id from variable 25 | dag4 = DAG(dag_id=f"{dagname}foo") # test dag_id as f-string 26 | """ 27 | ast = astroid.parse(testcase) 28 | expected_msg_node = ast.body[4].value 29 | with self.assertAddsMessages( 30 | Message(msg_id="duplicate-dag-name", node=expected_msg_node, args="lintme") 31 | ): 32 | self.checker.visit_module(ast) 33 | 34 | def test_no_duplicate_dag(self): 35 | """Test for multiple DAG instances without identical names - this should be fine.""" 36 | testcase = """ 37 | from airflow.models import DAG 38 | dag = DAG(dag_id="lintme") 39 | dag2 = DAG(dag_id="foobar") 40 | """ 41 | ast = astroid.parse(testcase) 42 | with self.assertNoMessages(): 43 | self.checker.visit_module(ast) 44 | -------------------------------------------------------------------------------- /tests/pylint_airflow/checkers/test_xcom.py: -------------------------------------------------------------------------------- 1 | """Tests for the XCom checker.""" 2 | 3 | import astroid 4 | from pylint.testutils import CheckerTestCase, Message 5 | 6 | import pylint_airflow 7 | 8 | 9 | class TestXComChecker(CheckerTestCase): 10 | """Tests for the XCom checker.""" 11 | 12 | CHECKER_CLASS = pylint_airflow.checkers.xcom.XComChecker 13 | 14 | def test_used_xcom(self): 15 | """Test valid case: _pushtask() returns a value and _pulltask pulls and uses it.""" 16 | testcase = """ 17 | from airflow.operators.python_operator import PythonOperator 18 | 19 | def _pushtask(): 20 | print("do stuff") 21 | return "foobar" 22 | 23 | pushtask = PythonOperator(task_id="pushtask", python_callable=_pushtask) 24 | 25 | def _pulltask(task_instance, **_): 26 | print(task_instance.xcom_pull(task_ids="pushtask")) 27 | 28 | pulltask = PythonOperator(task_id="pulltask", python_callable=_pulltask, provide_context=True) 29 | """ 30 | ast = astroid.parse(testcase) 31 | with self.assertNoMessages(): 32 | self.checker.visit_module(ast) 33 | 34 | def test_unused_xcom(self): 35 | """Test invalid case: _pushtask() returns a value but it's never used.""" 36 | testcase = """ 37 | from airflow.operators.python_operator import PythonOperator 38 | 39 | def _pushtask(): 40 | print("do stuff") 41 | return "foobar" 42 | 43 | pushtask = PythonOperator(task_id="pushtask", python_callable=_pushtask) 44 | 45 | def _pulltask(): 46 | print("foobar") 47 | 48 | pulltask = PythonOperator(task_id="pulltask", python_callable=_pulltask) 49 | """ 50 | ast = astroid.parse(testcase) 51 | expected_msg_node = ast.body[2].value 52 | expected_args = "_pushtask" 53 | with self.assertAddsMessages( 54 | Message(msg_id="unused-xcom", node=expected_msg_node, args=expected_args) 55 | ): 56 | self.checker.visit_module(ast) 57 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | """ 2 | Configuration file for the Sphinx documentation builder. 3 | 4 | This file only contains a selection of the most common options. For a full 5 | list see the documentation: 6 | http://www.sphinx-doc.org/en/master/config 7 | 8 | -- Path setup -------------------------------------------------------------- 9 | 10 | If extensions (or modules to document with autodoc) are in another directory, 11 | add these directories to sys.path here. If the directory is relative to the 12 | documentation root, use os.path.abspath to make it absolute, like shown here. 13 | 14 | import os 15 | import sys 16 | sys.path.insert(0, os.path.abspath('.')) 17 | """ 18 | # pylint: disable=redefined-builtin 19 | 20 | # -- Project information ----------------------------------------------------- 21 | 22 | project = "pylint-airflow" 23 | copyright = "2019, Bas Harenslak" 24 | author = "Bas Harenslak" 25 | 26 | # The full version, including alpha/beta/rc tags 27 | release = "0.0.1-alpha" 28 | 29 | 30 | # -- General configuration --------------------------------------------------- 31 | 32 | # The master toctree document. 33 | master_doc = "index" 34 | 35 | # Add any Sphinx extension module names here, as strings. They can be 36 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 37 | # ones. 38 | extensions = [] 39 | 40 | # Add any paths that contain templates here, relative to this directory. 41 | templates_path = ["_templates"] 42 | 43 | # List of patterns, relative to source directory, that match files and 44 | # directories to ignore when looking for source files. 45 | # This pattern also affects html_static_path and html_extra_path. 46 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 47 | 48 | 49 | # -- Options for HTML output ------------------------------------------------- 50 | 51 | # The theme to use for HTML and HTML Help pages. See the documentation for 52 | # a list of builtin themes. 53 | # 54 | html_theme = "sphinx_rtd_theme" 55 | 56 | # Add any paths that contain custom static files (such as style sheets) here, 57 | # relative to this directory. They are copied after the builtin static files, 58 | # so a file named "default.css" will overwrite the builtin "default.css". 59 | html_static_path = ["_static"] 60 | -------------------------------------------------------------------------------- /scripts/ci_validate_msg_ids.py: -------------------------------------------------------------------------------- 1 | """ 2 | This script is used by the CI to: 3 | 1. Validate if message ids are defined correctly (e.g. missed comma might evaluate correctly, but 4 | will be interpreted incorrectly). 5 | 2. For each message type, check if codes start at 0 and increment by 1, e.g. C8300, C8301, ... 6 | """ 7 | import os 8 | from collections import defaultdict 9 | from typing import List 10 | 11 | from pylint.lint import PyLinter 12 | 13 | from pylint_airflow.__pkginfo__ import BASE_ID 14 | from pylint_airflow.checkers import register_checkers 15 | 16 | 17 | def is_class_part_of_pylint_airflow(class_): 18 | """Expected input e.g. """ 19 | return class_.__module__.split(".")[0] == "pylint_airflow" 20 | 21 | 22 | def check_if_msg_ids_increment(message_ids: List[str]): 23 | """ 24 | Check if the message IDs (within 1 group) start at 0 and increment by 1. E.g. C8300, C8301, ... 25 | :param List[str] message_ids: Message IDs within single group 26 | """ 27 | 28 | # Fetch only last 2 characters of the id 29 | ids = sorted([int(msg_id[3:5]) for msg_id in message_ids]) 30 | 31 | # ids should start at 0, should be sorted, and should increment by 1. 32 | # So with e.g. 5 ids, check if the last id is 4. 33 | maxid = len(ids) - 1 34 | if ids[-1] != maxid: 35 | # Could come up with some sorting function for message_ids, but sorting numerically and 36 | # checking last id proved easier. 37 | formatted_message_ids = [f"{msg_type}{BASE_ID}{str(id_).zfill(2)}" for id_ in ids] 38 | raise AssertionError(f"Message ids should increment by 1. {formatted_message_ids}") 39 | 40 | 41 | def check_if_msg_ids_in_readme(message_ids: List[str]): 42 | """ 43 | Check if message IDs are listed in the README. 44 | :param List[str] message_ids: All message IDs found in pylint-airflow 45 | """ 46 | 47 | readme_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../README.rst") 48 | with open(readme_path, "r") as readme_: 49 | readme = readme_.read() 50 | 51 | not_found = [msg_id for msg_id in message_ids if msg_id not in readme] 52 | if not_found: 53 | raise AssertionError( 54 | f"Message IDs {not_found} not found in README. All message IDs should be documented." 55 | ) 56 | 57 | 58 | # Construct dict of {message type: [message ids]} 59 | messages = defaultdict(list) 60 | linter = PyLinter() 61 | register_checkers(linter) 62 | # Running register_checkers automatically validates there are no duplicate message ids 63 | for message in linter.msgs_store.messages: 64 | if is_class_part_of_pylint_airflow(message.checker): 65 | msg_type = message.msgid[0] 66 | messages[msg_type].append(message.msgid) 67 | 68 | for msg_type, msg_ids in messages.items(): 69 | check_if_msg_ids_increment(msg_ids) 70 | 71 | check_if_msg_ids_in_readme([msg_id for msg_list in messages.values() for msg_id in msg_list]) 72 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | """Globally accessible helpers functions/classes in all tests.""" 2 | 3 | import io 4 | import os 5 | 6 | import pytest 7 | from pylint.test.test_functional import ( 8 | LintModuleTest, 9 | FunctionalTestFile, 10 | multiset_difference, 11 | get_expected_messages, 12 | ) 13 | 14 | pytest_plugins = ["helpers_namespace"] 15 | 16 | 17 | class PylintAirflowLintModuleTest(LintModuleTest): 18 | """ 19 | Implemented this class because I didn't want tests in the pylint-airflow package itself. Had to 20 | do some yak-shaving to get it to work. 21 | 22 | Picked the useful parts from Pylint, and inspired by 23 | https://github.com/PyCQA/pylint-django/blob/2.0.5/pylint_django/tests/test_func.py#L18-L24 24 | 25 | With this class, you can now simply pass a file path, and run the test. 26 | Messages can be ignored in the file itself with '# [symbol]', e.g.: 27 | 28 | foobar = Magic() # [no-magic] 29 | 30 | Defining expected messages in a .txt is not supported with this. 31 | And ignore messages in the file itself with # pylint:disable=[symbol] 32 | """ 33 | 34 | def __init__(self, test_filepath): 35 | test_dirname = os.path.dirname(test_filepath) 36 | test_basename = os.path.basename(test_filepath) 37 | func_test = FunctionalTestFile(directory=test_dirname, filename=test_basename) 38 | super().__init__(func_test) 39 | 40 | self._test_filepath = test_filepath 41 | self._linter.load_plugin_modules(["pylint_airflow"]) 42 | 43 | def _get_expected(self): 44 | with io.open(self._test_filepath, encoding="utf8") as fobj: 45 | return get_expected_messages(fobj) 46 | 47 | def check_file(self): 48 | """Run Pylint on a file.""" 49 | self._linter.check(self._test_filepath) 50 | 51 | expected_msgs = self._get_expected() 52 | received_msgs, received_text = self._get_received() 53 | linesymbol_text = {(ol.lineno, ol.symbol): ol.msg for ol in received_text} 54 | 55 | if expected_msgs != received_msgs: 56 | missing, unexpected = multiset_difference(expected_msgs, received_msgs) 57 | msg = [f"Wrong results for file '{self._test_file.base}':"] 58 | if missing: 59 | msg.append("\nExpected in testdata:") 60 | msg.extend( 61 | f" {line_nr:3d}: {symbol} - {linesymbol_text[(line_nr, symbol)]}" 62 | for line_nr, symbol in sorted(missing) 63 | ) 64 | if unexpected: 65 | msg.append("\nUnexpected in testdata:") 66 | msg.extend( 67 | f" {line_nr:3d}: {symbol} - {linesymbol_text[(line_nr,symbol)]}" 68 | for line_nr, symbol in sorted(unexpected) 69 | ) 70 | pytest.fail("\n".join(msg)) 71 | 72 | 73 | @pytest.helpers.register 74 | def functional_test(filepath): 75 | """Run Pylint on a file, given the path to the file.""" 76 | lint_test = PylintAirflowLintModuleTest(filepath) 77 | lint_test.check_file() 78 | 79 | 80 | @pytest.helpers.register 81 | def file_abspath(file): 82 | """Fetch the absolute path to the directory of a file.""" 83 | return os.path.abspath(os.path.dirname(file)) 84 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ############ 2 | Contributing 3 | ############ 4 | 5 | Ideas and contributions are always welcome! Please create an issue or PR on `GitHub `_. 6 | 7 | ********** 8 | Tools used 9 | ********** 10 | 11 | - `Black `_ for formatting 12 | - `Pylint `_ for linting 13 | - `Pytest `_ for testing 14 | - `Read the Docs `_ for hosting the documentation 15 | - `Sphinx `_ for documentation 16 | - `CircleCI `_ for CI/CD 17 | 18 | ******** 19 | Makefile 20 | ******** 21 | 22 | The project contains a Makefile with a few commonly used targets. The Makefile and CircleCI should align so that you can run the full CI pipeline locally with just `make ci`. Other useful targets are (check the Makefile for all targets): 23 | 24 | - ``make black``: check Black formatting 25 | - ``make pylint``: check Pylint 26 | - ``make pytest``: run all tests 27 | - ``make validate_message_ids``: check if messages are defined without gaps between ID numbers 28 | 29 | *************** 30 | Adding messages 31 | *************** 32 | - Check the README for existing messages and add your new message with type and highest ID + 1. 33 | - To generate the table in the README, run ``python scripts/generate_codes_table.py`` and copy-paste in the README.rst. 34 | - The CI checks if message IDs are defined and increment by 1, without gaps. 35 | - Always add tests. 36 | - Run CI pipeline locally with ``make ci``. 37 | 38 | *********** 39 | Conventions 40 | *********** 41 | - Line length is set to 100 characters in .pylintrc and Makefile black target. 42 | - Documentation is written in reStructuredText (rst) format. 43 | - Versioning according to SemVer versioning (Read the Docs parses git tags against `PEP 440 `_ rules to version the documentation). 44 | - Checkers are organized by Airflow component (DAG, Operator, Hook, etc). 45 | 46 | *************** 47 | Getting started 48 | *************** 49 | If you're using PyCharm/IntelliJ, mark the ``src`` directory as "Sources Root". That way pylint-airflow is recognised as the local application, and "Optimize imports" groups those imports correctly. 50 | 51 | Some pointers if you're new to Pylint plugin development: 52 | 53 | - Check AST tokens on any script by running on the terminal: 54 | 55 | .. code-block:: python 56 | 57 | import astroid 58 | print(astroid.parse('''your script here''').repr_tree()) 59 | 60 | # For example: 61 | print(astroid.parse('''test = "foobar"''').repr_tree()) 62 | 63 | Module( 64 | name='', 65 | doc=None, 66 | file='', 67 | path=[''], 68 | package=False, 69 | pure_python=True, 70 | future_imports=set(), 71 | body=[Assign( 72 | targets=[AssignName(name='test')], 73 | value=Const(value='foobar'))]) 74 | 75 | - Or in a debugging session on any given AST node: 76 | 77 | .. code-block:: python 78 | 79 | print(node.repr_tree()) 80 | 81 | # For example: 82 | node = astroid.parse('''test = "foobar"''') 83 | print(node.repr_tree()) 84 | 85 | - Define ``visit_[token]`` or ``leave_[token]`` methods to process respective tokens, e.g. ``visit_module()``. 86 | - All available token types: http://pylint.pycqa.org/projects/astroid/en/latest/api/astroid.nodes.html. 87 | - Pylint searches for the variable ``msgs`` in a checker, make sure it is named exactly that. 88 | - Pylint lends itself nicely for test driven development: add one or more test cases (preferably both valid and invalid cases), and then implement a checker to run the test cases successfully. 89 | - Some useful resources to learn about Pylint: 90 | 91 | - `How to write Pylint plugins by Alexander Todorov - PiterPy 2018 `_ 92 | - `Pylint source code `_ 93 | - `Pylint Django plugin source code `_ 94 | -------------------------------------------------------------------------------- /src/pylint_airflow/checkers/xcom.py: -------------------------------------------------------------------------------- 1 | """Checks on Airflow XComs.""" 2 | 3 | import astroid 4 | from pylint import checkers 5 | from pylint import interfaces 6 | from pylint.checkers import utils 7 | 8 | from pylint_airflow.__pkginfo__ import BASE_ID 9 | 10 | 11 | class XComChecker(checkers.BaseChecker): 12 | """Checks on Airflow XComs.""" 13 | 14 | __implements__ = interfaces.IAstroidChecker 15 | 16 | msgs = { 17 | f"R{BASE_ID}00": ( 18 | "Return value from %s is stored as XCom but not used anywhere", 19 | "unused-xcom", 20 | "Return values from a python_callable function or execute() method are " 21 | "automatically pushed as XCom.", 22 | ) 23 | } 24 | 25 | @utils.check_messages("unused-xcom") 26 | def visit_module(self, node: astroid.Module): 27 | """ 28 | Check for unused XComs. 29 | XComs can be set (pushed) implicitly via return of a python_callable or 30 | execute() of an operator. And explicitly by calling xcom_push(). 31 | 32 | Currently this only checks unused XComs from return value of a python_callable. 33 | """ 34 | # pylint: disable=too-many-locals,too-many-branches,too-many-nested-blocks 35 | assign_nodes = [n for n in node.body if isinstance(n, astroid.Assign)] 36 | call_nodes = [n.value for n in assign_nodes if isinstance(n.value, astroid.Call)] 37 | 38 | # Store nodes containing python_callable arg as: 39 | # {task_id: (call node, python_callable func name)} 40 | python_callable_nodes = dict() 41 | for call_node in call_nodes: 42 | if call_node.keywords: 43 | task_id = "" 44 | python_callable = "" 45 | for keyword in call_node.keywords: 46 | if keyword.arg == "python_callable": 47 | python_callable = keyword.value.name 48 | continue 49 | elif keyword.arg == "task_id": 50 | task_id = keyword.value.value 51 | 52 | if python_callable: 53 | python_callable_nodes[task_id] = (call_node, python_callable) 54 | 55 | # Now fetch the functions mentioned by python_callable args 56 | xcoms_pushed = dict() 57 | xcoms_pulled_taskids = set() 58 | for (task_id, (python_callable, callable_func_name)) in python_callable_nodes.items(): 59 | if callable_func_name != "": 60 | # TODO support lambdas 61 | callable_func = node.getattr(callable_func_name)[0] 62 | 63 | if isinstance(callable_func, astroid.FunctionDef): 64 | # Callable_func is str not FunctionDef when imported 65 | callable_func = node.getattr(callable_func_name)[0] 66 | 67 | # Check if the function returns any values 68 | if any([isinstance(n, astroid.Return) for n in callable_func.body]): 69 | # Found a return statement 70 | xcoms_pushed[task_id] = (python_callable, callable_func_name) 71 | 72 | # Check if the function pulls any XComs 73 | callable_func_calls = callable_func.nodes_of_class(astroid.Call) 74 | for callable_func_call in callable_func_calls: 75 | if ( 76 | isinstance(callable_func_call.func, astroid.Attribute) 77 | and callable_func_call.func.attrname == "xcom_pull" 78 | ): 79 | for keyword in callable_func_call.keywords: 80 | if keyword.arg == "task_ids": 81 | xcoms_pulled_taskids.add(keyword.value.value) 82 | 83 | remainder = xcoms_pushed.keys() - xcoms_pulled_taskids 84 | if remainder: 85 | # There's a remainder in xcoms_pushed_taskids which should've been xcom_pulled. 86 | for remainder_task_id in remainder: 87 | python_callable, callable_func_name = xcoms_pushed[remainder_task_id] 88 | self.add_message("unused-xcom", node=python_callable, args=callable_func_name) 89 | 90 | # pylint: enable=too-many-locals,too-many-branches,too-many-nested-blocks 91 | -------------------------------------------------------------------------------- /scripts/generate_codes_table.py: -------------------------------------------------------------------------------- 1 | """ 2 | This script fetches all messages from the Pylint-Airflow plugin and writes into a 3 | Markdown table, to be copied into README.rst. 4 | 5 | Made a custom script because I wanted messages ordered by 6 | 1. pylint message type (I, C, R, W, E, F) and 7 | 2. message code 8 | 9 | For example: 10 | +-------+---------+-------------+ 11 | | Code | Symbol | Description | 12 | +=======+=========+=============+ 13 | | C8300 | symbol1 | Lorem ipsum | 14 | +-------+---------+-------------+ 15 | | C8301 | symbol2 | Lorem ipsum | 16 | +-------+---------+-------------+ 17 | | R8300 | symbol3 | Lorem ipsum | 18 | +-------+---------+-------------+ 19 | | E8300 | symbol3 | Lorem ipsum | 20 | +-------+---------+-------------+ 21 | | E8301 | symbol3 | Lorem ipsum | 22 | +-------+---------+-------------+ 23 | """ 24 | 25 | from collections import defaultdict 26 | from typing import List, Dict, Tuple 27 | 28 | from pylint.lint import PyLinter 29 | 30 | from pylint_airflow.checkers import register_checkers 31 | 32 | 33 | def is_class_part_of_pylint_airflow(class_): 34 | """Expected input e.g. """ 35 | return class_.__module__.split(".")[0] == "pylint_airflow" 36 | 37 | 38 | def _rst_escape_string(string: str) -> str: 39 | """ 40 | Escape special characters. 41 | :param str string: Input string 42 | :return: String with escaped characters 43 | """ 44 | return string.replace("*", "\\*") 45 | 46 | 47 | def gen_splitter(symbol: str, lengths: List[int]): 48 | """ 49 | Generate a "splitter" line for an rst table. 50 | E.g. +-------+---------+-------------+ 51 | :param str symbol: The character to use for filling cells 52 | :param List[int] lengths: The length of each cell to generate 53 | :return: Splitter line 54 | :rtype: str 55 | """ 56 | 57 | content = f"{symbol}+{symbol}".join(f"{symbol*nchars}" for nchars in lengths) 58 | return f"+{symbol}{content}{symbol}+" 59 | 60 | 61 | def gen_single_row(content: List, lengths: List[int]): 62 | """ 63 | Generate a row for an rst table. 64 | E.g. "| C8300 | symbol1 | Lorem ipsum |" 65 | :param List content: The values of each cell 66 | :param List[int] lengths: The length of each cell to generate 67 | :return: Table row with whitespace padded cells 68 | :rtype: str 69 | """ 70 | assert len(content) == len(lengths) 71 | content_length = list(zip(content, lengths)) 72 | row = " | ".join(f"{value.ljust(length)}" for value, length in content_length) 73 | return f"| {row} |" 74 | 75 | 76 | def gen_content(msgs: Dict[str, Dict[str, Tuple[str, str]]], lengths: List[int]): 77 | """ 78 | Generate the content part of an rst table. 79 | :param Dict[str, Dict[str, Tuple[str, str]]] msgs: all values for the table 80 | ({msg type: {msg number: (symbol, description)}}) 81 | :param List[int] lengths: The length of each cell to generate 82 | :return: Formatted table rows 83 | :rtype: str 84 | """ 85 | lines = [] 86 | splitter = gen_splitter(symbol="-", lengths=lengths) 87 | 88 | pylint_message_order = ["I", "C", "R", "W", "E", "F"] 89 | for msgid_char, char_msgs in sorted( 90 | msgs.items(), key=lambda i: pylint_message_order.index(i[0]) 91 | ): 92 | for msgid_nums, msg in sorted(char_msgs.items()): 93 | content = [msgid_char + msgid_nums, msg[0], msg[1]] 94 | lines.append(gen_single_row(content=content, lengths=lengths)) 95 | 96 | return f"\n{splitter}\n".join(lines) 97 | 98 | 99 | # Collect all pylint_airflow messages 100 | # Store messages as {"type": {"msgid numbers": (symbol, description)}} for easy sorting 101 | # E.g. {'E': {'8300': ('duplicate-dag-name', 'DAG name should be unique.')}} 102 | messages = defaultdict(dict) 103 | max_symbol_length = len("Symbol") 104 | max_description_length = len("Description") 105 | linter = PyLinter() 106 | register_checkers(linter) 107 | for message in linter.msgs_store.messages: 108 | if is_class_part_of_pylint_airflow(message.checker): 109 | description = _rst_escape_string(message.descr) 110 | messages[message.msgid[0]][message.msgid[-4:]] = (message.symbol, description) 111 | 112 | if len(message.symbol) > max_symbol_length: 113 | max_symbol_length = len(message.symbol) 114 | if len(message.descr) > max_description_length: 115 | max_description_length = len(message.descr) 116 | 117 | # Generate Markdown table 118 | col_lengths = [5, max_symbol_length, max_description_length] 119 | table = [ 120 | gen_splitter(symbol="-", lengths=col_lengths), 121 | gen_single_row(content=["Code", "Symbol", "Description"], lengths=col_lengths), 122 | gen_splitter(symbol="=", lengths=col_lengths), 123 | gen_content(msgs=messages, lengths=col_lengths), 124 | gen_splitter(symbol="-", lengths=col_lengths), 125 | ] 126 | result = "\n".join(table) 127 | 128 | print( 129 | "{color_red}Copy the following into README.rst:{no_color}\n".format( 130 | color_red="\x1b[1;31;40m", no_color="\x1b[0m" 131 | ) 132 | ) 133 | print(result) 134 | -------------------------------------------------------------------------------- /src/pylint_airflow/checkers/operator.py: -------------------------------------------------------------------------------- 1 | """Checks on Airflow operators.""" 2 | 3 | import astroid 4 | from pylint import checkers 5 | from pylint import interfaces 6 | from pylint.checkers import utils 7 | from pylint.checkers.utils import safe_infer 8 | 9 | from pylint_airflow.__pkginfo__ import BASE_ID 10 | 11 | 12 | class OperatorChecker(checkers.BaseChecker): 13 | """Checks on Airflow operators.""" 14 | 15 | __implements__ = (interfaces.IAstroidChecker,) 16 | 17 | msgs = { 18 | f"C{BASE_ID}00": ( 19 | "Operator variable name and task_id argument should match", 20 | "different-operator-varname-taskid", 21 | "For consistency assign the same variable name and task_id to operators.", 22 | ), 23 | f"C{BASE_ID}01": ( 24 | "Name the python_callable function '_[task_id]'", 25 | "match-callable-taskid", 26 | "For consistency name the callable function '_[task_id]', e.g. " 27 | "PythonOperator(task_id='mytask', python_callable=_mytask).", 28 | ), 29 | f"C{BASE_ID}02": ( 30 | "Avoid mixing task dependency directions", 31 | "mixed-dependency-directions", 32 | "For consistency don't mix directions in a single statement, instead split " 33 | "over multiple statements.", 34 | ), 35 | f"C{BASE_ID}03": ( 36 | "Task {} has no dependencies. Verify or disable message.", 37 | "task-no-dependencies", 38 | "Sometimes a task without any dependency is desired, however often it is " 39 | "the result of a forgotten dependency.", 40 | ), 41 | f"C{BASE_ID}04": ( 42 | "Rename **kwargs variable to **context to show intent for Airflow task context", 43 | "task-context-argname", 44 | "Indicate you expect Airflow task context variables in the **kwargs " 45 | "argument by renaming to **context.", 46 | ), 47 | f"C{BASE_ID}05": ( 48 | "Extract variables from keyword arguments for explicitness", 49 | "task-context-separate-arg", 50 | "To avoid unpacking kwargs from the Airflow task context in a function, you " 51 | "can set the needed variables as arguments in the function.", 52 | ), 53 | } 54 | 55 | @utils.check_messages("different-operator-varname-taskid", "match-callable-taskid") 56 | def visit_assign(self, node): 57 | """ 58 | TODO rewrite this 59 | Check if operators using python_callable argument call a function with name 60 | '_[task_id]'. For example: 61 | Valid -> 62 | def _mytask(): print("dosomething") 63 | mytask = PythonOperator(task_id="mytask", python_callable=_mytask) 64 | 65 | Invalid -> 66 | def invalidname(): print("dosomething") 67 | mytask = PythonOperator(task_id="mytask", python_callable=invalidname) 68 | """ 69 | if isinstance(node.value, astroid.Call): 70 | function_node = safe_infer(node.value.func) 71 | if ( 72 | function_node is not None 73 | and not isinstance(function_node, astroid.bases.BoundMethod) 74 | and hasattr(function_node, "is_subtype_of") 75 | and ( 76 | function_node.is_subtype_of("airflow.models.BaseOperator") 77 | or function_node.is_subtype_of("airflow.models.baseoperator.BaseOperator") 78 | ) 79 | ): 80 | var_name = node.targets[0].name 81 | task_id = None 82 | python_callable_name = None 83 | 84 | for keyword in node.value.keywords: 85 | if keyword.arg == "task_id" and isinstance(keyword.value, astroid.Const): 86 | # TODO support other values than constants 87 | task_id = keyword.value.value 88 | continue 89 | elif keyword.arg == "python_callable": 90 | python_callable_name = keyword.value.name 91 | 92 | if var_name != task_id: 93 | self.add_message("different-operator-varname-taskid", node=node) 94 | 95 | if python_callable_name and f"_{task_id}" != python_callable_name: 96 | self.add_message("match-callable-taskid", node=node) 97 | 98 | @utils.check_messages("mixed-dependency-directions") 99 | def visit_binop(self, node): 100 | """Check for mixed dependency directions.""" 101 | 102 | def fetch_binops(node_): 103 | """ 104 | Method fetching binary operations (>> and/or <<). 105 | Resides in separate function for recursion. 106 | """ 107 | binops_found = set() 108 | if isinstance(node_.left, astroid.BinOp): 109 | binops_found.update(fetch_binops(node_.left)) 110 | if isinstance(node_.right, astroid.BinOp): 111 | binops_found.update(fetch_binops(node_.right)) 112 | if node_.op == ">>" or node_.op == "<<": 113 | binops_found.add(node_.op) 114 | 115 | return binops_found 116 | 117 | binops = fetch_binops(node) 118 | if ">>" in binops and "<<" in binops: 119 | self.add_message("mixed-dependency-directions", node=node) 120 | -------------------------------------------------------------------------------- /tests/pylint_airflow/checkers/test_operator.py: -------------------------------------------------------------------------------- 1 | """Tests for the Operator checker.""" 2 | 3 | import astroid 4 | import pytest 5 | from pylint.testutils import CheckerTestCase, Message 6 | 7 | import pylint_airflow 8 | 9 | 10 | class TestOperatorChecker(CheckerTestCase): 11 | """Tests for the Operator checker.""" 12 | 13 | CHECKER_CLASS = pylint_airflow.checkers.operator.OperatorChecker 14 | 15 | def test_different_operator_varname_taskid(self): 16 | """task_id and operator instance name should match, but differ so should add message.""" 17 | testcase = """ 18 | from airflow.operators.dummy_operator import DummyOperator 19 | mytask = DummyOperator(task_id="foo") #@ 20 | """ 21 | expected_message = "different-operator-varname-taskid" 22 | 23 | assign_node = astroid.extract_node(testcase) 24 | with self.assertAddsMessages(Message(msg_id=expected_message, node=assign_node)): 25 | self.checker.visit_assign(assign_node) 26 | 27 | def test_different_operator_varname_taskid_baseoperator(self): 28 | """ 29 | task_id and operator instance name should match, but differ, so should add message, also 30 | when using BaseOperator. 31 | """ 32 | testcase = """ 33 | from airflow.models import BaseOperator 34 | mytask = BaseOperator(task_id="foo") #@ 35 | """ 36 | expected_message = "different-operator-varname-taskid" 37 | 38 | assign_node = astroid.extract_node(testcase) 39 | with self.assertAddsMessages(Message(msg_id=expected_message, node=assign_node)): 40 | self.checker.visit_assign(assign_node) 41 | 42 | def test_different_operator_varname_taskid_valid(self): 43 | """task_id and operator instance name are identical so no message should be added.""" 44 | testcase = """ 45 | from airflow.operators.dummy_operator import DummyOperator 46 | mytask = DummyOperator(task_id="mytask") #@ 47 | """ 48 | 49 | assign_node = astroid.extract_node(testcase) 50 | with self.assertNoMessages(): 51 | self.checker.visit_assign(assign_node) 52 | 53 | @pytest.mark.parametrize( 54 | "imports,operator_def", 55 | [ 56 | ( 57 | "from airflow.operators.python_operator import PythonOperator", 58 | 'mytask = PythonOperator(task_id="mytask", python_callable=foo) #@', 59 | ), 60 | ( 61 | "from airflow.operators import python_operator", 62 | 'mytask = python_operator.PythonOperator(task_id="mytask", python_callable=foo) #@', 63 | ), 64 | ( 65 | "import airflow.operators.python_operator", 66 | 'mytask = airflow.operators.python_operator.PythonOperator(task_id="mytask", python_callable=foo) #@', # pylint: disable=line-too-long 67 | ), 68 | ], 69 | ) 70 | def test_match_callable_taskid(self, imports, operator_def): 71 | """tests matching match_callable_taskid""" 72 | testcase = f"{imports}\ndef foo(): print('dosomething')\n{operator_def}" 73 | expected_message = "match-callable-taskid" 74 | 75 | assign_node = astroid.extract_node(testcase) 76 | with self.assertAddsMessages(Message(msg_id=expected_message, node=assign_node)): 77 | self.checker.visit_assign(assign_node) 78 | 79 | def test_not_match_callable_taskid(self): 80 | """python_callable function name matches _[task_id], expect no message.""" 81 | testcase = """ 82 | from airflow.operators.python_operator import PythonOperator 83 | 84 | def _mytask(): 85 | print("dosomething") 86 | 87 | mytask = PythonOperator(task_id="mytask", python_callable=_mytask) #@ 88 | """ 89 | 90 | assign_node = astroid.extract_node(testcase) 91 | with self.assertNoMessages(): 92 | self.checker.visit_assign(assign_node) 93 | 94 | @pytest.mark.parametrize( 95 | "dependencies,expect_msg", 96 | [ 97 | ("t1 >> t2", False), 98 | ("t1 >> t2 << t3", True), 99 | ("t1 >> t2 >> t3 >> t4 >> t5", False), 100 | ("t1 >> t2 << t3 >> t4 << t5", True), 101 | ("t1 >> [t2, t3]", False), 102 | ("[t1, t2] >> t3", False), 103 | ("[t1, t2] >> t3 << t4", True), 104 | ("t1 >> t2 << [t3, t4]", True), 105 | ("[t1, t2] >> t3 << [t4, t5]", True), 106 | ], 107 | ) 108 | def test_mixed_dependency_directions(self, dependencies, expect_msg): 109 | """ 110 | Test various ways (both directions & single task/lists) to set dependencies using bitshift 111 | operators. Should add message when mixing directions. 112 | """ 113 | testcase = f""" 114 | from airflow.operators.dummy_operator import DummyOperator 115 | t1 = DummyOperator(task_id="t1") 116 | t2 = DummyOperator(task_id="t2") 117 | t3 = DummyOperator(task_id="t3") 118 | t4 = DummyOperator(task_id="t4") 119 | t5 = DummyOperator(task_id="t5") 120 | {dependencies} #@ 121 | """ 122 | message = "mixed-dependency-directions" 123 | binop_node = astroid.extract_node(testcase) 124 | 125 | if expect_msg: 126 | with self.assertAddsMessages(Message(msg_id=message, node=binop_node)): 127 | self.checker.visit_binop(binop_node) 128 | else: 129 | with self.assertNoMessages(): 130 | self.checker.visit_binop(binop_node) 131 | -------------------------------------------------------------------------------- /src/pylint_airflow/checkers/dag.py: -------------------------------------------------------------------------------- 1 | """Checks on Airflow DAGs.""" 2 | 3 | from collections import defaultdict 4 | from typing import Tuple, Union 5 | 6 | import astroid 7 | from pylint import checkers 8 | from pylint import interfaces 9 | from pylint.checkers import utils 10 | from pylint.checkers.utils import safe_infer 11 | 12 | from pylint_airflow.__pkginfo__ import BASE_ID 13 | 14 | 15 | class DagChecker(checkers.BaseChecker): 16 | """Checks conditions in the context of (a) complete DAG(s).""" 17 | 18 | __implements__ = interfaces.IAstroidChecker 19 | 20 | msgs = { 21 | f"W{BASE_ID}00": ( 22 | "Don't place BaseHook calls at the top level of DAG script", 23 | "basehook-top-level", 24 | "Airflow executes DAG scripts periodically and anything at the top level " 25 | "of a script is executed. Therefore, move BaseHook calls into " 26 | "functions/hooks/operators.", 27 | ), 28 | f"E{BASE_ID}00": ( 29 | "DAG name %s already used", 30 | "duplicate-dag-name", 31 | "DAG name should be unique.", 32 | ), 33 | f"E{BASE_ID}01": ( 34 | "Task name {} already used", 35 | "duplicate-task-name", 36 | "Task name within a DAG should be unique.", 37 | ), 38 | f"E{BASE_ID}02": ( 39 | "Task dependency {}->{} already set", 40 | "duplicate-dependency", 41 | "Task dependencies can be defined only once.", 42 | ), 43 | f"E{BASE_ID}03": ( 44 | "DAG {} contains cycles", 45 | "dag-with-cycles", 46 | "A DAG is acyclic and cannot contain cycles.", 47 | ), 48 | f"E{BASE_ID}04": ( 49 | "Task {} is not bound to any DAG instance", 50 | "task-no-dag", 51 | "A task must know a DAG instance to run.", 52 | ), 53 | f"C{BASE_ID}06": ( 54 | "For consistency match the DAG filename with the dag_id", 55 | "match-dagid-filename", 56 | "For consistency match the DAG filename with the dag_id.", 57 | ), 58 | } 59 | 60 | @utils.check_messages("duplicate-dag-name", "match-dagid-filename") 61 | def visit_module(self, node: astroid.Module): 62 | """Checks in the context of (a) complete DAG(s).""" 63 | dagids_nodes = defaultdict(list) 64 | assigns = node.nodes_of_class(astroid.Assign) 65 | withs = node.nodes_of_class(astroid.With) 66 | 67 | def _find_dag( 68 | call_node: astroid.Call, func: Union[astroid.Name, astroid.Attribute] 69 | ) -> Tuple[Union[str, None], Union[astroid.Assign, astroid.Call, None]]: 70 | """ 71 | Find DAG in a call_node. 72 | :param call_node: 73 | :param func: 74 | :return: (dag_id, node) 75 | :rtype: Tuple 76 | """ 77 | if (hasattr(func, "name") and func.name == "DAG") or ( 78 | hasattr(func, "attrname") and func.attrname == "DAG" 79 | ): 80 | function_node = safe_infer(func) 81 | if function_node.is_subtype_of("airflow.models.DAG") or function_node.is_subtype_of( 82 | "airflow.models.dag.DAG" 83 | ): 84 | # Check for "dag_id" as keyword arg 85 | if call_node.keywords is not None: 86 | for keyword in call_node.keywords: 87 | # Only constants supported 88 | if keyword.arg == "dag_id" and isinstance(keyword.value, astroid.Const): 89 | return str(keyword.value.value), call_node 90 | 91 | if call_node.args: 92 | if not hasattr(call_node.args[0], "value"): 93 | # TODO Support dynamic dag_id. If dag_id is set from variable, it has no value attr. # pylint: disable=line-too-long 94 | return None, None 95 | return call_node.args[0].value, call_node 96 | 97 | return None, None 98 | 99 | # Find DAGs in assignments 100 | for assign in assigns: 101 | if isinstance(assign.value, astroid.Call): 102 | func = assign.value.func 103 | dagid, dagnode = _find_dag(assign.value, func) 104 | if dagid and dagnode: # Checks if there are no Nones 105 | dagids_nodes[dagid].append(dagnode) 106 | 107 | # Find DAGs in context managers 108 | for with_ in withs: 109 | for with_item in with_.items: 110 | call_node = with_item[0] 111 | if isinstance(call_node, astroid.Call): 112 | func = call_node.func 113 | dagid, dagnode = _find_dag(call_node, func) 114 | if dagid and dagnode: # Checks if there are no Nones 115 | dagids_nodes[dagid].append(dagnode) 116 | 117 | # Check if single DAG and if equals filename 118 | # Unit test nodes have file "" 119 | if len(dagids_nodes) == 1 and node.file != "": 120 | dagid, _ = list(dagids_nodes.items())[0] 121 | expected_filename = f"{dagid}.py" 122 | current_filename = node.file.split("/")[-1] 123 | if expected_filename != current_filename: 124 | self.add_message("match-dagid-filename", node=node) 125 | 126 | duplicate_dagids = [ 127 | (dagid, nodes) 128 | for dagid, nodes in dagids_nodes.items() 129 | if len(nodes) >= 2 and dagid is not None 130 | ] 131 | for (dagid, assign_nodes) in duplicate_dagids: 132 | self.add_message("duplicate-dag-name", node=assign_nodes[-1], args=dagid) 133 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ############## 2 | Pylint-Airflow 3 | ############## 4 | 5 | .. image:: https://img.shields.io/badge/code%20style-black-000000.svg 6 | :alt: Code style: Black 7 | :target: https://github.com/ambv/black 8 | 9 | .. image:: https://img.shields.io/badge/License-MIT-blue.svg 10 | :alt: License: MIT 11 | :target: https://github.com/BasPH/airflow-examples/blob/master/LICENSE 12 | 13 | .. image:: https://img.shields.io/circleci/project/github/BasPH/pylint-airflow/master.svg 14 | :target: https://circleci.com/gh/BasPH/workflows/pylint-airflow/tree/master 15 | 16 | .. image:: images/pylint-airflow.png 17 | :align: right 18 | 19 | Pylint plugin for static code analysis on Airflow code. 20 | 21 | ***** 22 | Usage 23 | ***** 24 | 25 | Installation: 26 | 27 | .. code-block:: bash 28 | 29 | pip install pylint-airflow 30 | 31 | Usage: 32 | 33 | .. code-block:: bash 34 | 35 | pylint --load-plugins=pylint_airflow [your_file] 36 | 37 | This plugin runs on Python 3.6 and higher. 38 | 39 | *********** 40 | Error codes 41 | *********** 42 | 43 | The Pylint-Airflow codes follow the structure ``{I,C,R,W,E,F}83{0-9}{0-9}``, where: 44 | 45 | - The characters show: 46 | 47 | - ``I`` = Info 48 | - ``C`` = Convention 49 | - ``R`` = Refactor 50 | - ``W`` = Warning 51 | - ``E`` = Error 52 | - ``F`` = Fatal 53 | 54 | - ``83`` is the base id (see all here https://github.com/PyCQA/pylint/blob/master/pylint/checkers/__init__.py) 55 | - ``{0-9}{0-9}`` is any number 00-99 56 | 57 | The current codes are: 58 | 59 | +-------+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ 60 | | Code | Symbol | Description | 61 | +=======+===================================+=================================================================================================================================================================+ 62 | | C8300 | different-operator-varname-taskid | For consistency assign the same variable name and task_id to operators. | 63 | +-------+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ 64 | | C8301 | match-callable-taskid | For consistency name the callable function '_[task_id]', e.g. PythonOperator(task_id='mytask', python_callable=_mytask). | 65 | +-------+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ 66 | | C8302 | mixed-dependency-directions | For consistency don't mix directions in a single statement, instead split over multiple statements. | 67 | +-------+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ 68 | | C8303 | task-no-dependencies | Sometimes a task without any dependency is desired, however often it is the result of a forgotten dependency. | 69 | +-------+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ 70 | | C8304 | task-context-argname | Indicate you expect Airflow task context variables in the \*\*kwargs argument by renaming to \*\*context. | 71 | +-------+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ 72 | | C8305 | task-context-separate-arg | To avoid unpacking kwargs from the Airflow task context in a function, you can set the needed variables as arguments in the function. | 73 | +-------+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ 74 | | C8306 | match-dagid-filename | For consistency match the DAG filename with the dag_id. | 75 | +-------+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ 76 | | R8300 | unused-xcom | Return values from a python_callable function or execute() method are automatically pushed as XCom. | 77 | +-------+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ 78 | | W8300 | basehook-top-level | Airflow executes DAG scripts periodically and anything at the top level of a script is executed. Therefore, move BaseHook calls into functions/hooks/operators. | 79 | +-------+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ 80 | | E8300 | duplicate-dag-name | DAG name should be unique. | 81 | +-------+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ 82 | | E8301 | duplicate-task-name | Task name within a DAG should be unique. | 83 | +-------+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ 84 | | E8302 | duplicate-dependency | Task dependencies can be defined only once. | 85 | +-------+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ 86 | | E8303 | dag-with-cycles | A DAG is acyclic and cannot contain cycles. | 87 | +-------+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ 88 | | E8304 | task-no-dag | A task must know a DAG instance to run. | 89 | +-------+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ 90 | 91 | ************* 92 | Documentation 93 | ************* 94 | 95 | Documentation is available on `Read the Docs `_. 96 | 97 | ************ 98 | Contributing 99 | ************ 100 | 101 | Suggestions for more checks are always welcome, please create an issue on GitHub. Read `CONTRIBUTING.rst `_ for more details. 102 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # A comma-separated list of package or module names from where C extensions may 4 | # be loaded. Extensions are loading into the active Python interpreter and may 5 | # run arbitrary code. 6 | extension-pkg-whitelist= 7 | 8 | # Add files or directories to the blacklist. They should be base names, not 9 | # paths. 10 | ignore=CVS 11 | 12 | # Add files or directories matching the regex patterns to the blacklist. The 13 | # regex matches against base names, not paths. 14 | ignore-patterns= 15 | 16 | # Python code to execute, usually for sys.path manipulation such as 17 | # pygtk.require(). 18 | #init-hook= 19 | 20 | # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the 21 | # number of processors available to use. 22 | jobs=1 23 | 24 | # Control the amount of potential inferred values when inferring a single 25 | # object. This can help the performance when dealing with large functions or 26 | # complex, nested conditions. 27 | limit-inference-results=100 28 | 29 | # List of plugins (as comma separated values of python modules names) to load, 30 | # usually to register additional checkers. 31 | load-plugins= 32 | 33 | # Pickle collected data for later comparisons. 34 | persistent=yes 35 | 36 | # Specify a configuration file. 37 | #rcfile= 38 | 39 | # When enabled, pylint would attempt to guess common misconfiguration and emit 40 | # user-friendly hints instead of false-positive error messages. 41 | suggestion-mode=yes 42 | 43 | # Allow loading of arbitrary C extensions. Extensions are imported into the 44 | # active Python interpreter and may run arbitrary code. 45 | unsafe-load-any-extension=no 46 | 47 | 48 | [MESSAGES CONTROL] 49 | 50 | # Only show warnings with the listed confidence levels. Leave empty to show 51 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. 52 | confidence= 53 | 54 | # Disable the message, report, category or checker with the given id(s). You 55 | # can either give multiple identifiers separated by comma (,) or put this 56 | # option multiple times (only on the command line, not in the configuration 57 | # file where it should appear only once). You can also use "--disable=all" to 58 | # disable everything first and then reenable specific checks. For example, if 59 | # you want to run only the similarities checker, you can use "--disable=all 60 | # --enable=similarities". If you want to run only the classes checker, but have 61 | # no Warning level messages displayed, use "--disable=all --enable=classes 62 | # --disable=W". 63 | disable=print-statement, 64 | parameter-unpacking, 65 | unpacking-in-except, 66 | old-raise-syntax, 67 | backtick, 68 | long-suffix, 69 | old-ne-operator, 70 | old-octal-literal, 71 | import-star-module-level, 72 | non-ascii-bytes-literal, 73 | raw-checker-failed, 74 | bad-inline-option, 75 | locally-disabled, 76 | file-ignored, 77 | suppressed-message, 78 | useless-suppression, 79 | deprecated-pragma, 80 | use-symbolic-message-instead, 81 | apply-builtin, 82 | basestring-builtin, 83 | buffer-builtin, 84 | cmp-builtin, 85 | coerce-builtin, 86 | execfile-builtin, 87 | file-builtin, 88 | long-builtin, 89 | raw_input-builtin, 90 | reduce-builtin, 91 | standarderror-builtin, 92 | unicode-builtin, 93 | xrange-builtin, 94 | coerce-method, 95 | delslice-method, 96 | getslice-method, 97 | setslice-method, 98 | no-absolute-import, 99 | old-division, 100 | dict-iter-method, 101 | dict-view-method, 102 | next-method-called, 103 | metaclass-assignment, 104 | indexing-exception, 105 | raising-string, 106 | reload-builtin, 107 | oct-method, 108 | hex-method, 109 | nonzero-method, 110 | cmp-method, 111 | input-builtin, 112 | round-builtin, 113 | intern-builtin, 114 | unichr-builtin, 115 | map-builtin-not-iterating, 116 | zip-builtin-not-iterating, 117 | range-builtin-not-iterating, 118 | filter-builtin-not-iterating, 119 | using-cmp-argument, 120 | eq-without-hash, 121 | div-method, 122 | idiv-method, 123 | rdiv-method, 124 | exception-message-attribute, 125 | invalid-str-codec, 126 | sys-max-int, 127 | bad-python3-import, 128 | deprecated-string-function, 129 | deprecated-str-translate-call, 130 | deprecated-itertools-function, 131 | deprecated-types-field, 132 | next-method-defined, 133 | dict-items-not-iterating, 134 | dict-keys-not-iterating, 135 | dict-values-not-iterating, 136 | deprecated-operator-function, 137 | deprecated-urllib-function, 138 | xreadlines-attribute, 139 | deprecated-sys-function, 140 | exception-escape, 141 | comprehension-escape, 142 | bad-continuation, # https://github.com/PyCQA/pylint/issues/289 143 | duplicate-code, 144 | fixme 145 | 146 | # Enable the message, report, category or checker with the given id(s). You can 147 | # either give multiple identifier separated by comma (,) or put this option 148 | # multiple time (only on the command line, not in the configuration file where 149 | # it should appear only once). See also the "--disable" option for examples. 150 | enable=c-extension-no-member 151 | 152 | 153 | [REPORTS] 154 | 155 | # Python expression which should return a note less than 10 (10 is the highest 156 | # note). You have access to the variables errors warning, statement which 157 | # respectively contain the number of errors / warnings messages and the total 158 | # number of statements analyzed. This is used by the global evaluation report 159 | # (RP0004). 160 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 161 | 162 | # Template used to display messages. This is a python new-style format string 163 | # used to format the message information. See doc for all details. 164 | #msg-template= 165 | 166 | # Set the output format. Available formats are text, parseable, colorized, json 167 | # and msvs (visual studio). You can also give a reporter class, e.g. 168 | # mypackage.mymodule.MyReporterClass. 169 | output-format=text 170 | 171 | # Tells whether to display a full report or only the messages. 172 | reports=no 173 | 174 | # Activate the evaluation score. 175 | score=yes 176 | 177 | 178 | [REFACTORING] 179 | 180 | # Maximum number of nested blocks for function / method body 181 | max-nested-blocks=5 182 | 183 | # Complete name of functions that never returns. When checking for 184 | # inconsistent-return-statements if a never returning function is called then 185 | # it will be considered as an explicit return statement and no message will be 186 | # printed. 187 | never-returning-functions=sys.exit 188 | 189 | 190 | [LOGGING] 191 | 192 | # Format style used to check logging format string. `old` means using % 193 | # formatting, while `new` is for `{}` formatting. 194 | logging-format-style=old 195 | 196 | # Logging modules to check that the string format arguments are in logging 197 | # function parameter format. 198 | logging-modules=logging 199 | 200 | 201 | [SPELLING] 202 | 203 | # Limits count of emitted suggestions for spelling mistakes. 204 | max-spelling-suggestions=4 205 | 206 | # Spelling dictionary name. Available dictionaries: none. To make it working 207 | # install python-enchant package.. 208 | spelling-dict= 209 | 210 | # List of comma separated words that should not be checked. 211 | spelling-ignore-words= 212 | 213 | # A path to a file that contains private dictionary; one word per line. 214 | spelling-private-dict-file= 215 | 216 | # Tells whether to store unknown words to indicated private dictionary in 217 | # --spelling-private-dict-file option instead of raising a message. 218 | spelling-store-unknown-words=no 219 | 220 | 221 | [MISCELLANEOUS] 222 | 223 | # List of note tags to take in consideration, separated by a comma. 224 | notes=FIXME, 225 | XXX, 226 | TODO 227 | 228 | 229 | [TYPECHECK] 230 | 231 | # List of decorators that produce context managers, such as 232 | # contextlib.contextmanager. Add to this list to register other decorators that 233 | # produce valid context managers. 234 | contextmanager-decorators=contextlib.contextmanager 235 | 236 | # List of members which are set dynamically and missed by pylint inference 237 | # system, and so shouldn't trigger E1101 when accessed. Python regular 238 | # expressions are accepted. 239 | generated-members=helpers 240 | 241 | # Tells whether missing members accessed in mixin class should be ignored. A 242 | # mixin class is detected if its name ends with "mixin" (case insensitive). 243 | ignore-mixin-members=yes 244 | 245 | # Tells whether to warn about missing members when the owner of the attribute 246 | # is inferred to be None. 247 | ignore-none=yes 248 | 249 | # This flag controls whether pylint should warn about no-member and similar 250 | # checks whenever an opaque object is returned when inferring. The inference 251 | # can return multiple potential results while evaluating a Python object, but 252 | # some branches might not be evaluated, which results in partial inference. In 253 | # that case, it might be useful to still emit no-member and other checks for 254 | # the rest of the inferred objects. 255 | ignore-on-opaque-inference=yes 256 | 257 | # List of class names for which member attributes should not be checked (useful 258 | # for classes with dynamically set attributes). This supports the use of 259 | # qualified names. 260 | ignored-classes=optparse.Values,thread._local,_thread._local 261 | 262 | # List of module names for which member attributes should not be checked 263 | # (useful for modules/projects where namespaces are manipulated during runtime 264 | # and thus existing member attributes cannot be deduced by static analysis. It 265 | # supports qualified module names, as well as Unix pattern matching. 266 | ignored-modules= 267 | 268 | # Show a hint with possible names when a member name was not found. The aspect 269 | # of finding the hint is based on edit distance. 270 | missing-member-hint=yes 271 | 272 | # The minimum edit distance a name should have in order to be considered a 273 | # similar match for a missing member name. 274 | missing-member-hint-distance=1 275 | 276 | # The total number of similar names that should be taken in consideration when 277 | # showing a hint for a missing member. 278 | missing-member-max-choices=1 279 | 280 | 281 | [VARIABLES] 282 | 283 | # List of additional names supposed to be defined in builtins. Remember that 284 | # you should avoid defining new builtins when possible. 285 | additional-builtins= 286 | 287 | # Tells whether unused global variables should be treated as a violation. 288 | allow-global-unused-variables=yes 289 | 290 | # List of strings which can identify a callback function by name. A callback 291 | # name must start or end with one of those strings. 292 | callbacks=cb_, 293 | _cb 294 | 295 | # A regular expression matching the name of dummy variables (i.e. expected to 296 | # not be used). 297 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 298 | 299 | # Argument names that match this expression will be ignored. Default to name 300 | # with leading underscore. 301 | ignored-argument-names=_.*|^ignored_|^unused_ 302 | 303 | # Tells whether we should check for unused import in __init__ files. 304 | init-import=no 305 | 306 | # List of qualified module names which can have objects that can redefine 307 | # builtins. 308 | redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io 309 | 310 | 311 | [FORMAT] 312 | 313 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 314 | expected-line-ending-format= 315 | 316 | # Regexp for a line that is allowed to be longer than the limit. 317 | ignore-long-lines=^\s*(# )??$ 318 | 319 | # Number of spaces of indent required inside a hanging or continued line. 320 | indent-after-paren=4 321 | 322 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 323 | # tab). 324 | indent-string=' ' 325 | 326 | # Maximum number of characters on a single line. 327 | max-line-length=100 328 | 329 | # Maximum number of lines in a module. 330 | max-module-lines=1000 331 | 332 | # List of optional constructs for which whitespace checking is disabled. `dict- 333 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 334 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 335 | # `empty-line` allows space-only lines. 336 | no-space-check=trailing-comma, 337 | dict-separator 338 | 339 | # Allow the body of a class to be on the same line as the declaration if body 340 | # contains single statement. 341 | single-line-class-stmt=no 342 | 343 | # Allow the body of an if to be on the same line as the test if there is no 344 | # else. 345 | single-line-if-stmt=no 346 | 347 | 348 | [SIMILARITIES] 349 | 350 | # Ignore comments when computing similarities. 351 | ignore-comments=yes 352 | 353 | # Ignore docstrings when computing similarities. 354 | ignore-docstrings=yes 355 | 356 | # Ignore imports when computing similarities. 357 | ignore-imports=no 358 | 359 | # Minimum lines number of a similarity. 360 | min-similarity-lines=4 361 | 362 | 363 | [BASIC] 364 | 365 | # Naming style matching correct argument names. 366 | argument-naming-style=snake_case 367 | 368 | # Regular expression matching correct argument names. Overrides argument- 369 | # naming-style. 370 | #argument-rgx= 371 | 372 | # Naming style matching correct attribute names. 373 | attr-naming-style=snake_case 374 | 375 | # Regular expression matching correct attribute names. Overrides attr-naming- 376 | # style. 377 | #attr-rgx= 378 | 379 | # Bad variable names which should always be refused, separated by a comma. 380 | bad-names=foo, 381 | bar, 382 | baz, 383 | toto, 384 | tutu, 385 | tata 386 | 387 | # Naming style matching correct class attribute names. 388 | class-attribute-naming-style=any 389 | 390 | # Regular expression matching correct class attribute names. Overrides class- 391 | # attribute-naming-style. 392 | #class-attribute-rgx= 393 | 394 | # Naming style matching correct class names. 395 | class-naming-style=PascalCase 396 | 397 | # Regular expression matching correct class names. Overrides class-naming- 398 | # style. 399 | #class-rgx= 400 | 401 | # Naming style matching correct constant names. 402 | const-naming-style=any 403 | 404 | # Regular expression matching correct constant names. Overrides const-naming- 405 | # style. 406 | #const-rgx= 407 | 408 | # Minimum line length for functions/classes that require docstrings, shorter 409 | # ones are exempt. 410 | docstring-min-length=-1 411 | 412 | # Naming style matching correct function names. 413 | function-naming-style=snake_case 414 | 415 | # Regular expression matching correct function names. Overrides function- 416 | # naming-style. 417 | #function-rgx= 418 | 419 | # Good variable names which should always be accepted, separated by a comma. 420 | good-names=i, 421 | j, 422 | k, 423 | ex, 424 | Run, 425 | _ 426 | 427 | # Include a hint for the correct naming format with invalid-name. 428 | include-naming-hint=no 429 | 430 | # Naming style matching correct inline iteration names. 431 | inlinevar-naming-style=any 432 | 433 | # Regular expression matching correct inline iteration names. Overrides 434 | # inlinevar-naming-style. 435 | #inlinevar-rgx= 436 | 437 | # Naming style matching correct method names. 438 | method-naming-style=snake_case 439 | 440 | # Regular expression matching correct method names. Overrides method-naming- 441 | # style. 442 | #method-rgx= 443 | 444 | # Naming style matching correct module names. 445 | module-naming-style=snake_case 446 | 447 | # Regular expression matching correct module names. Overrides module-naming- 448 | # style. 449 | #module-rgx= 450 | 451 | # Colon-delimited sets of names that determine each other's naming style when 452 | # the name regexes allow several styles. 453 | name-group= 454 | 455 | # Regular expression which should only match function or class names that do 456 | # not require a docstring. 457 | no-docstring-rgx=^_ 458 | 459 | # List of decorators that produce properties, such as abc.abstractproperty. Add 460 | # to this list to register other decorators that produce valid properties. 461 | # These decorators are taken in consideration only for invalid-name. 462 | property-classes=abc.abstractproperty 463 | 464 | # Naming style matching correct variable names. 465 | variable-naming-style=snake_case 466 | 467 | # Regular expression matching correct variable names. Overrides variable- 468 | # naming-style. 469 | #variable-rgx= 470 | 471 | 472 | [STRING] 473 | 474 | # This flag controls whether the implicit-str-concat-in-sequence should 475 | # generate a warning on implicit string concatenation in sequences defined over 476 | # several lines. 477 | check-str-concat-over-line-jumps=no 478 | 479 | 480 | [IMPORTS] 481 | 482 | # Allow wildcard imports from modules that define __all__. 483 | allow-wildcard-with-all=no 484 | 485 | # Analyse import fallback blocks. This can be used to support both Python 2 and 486 | # 3 compatible code, which means that the block might have code that exists 487 | # only in one or another interpreter, leading to false positives when analysed. 488 | analyse-fallback-blocks=no 489 | 490 | # Deprecated modules which should not be used, separated by a comma. 491 | deprecated-modules=optparse,tkinter.tix 492 | 493 | # Create a graph of external dependencies in the given file (report RP0402 must 494 | # not be disabled). 495 | ext-import-graph= 496 | 497 | # Create a graph of every (i.e. internal and external) dependencies in the 498 | # given file (report RP0402 must not be disabled). 499 | import-graph= 500 | 501 | # Create a graph of internal dependencies in the given file (report RP0402 must 502 | # not be disabled). 503 | int-import-graph= 504 | 505 | # Force import order to recognize a module as part of the standard 506 | # compatibility libraries. 507 | known-standard-library= 508 | 509 | # Force import order to recognize a module as part of a third party library. 510 | known-third-party=enchant 511 | 512 | 513 | [CLASSES] 514 | 515 | # List of method names used to declare (i.e. assign) instance attributes. 516 | defining-attr-methods=__init__, 517 | __new__, 518 | setUp 519 | 520 | # List of member names, which should be excluded from the protected access 521 | # warning. 522 | exclude-protected=_asdict, 523 | _fields, 524 | _replace, 525 | _source, 526 | _make 527 | 528 | # List of valid names for the first argument in a class method. 529 | valid-classmethod-first-arg=cls 530 | 531 | # List of valid names for the first argument in a metaclass class method. 532 | valid-metaclass-classmethod-first-arg=cls 533 | 534 | 535 | [DESIGN] 536 | 537 | # Maximum number of arguments for function / method. 538 | max-args=5 539 | 540 | # Maximum number of attributes for a class (see R0902). 541 | max-attributes=7 542 | 543 | # Maximum number of boolean expressions in an if statement. 544 | max-bool-expr=5 545 | 546 | # Maximum number of branch for function / method body. 547 | max-branches=12 548 | 549 | # Maximum number of locals for function / method body. 550 | max-locals=25 551 | 552 | # Maximum number of parents for a class (see R0901). 553 | max-parents=7 554 | 555 | # Maximum number of public methods for a class (see R0904). 556 | max-public-methods=20 557 | 558 | # Maximum number of return / yield for function / method body. 559 | max-returns=6 560 | 561 | # Maximum number of statements in function / method body. 562 | max-statements=50 563 | 564 | # Minimum number of public methods for a class (see R0903). 565 | min-public-methods=2 566 | 567 | 568 | [EXCEPTIONS] 569 | 570 | # Exceptions that will emit a warning when being caught. Defaults to 571 | # "BaseException, Exception". 572 | overgeneral-exceptions=BaseException, 573 | Exception 574 | --------------------------------------------------------------------------------