├── .cookiecutter.json ├── .dockerignore ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md ├── pull_request_template.md └── workflows │ ├── ci.yml │ ├── coverage.yml │ ├── rebake.yml │ └── upstream_testing.yml ├── .gitignore ├── .readthedocs.yaml ├── .yamllint.yml ├── LICENSE ├── README.md ├── changes ├── +nautobot-app-v2-4-1.housekeeping ├── +nautobot-app-v2-4-2.housekeeping ├── +nautobot-app-v2.5.0.housekeeping ├── .gitignore └── 191.fixed ├── development ├── Dockerfile ├── app_config_schema.py ├── creds.example.env ├── development.env ├── development_mysql.env ├── docker-compose.base.yml ├── docker-compose.dev.yml ├── docker-compose.mysql.yml ├── docker-compose.postgres.yml ├── docker-compose.redis.yml ├── nautobot_config.py └── towncrier_template.j2 ├── docs ├── admin │ ├── compatibility_matrix.md │ ├── install.md │ ├── release_notes │ │ ├── index.md │ │ ├── version_1.0.md │ │ ├── version_2.0.md │ │ ├── version_2.1.md │ │ ├── version_2.2.md │ │ ├── version_3.0.md │ │ ├── version_3.1.md │ │ └── version_3.2.md │ ├── uninstall.md │ └── upgrade.md ├── assets │ ├── extra.css │ ├── favicon.ico │ ├── nautobot_logo.png │ ├── nautobot_logo.svg │ ├── networktocode_bw.png │ └── overrides │ │ └── partials │ │ └── copyright.html ├── dev │ ├── code_reference │ │ ├── api.md │ │ ├── index.md │ │ └── package.md │ ├── contributing.md │ ├── dev_environment.md │ ├── extending.md │ └── release_checklist.md ├── images │ ├── data-compliance-filtered-results-list.png │ ├── data-compliance-object-tab-invalid.png │ ├── data-compliance-object-tab-valid.png │ ├── data-compliance-results-list.png │ ├── data-compliance-run-registered-data-compliance-rules-job.png │ ├── dropdown.png │ ├── icon-DataValidationEngine.png │ ├── min-max-rules-edit.png │ ├── min-max-rules-enforcement.png │ ├── min-max-rules-list.png │ ├── regex-rules-edit.png │ ├── regex-rules-enforcement.png │ ├── regex-rules-jinja2-context-processing.png │ ├── regex-rules-list.png │ ├── required-rules-edit.png │ ├── required-rules-enforcement.png │ ├── required-rules-list.png │ ├── unique-rules-edit.png │ ├── unique-rules-enforcement.png │ └── unique-rules-list.png ├── index.md ├── requirements.txt └── user │ ├── app_data_compliance.md │ ├── app_getting_started.md │ ├── app_overview.md │ ├── app_use_cases.md │ └── faq.md ├── invoke.example.yml ├── invoke.mysql.yml ├── mkdocs.yml ├── nautobot_data_validation_engine ├── __init__.py ├── api │ ├── __init__.py │ ├── serializers.py │ ├── urls.py │ └── views.py ├── app-config-schema.json ├── custom_validators.py ├── datasources.py ├── filters.py ├── forms.py ├── jobs.py ├── migrations │ ├── 0001_initial.py │ ├── 0002_required_unique_types_regex_context.py │ ├── 0003_datacompliance.py │ ├── 0004_created_datetime.py │ ├── 0005_remove_slugs_alter_tags.py │ ├── 0006_add_field_defaults.py │ ├── 0007_alter_datacompliance_compliance_class_name_and_more.py │ └── __init__.py ├── models.py ├── navigation.py ├── tables.py ├── template_content.py ├── templates │ └── nautobot_data_validation_engine │ │ ├── datacompliance_retrieve.html │ │ ├── datacompliance_tab.html │ │ ├── minmaxvalidationrule_retrieve.html │ │ ├── regularexpressionvalidationrule_retrieve.html │ │ ├── requiredvalidationrule_retrieve.html │ │ └── uniquevalidationrule_retrieve.html ├── tests │ ├── __init__.py │ ├── test_api.py │ ├── test_basic.py │ ├── test_custom_validators.py │ ├── test_data_compliance_rules.py │ ├── test_filters.py │ ├── test_models.py │ └── test_views.py ├── urls.py └── views.py ├── poetry.lock ├── pyproject.toml └── tasks.py /.cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "cookiecutter": { 3 | "codeowner_github_usernames": "@nautobot/plugin-data-validation", 4 | "full_name": "Network to Code, LLC", 5 | "email": "info@networktocode.com", 6 | "github_org": "nautobot", 7 | "app_name": "nautobot_data_validation_engine", 8 | "verbose_name": "Data Validation Engine", 9 | "app_slug": "nautobot-data-validation-engine", 10 | "project_slug": "nautobot-app-data-validation-engine", 11 | "repo_url": "https://github.com/nautobot/nautobot-app-data-validation-engine", 12 | "base_url": "nautobot-data-validation-engine", 13 | "min_nautobot_version": "2.1.9", 14 | "max_nautobot_version": "2.9999", 15 | "camel_name": "NautobotDataValidationEngine", 16 | "project_short_description": "Provides UI to build custom data validation rules for data in Nautobot", 17 | "model_class_name": "ValidationRule", 18 | "open_source_license": "Apache-2.0", 19 | "docs_base_url": "https://docs.nautobot.com", 20 | "docs_app_url": "https://docs.nautobot.com/projects/data-validation/en/latest", 21 | "_drift_manager": { 22 | "template": "https://github.com/nautobot/cookiecutter-nautobot-app.git", 23 | "template_dir": "nautobot-app", 24 | "template_ref": "refs/tags/nautobot-app-v2.5.0", 25 | "cookie_dir": "", 26 | "branch_prefix": "drift-manager", 27 | "pull_request_strategy": "create", 28 | "post_actions": [ 29 | "ruff", 30 | "poetry" 31 | ], 32 | "draft": false, 33 | "baked_commit_ref": "ee3f9105f680de8ced0851bd5625b5cffa69df2c" 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # Docker related 2 | development/Dockerfile 3 | development/docker-compose*.yml 4 | development/*.env 5 | *.env 6 | environments/ 7 | 8 | # Python 9 | **/*.pyc 10 | **/*.pyo 11 | **/__pycache__/ 12 | **/.pytest_cache/ 13 | **/.venv/ 14 | 15 | 16 | # Other 17 | docs/_build 18 | FAQ.md 19 | .git/ 20 | .gitignore 21 | .github 22 | LICENSE 23 | **/*.log 24 | **/.vscode/ 25 | invoke*.yml 26 | tasks.py 27 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Default owner(s) of all files in this repository 2 | * @nautobot/plugin-data-validation 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🐛 Bug Report 3 | about: Report a reproducible bug in the current release of nautobot-data-validation-engine 4 | --- 5 | 6 | ### Environment 7 | * Python version: 8 | * Nautobot version: 9 | * nautobot-data-validation-engine version: 10 | 11 | 12 | ### Expected Behavior 13 | 14 | 15 | 16 | ### Observed Behavior 17 | 18 | 22 | ### Steps to Reproduce 23 | 1. 24 | 2. 25 | 3. 26 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: ✨ Feature Request 3 | about: Propose a new feature or enhancement 4 | 5 | --- 6 | 7 | ### Environment 8 | * Nautobot version: 9 | * nautobot-data-validation-engine version: 10 | 11 | 14 | ### Proposed Functionality 15 | 16 | 21 | ### Use Case 22 | 23 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 10 | 11 | # Closes: # 12 | 13 | ## What's Changed 14 | 15 | 23 | 24 | ## To Do 25 | 26 | 29 | - [ ] Explanation of Change(s) 30 | - [ ] Added change log fragment(s) (for more information see [the documentation](https://docs.nautobot.com/projects/core/en/stable/development/core/#creating-changelog-fragments)) 31 | - [ ] Attached Screenshots, Payload Example 32 | - [ ] Unit, Integration Tests 33 | - [ ] Documentation Updates (when adding/changing features) 34 | - [ ] Outline Remaining Work, Constraints from Design 35 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 10 | 11 | # Closes: # 12 | 13 | ## What's Changed 14 | 15 | 23 | 24 | ## To Do 25 | 26 | 29 | - [ ] Explanation of Change(s) 30 | - [ ] Added change log fragment(s) (for more information see [the documentation](https://docs.nautobot.com/projects/core/en/stable/development/core/#creating-changelog-fragments)) 31 | - [ ] Attached Screenshots, Payload Example 32 | - [ ] Unit, Integration Tests 33 | - [ ] Documentation Updates (when adding/changing features) 34 | - [ ] Outline Remaining Work, Constraints from Design 35 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "Post coverage comment" 3 | 4 | on: # yamllint disable-line rule:truthy rule:comments 5 | workflow_run: 6 | workflows: ["CI"] 7 | types: 8 | - "completed" 9 | 10 | jobs: 11 | test: 12 | name: "Post coverage comment to PR" 13 | runs-on: "ubuntu-latest" 14 | if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' # yamllint disable-line rule:quoted-strings rule:comments 15 | permissions: 16 | # Gives the action the necessary permissions for publishing new 17 | # comments in pull requests. 18 | pull-requests: "write" 19 | # Gives the action the necessary permissions for editing existing 20 | # comments (to avoid publishing multiple comments in the same PR) 21 | contents: "write" # yamllint disable-line rule:indentation rule:comments 22 | # Gives the action the necessary permissions for looking up the 23 | # workflow that launched this workflow, and download the related 24 | # artifact that contains the comment to be published 25 | actions: "read" 26 | steps: 27 | # DO NOT run actions/checkout here, for security reasons 28 | # For details, refer to https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ 29 | - name: "Post comment" 30 | uses: "py-cov-action/python-coverage-comment-action@d1ff8fbb5ff80feedb3faa0f6d7b424f417ad0e13" # v3.30 31 | with: 32 | GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" 33 | GITHUB_PR_RUN_ID: "${{ github.event.workflow_run.id }}" 34 | # Update those if you changed the default values: 35 | # COMMENT_ARTIFACT_NAME: python-coverage-comment-action 36 | # COMMENT_FILENAME: python-coverage-comment-action.txt 37 | -------------------------------------------------------------------------------- /.github/workflows/rebake.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "Rebake Cookie" 3 | on: # yamllint disable-line rule:truthy 4 | workflow_call: 5 | inputs: 6 | cookie: 7 | description: "The cookie to rebake" 8 | type: "string" 9 | default: "" 10 | draft: 11 | description: "Whether to create the pull request as a draft" 12 | type: "string" 13 | default: "" 14 | pull-request: 15 | description: "The pull request strategy" 16 | type: "string" 17 | default: "" 18 | template: 19 | description: "The template repository URL" 20 | type: "string" 21 | default: "" 22 | template-dir: 23 | description: "The directory within the template repository to use as the template" 24 | type: "string" 25 | default: "" 26 | template-ref: 27 | description: "The branch or tag to use for the template" 28 | type: "string" 29 | default: "" 30 | drift-manager-tag: 31 | description: "The drift manager Docker image tag to use" 32 | type: "string" 33 | default: "latest" 34 | workflow_dispatch: 35 | inputs: 36 | cookie: 37 | description: "The cookie to rebake" 38 | type: "string" 39 | default: "" 40 | draft: 41 | description: "Whether to create the pull request as a draft" 42 | type: "string" 43 | default: "" 44 | pull-request: 45 | description: "The pull request strategy" 46 | type: "string" 47 | default: "" 48 | template: 49 | description: "The template repository URL" 50 | type: "string" 51 | default: "" 52 | template-dir: 53 | description: "The directory within the template repository to use as the template" 54 | type: "string" 55 | default: "" 56 | template-ref: 57 | description: "The branch or tag to use for the template" 58 | type: "string" 59 | default: "" 60 | drift-manager-tag: 61 | description: "The drift manager Docker image tag to use" 62 | type: "string" 63 | default: "latest" 64 | jobs: 65 | rebake: 66 | runs-on: "ubuntu-22.04" 67 | permissions: 68 | actions: "write" 69 | contents: "write" 70 | packages: "read" 71 | pull-requests: "write" 72 | container: "ghcr.io/nautobot/cookiecutter-nautobot-app-drift-manager/prod:${{ github.event.inputs.drift-manager-tag }}" 73 | env: 74 | GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" 75 | steps: 76 | - name: "Configure Rebake Arguments" 77 | id: "config" 78 | shell: "bash" 79 | run: | 80 | ARGS='--push' 81 | 82 | if [[ '${{ github.event.inputs.draft }}' == 'true' ]]; then 83 | ARGS="$ARGS --draft" 84 | elif [[ '${{ github.event.inputs.draft }}' == 'false' ]]; then 85 | ARGS="$ARGS --no-draft" 86 | elif [[ '${{ github.event.inputs.draft }}' == '' ]]; then 87 | echo "Using repo default value for --draft" 88 | else 89 | echo "ERROR: Invalid value for draft: '${{ github.event.inputs.draft }}'" 90 | exit 1 91 | fi 92 | 93 | if [[ '${{ github.event.inputs.pull-request }}' != '' ]]; then 94 | ARGS="$ARGS --pull-request='${{ github.event.inputs.pull-request }}'" 95 | fi 96 | 97 | if [[ '${{ github.event.inputs.template }}' != '' ]]; then 98 | ARGS="$ARGS --template='${{ github.event.inputs.template }}'" 99 | fi 100 | 101 | if [[ '${{ github.event.inputs.template-dir }}' != '' ]]; then 102 | ARGS="$ARGS --template-dir='${{ github.event.inputs.template-dir }}'" 103 | fi 104 | 105 | if [[ '${{ github.event.inputs.template-ref }}' != '' ]]; then 106 | ARGS="$ARGS --template-ref='${{ github.event.inputs.template-ref }}'" 107 | fi 108 | 109 | if [[ '${{ github.event.inputs.cookie }}' == '' ]]; then 110 | ARGS="$ARGS '${{ github.repositoryUrl }}'" 111 | else 112 | ARGS="$ARGS '${{ github.event.inputs.cookie }}'" 113 | fi 114 | 115 | echo "args=$ARGS" >> $GITHUB_OUTPUT 116 | - name: "Rebake" 117 | run: | 118 | python -m ntc_cookie_drift_manager rebake ${{ steps.config.outputs.args }} 119 | -------------------------------------------------------------------------------- /.github/workflows/upstream_testing.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "Nautobot Upstream Monitor" 3 | 4 | on: # yamllint disable-line rule:truthy rule:comments 5 | schedule: 6 | - cron: "0 4 */2 * *" # every other day at midnight 7 | workflow_dispatch: 8 | 9 | jobs: 10 | upstream-test: 11 | uses: "nautobot/nautobot/.github/workflows/plugin_upstream_testing_base.yml@develop" 12 | with: # Below could potentially be collapsed into a single argument if a concrete relationship between both is enforced 13 | invoke_context_name: "NAUTOBOT_DATA_VALIDATION_ENGINE" 14 | plugin_name: "nautobot-data-validation-engine" 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ansible Retry Files 2 | *.retry 3 | 4 | # Swap files 5 | *.swp 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | pip-wheel-metadata/ 30 | share/python-wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | MANIFEST 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .nox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | *.cover 56 | *.py,cover 57 | .hypothesis/ 58 | .pytest_cache/ 59 | lcov.info 60 | 61 | # Translations 62 | *.mo 63 | *.pot 64 | 65 | # Django stuff: 66 | *.log 67 | local_settings.py 68 | db.sqlite3 69 | db.sqlite3-journal 70 | 71 | # Flask stuff: 72 | instance/ 73 | .webassets-cache 74 | 75 | # Scrapy stuff: 76 | .scrapy 77 | 78 | # Sphinx documentation 79 | docs/_build/ 80 | 81 | # PyBuilder 82 | target/ 83 | 84 | # Jupyter Notebook 85 | .ipynb_checkpoints 86 | 87 | # IPython 88 | profile_default/ 89 | ipython_config.py 90 | 91 | # pyenv 92 | .python-version 93 | 94 | # pipenv 95 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 96 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 97 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 98 | # install all needed dependencies. 99 | #Pipfile.lock 100 | 101 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 102 | __pypackages__/ 103 | 104 | # Celery stuff 105 | celerybeat-schedule 106 | celerybeat.pid 107 | 108 | # SageMath parsed files 109 | *.sage.py 110 | 111 | # Environments 112 | .env 113 | .venv 114 | env/ 115 | venv/ 116 | ENV/ 117 | env.bak/ 118 | venv.bak/ 119 | 120 | # Spyder project settings 121 | .spyderproject 122 | .spyproject 123 | 124 | # Rope project settings 125 | .ropeproject 126 | 127 | # mkdocs documentation 128 | /site 129 | 130 | # mypy 131 | .mypy_cache/ 132 | .dmypy.json 133 | dmypy.json 134 | 135 | # Pyre type checker 136 | .pyre/ 137 | 138 | # Editor 139 | .vscode/ 140 | 141 | ### macOS ### 142 | # General 143 | .DS_Store 144 | .AppleDouble 145 | .LSOverride 146 | 147 | # Thumbnails 148 | ._* 149 | 150 | # Files that might appear in the root of a volume 151 | .DocumentRevisions-V100 152 | .fseventsd 153 | .Spotlight-V100 154 | .TemporaryItems 155 | .Trashes 156 | .VolumeIcon.icns 157 | .com.apple.timemachine.donotpresent 158 | 159 | # Directories potentially created on remote AFP share 160 | .AppleDB 161 | .AppleDesktop 162 | Network Trash Folder 163 | Temporary Items 164 | .apdisk 165 | 166 | ### Windows ### 167 | # Windows thumbnail cache files 168 | Thumbs.db 169 | Thumbs.db:encryptable 170 | ehthumbs.db 171 | ehthumbs_vista.db 172 | 173 | # Dump file 174 | *.stackdump 175 | 176 | # Folder config file 177 | [Dd]esktop.ini 178 | 179 | # Recycle Bin used on file shares 180 | $RECYCLE.BIN/ 181 | 182 | # Windows Installer files 183 | *.cab 184 | *.msi 185 | *.msix 186 | *.msm 187 | *.msp 188 | 189 | # Windows shortcuts 190 | *.lnk 191 | 192 | ### PyCharm ### 193 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 194 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 195 | 196 | # User-specific stuff 197 | .idea/**/workspace.xml 198 | .idea/**/tasks.xml 199 | .idea/**/usage.statistics.xml 200 | .idea/**/dictionaries 201 | .idea/**/shelf 202 | 203 | # Generated files 204 | .idea/**/contentModel.xml 205 | 206 | # Sensitive or high-churn files 207 | .idea/**/dataSources/ 208 | .idea/**/dataSources.ids 209 | .idea/**/dataSources.local.xml 210 | .idea/**/sqlDataSources.xml 211 | .idea/**/dynamic.xml 212 | .idea/**/uiDesigner.xml 213 | .idea/**/dbnavigator.xml 214 | 215 | # Gradle 216 | .idea/**/gradle.xml 217 | .idea/**/libraries 218 | 219 | # Gradle and Maven with auto-import 220 | # When using Gradle or Maven with auto-import, you should exclude module files, 221 | # since they will be recreated, and may cause churn. Uncomment if using 222 | # auto-import. 223 | # .idea/artifacts 224 | # .idea/compiler.xml 225 | # .idea/jarRepositories.xml 226 | # .idea/modules.xml 227 | # .idea/*.iml 228 | # .idea/modules 229 | # *.iml 230 | # *.ipr 231 | 232 | # CMake 233 | cmake-build-*/ 234 | 235 | # Mongo Explorer plugin 236 | .idea/**/mongoSettings.xml 237 | 238 | # File-based project format 239 | *.iws 240 | 241 | # IntelliJ 242 | out/ 243 | 244 | # mpeltonen/sbt-idea plugin 245 | .idea_modules/ 246 | 247 | # JIRA plugin 248 | atlassian-ide-plugin.xml 249 | 250 | # Cursive Clojure plugin 251 | .idea/replstate.xml 252 | 253 | # Crashlytics plugin (for Android Studio and IntelliJ) 254 | com_crashlytics_export_strings.xml 255 | crashlytics.properties 256 | crashlytics-build.properties 257 | fabric.properties 258 | 259 | # Editor-based Rest Client 260 | .idea/httpRequests 261 | 262 | # Android studio 3.1+ serialized cache file 263 | .idea/caches/build_file_checksums.ser 264 | 265 | ### PyCharm Patch ### 266 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 267 | 268 | # *.iml 269 | # modules.xml 270 | # .idea/misc.xml 271 | # *.ipr 272 | 273 | # Sonarlint plugin 274 | # https://plugins.jetbrains.com/plugin/7973-sonarlint 275 | .idea/**/sonarlint/ 276 | 277 | # SonarQube Plugin 278 | # https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin 279 | .idea/**/sonarIssues.xml 280 | 281 | # Markdown Navigator plugin 282 | # https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced 283 | .idea/**/markdown-navigator.xml 284 | .idea/**/markdown-navigator-enh.xml 285 | .idea/**/markdown-navigator/ 286 | 287 | # Cache file creation bug 288 | # See https://youtrack.jetbrains.com/issue/JBR-2257 289 | .idea/$CACHE_FILE$ 290 | 291 | # CodeStream plugin 292 | # https://plugins.jetbrains.com/plugin/12206-codestream 293 | .idea/codestream.xml 294 | 295 | ### vscode ### 296 | .vscode/* 297 | *.code-workspace 298 | 299 | # Rando 300 | creds.env 301 | development/*.txt 302 | 303 | # Invoke overrides 304 | invoke.yml 305 | 306 | # Docs 307 | public 308 | /compose.yaml 309 | /dump.sql 310 | /nautobot_data_validation_engine/static/nautobot_data_validation_engine/docs 311 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # .readthedocs.yaml 3 | # Read the Docs configuration file 4 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 5 | 6 | # Required 7 | version: 2 8 | 9 | # Set the version of Python in the build environment. 10 | build: 11 | os: "ubuntu-22.04" 12 | tools: 13 | python: "3.10" 14 | 15 | mkdocs: 16 | configuration: "mkdocs.yml" 17 | fail_on_warning: true 18 | 19 | # Use our docs/requirements.txt during installation. 20 | python: 21 | install: 22 | - requirements: "docs/requirements.txt" 23 | -------------------------------------------------------------------------------- /.yamllint.yml: -------------------------------------------------------------------------------- 1 | --- 2 | extends: "default" 3 | rules: 4 | comments: "enable" 5 | empty-values: "disable" 6 | indentation: 7 | indent-sequences: "consistent" 8 | line-length: "disable" 9 | quoted-strings: 10 | quote-type: "double" 11 | ignore: | 12 | .venv/ 13 | compose.yaml 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache Software License 2.0 2 | 3 | Copyright (c) 2025, Network to Code, LLC 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Data Validation Engine 2 | 3 |

4 | 5 |
6 | 7 | 8 | 9 | 10 |
11 | An App for Nautobot. 12 |

13 | 14 | ## Overview 15 | 16 | An app for [Nautobot](https://github.com/nautobot/nautobot) with a UI to build custom data validation rules for Source of Truth data. 17 | 18 | The Data Validation Engine app offers a set of user definable rules which are used to enforce business constraints on the data in Nautobot. These rules are tied to particular models and each rule is meant to enforce one aspect of a business use case. 19 | 20 | Supported rule types include: 21 | 22 | - Regular expression 23 | - Min/max value 24 | - Required fields 25 | - Unique values 26 | 27 | Another feature within the app called [Data Compliance](https://docs.nautobot.com/projects/data-validation/en/latest/user/app_data_compliance/) can audit any object within Nautobot ad-hoc according to a set of rules that you can define programmatically or from the built-in data validation rules. Rather than only checking for adherence to specified rules during the creation or modification of objects, Data Compliance will run a job that produces compliance statuses across all objects including pre-existing ones (such as all existing devices). 28 | 29 | ![Dropdown](https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/develop/docs/images/dropdown.png) 30 | 31 | ### Screenshots 32 | 33 | More screenshots can be found in the [Using the App](https://docs.nautobot.com/projects/data-validation/en/latest/user/app_use_cases/) page in the documentation. Here's a quick overview of some of the app's added functionality: 34 | 35 | **Min/Max Rules** 36 | 37 | ![Min/Max List](https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/develop/docs/images/min-max-rules-list.png) 38 | 39 | **Regular Expression Rules** 40 | 41 | ![Regex Rules List](https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/develop/docs/images/regex-rules-list.png) 42 | 43 | **Required Rules** 44 | 45 | ![Required Rules List](https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/develop/docs/images/required-rules-list.png) 46 | 47 | **Unique Rules** 48 | 49 | ![Unique Rules List](https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/develop/docs/images/unique-rules-list.png) 50 | 51 | **Data Compliance** 52 | 53 | ![Data Compliance Results List](https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/develop/docs/images/data-compliance-results-list.png) 54 | 55 | ## Try it out! 56 | 57 | This App is installed in the Nautobot Community Sandbox found over at [demo.nautobot.com](https://demo.nautobot.com/)! 58 | 59 | > For a full list of all the available always-on sandbox environments, head over to the main page on [networktocode.com](https://www.networktocode.com/nautobot/sandbox-environments/). 60 | 61 | ## Documentation 62 | 63 | Full web-based HTML documentation for this app can be found over on the [Nautobot Docs](https://docs.nautobot.com) website: 64 | 65 | - [User Guide](https://docs.nautobot.com/projects/data-validation/en/latest/user/app_overview/) - Overview, Getting Started, Using the App. 66 | - [Administrator Guide](https://docs.nautobot.com/projects/data-validation/en/latest/admin/install/) - How to Install, Configure, Upgrade, or Uninstall the App. 67 | - [Developer Guide](https://docs.nautobot.com/projects/data-validation/en/latest/dev/contributing/) - Extending the App, Code Reference, Contribution Guide. 68 | - [Release Notes / Changelog](https://docs.nautobot.com/projects/data-validation/en/latest/admin/release_notes/). 69 | - [Frequently Asked Questions](https://docs.nautobot.com/projects/data-validation/en/latest/user/faq/). 70 | 71 | ### Contributing to the Docs 72 | 73 | You can find all the Markdown source for the App documentation under the [docs](https://github.com/nautobot/nautobot-app-data-validation-engine/tree/develop/docs) folder in this repository. For simple edits, a Markdown capable editor is sufficient - clone the repository and edit away. 74 | 75 | If you need to view the fully generated documentation site, you can build it with [mkdocs](https://www.mkdocs.org/). A container hosting the docs will be started using the invoke commands (details in the [Development Environment Guide](https://docs.nautobot.com/projects/data-validation/en/latest/dev/dev_environment/#docker-development-environment)) on [http://localhost:8001](http://localhost:8001). As your changes are saved, the live docs will be automatically reloaded. 76 | 77 | Any PRs with fixes or improvements are very welcome! 78 | 79 | ## Questions 80 | 81 | For any questions or comments, please check the [FAQ](https://docs.nautobot.com/projects/data-validation/en/latest/user/faq/) first. Feel free to also swing by the [Network to Code Slack](https://networktocode.slack.com/) (channel `#nautobot`), sign up [here](http://slack.networktocode.com/) if you don't have an account. 82 | -------------------------------------------------------------------------------- /changes/+nautobot-app-v2-4-1.housekeeping: -------------------------------------------------------------------------------- 1 | Rebaked from the cookie `nautobot-app-v2.4.1`. 2 | -------------------------------------------------------------------------------- /changes/+nautobot-app-v2-4-2.housekeeping: -------------------------------------------------------------------------------- 1 | Rebaked from the cookie `nautobot-app-v2.4.2`. 2 | -------------------------------------------------------------------------------- /changes/+nautobot-app-v2.5.0.housekeeping: -------------------------------------------------------------------------------- 1 | Rebaked from the cookie `nautobot-app-v2.5.0`. 2 | -------------------------------------------------------------------------------- /changes/.gitignore: -------------------------------------------------------------------------------- 1 | !.gitignore 2 | -------------------------------------------------------------------------------- /changes/191.fixed: -------------------------------------------------------------------------------- 1 | Correct class inheritance on Bulk Edit Forms to resolve issue loading the Bulk Edit Views. 2 | -------------------------------------------------------------------------------- /development/Dockerfile: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------------- 2 | # Nautobot App Developement Dockerfile Template 3 | # Version: 1.1.0 4 | # 5 | # Apps that need to add additional steps or packages can do in the section below. 6 | # ------------------------------------------------------------------------------------- 7 | # !!! USE CAUTION WHEN MODIFYING LINES BELOW 8 | 9 | # Accepts a desired Nautobot version as build argument, default to 2.1.9 10 | ARG NAUTOBOT_VER="2.1.9" 11 | 12 | # Accepts a desired Python version as build argument, default to 3.11 13 | ARG PYTHON_VER="3.11" 14 | 15 | # Retrieve published development image of Nautobot base which should include most CI dependencies 16 | FROM ghcr.io/nautobot/nautobot-dev:${NAUTOBOT_VER}-py${PYTHON_VER} 17 | 18 | # Runtime argument and environment setup 19 | ARG NAUTOBOT_ROOT=/opt/nautobot 20 | 21 | ENV prometheus_multiproc_dir=/prom_cache 22 | ENV NAUTOBOT_ROOT=${NAUTOBOT_ROOT} 23 | ENV INVOKE_NAUTOBOT_DATA_VALIDATION_ENGINE_LOCAL=true 24 | 25 | # Install Poetry manually via its installer script; 26 | # We might be using an older version of Nautobot that includes an older version of Poetry 27 | # and CI and local development may have a newer version of Poetry 28 | # Since this is only used for development and we don't ship this container, pinning Poetry back is not expressly necessary 29 | # We also don't need virtual environments in container 30 | RUN which poetry || curl -sSL https://install.python-poetry.org | python3 - && \ 31 | poetry config virtualenvs.create false 32 | 33 | # !!! USE CAUTION WHEN MODIFYING LINES ABOVE 34 | # ------------------------------------------------------------------------------------- 35 | # App-specifc system build/test dependencies. 36 | # 37 | # Example: LDAP requires `libldap2-dev` to be apt-installed before the Python package. 38 | # ------------------------------------------------------------------------------------- 39 | # --> Start safe to modify section 40 | 41 | # Uncomment the lines below if you are apt-installing any package. 42 | # RUN apt-get -y update && apt-get -y install \ 43 | # libldap2-dev \ 44 | # && rm -rf /var/lib/apt/lists/* 45 | 46 | # --> Stop safe to modify section 47 | # ------------------------------------------------------------------------------------- 48 | # Install Nautobot App 49 | # ------------------------------------------------------------------------------------- 50 | # !!! USE CAUTION WHEN MODIFYING LINES BELOW 51 | 52 | # Copy in the source code 53 | WORKDIR /source 54 | COPY . /source 55 | 56 | # Build args must be declared in each stage 57 | ARG NAUTOBOT_VER 58 | ARG PYTHON_VER 59 | 60 | # Constrain the Nautobot version to NAUTOBOT_VER, fall back to installing from git branch if not available on PyPi 61 | # In CI, this should be done outside of the Dockerfile to prevent cross-compile build failures 62 | ARG CI 63 | RUN if [ -z "${CI+x}" ]; then \ 64 | INSTALLED_NAUTOBOT_VER=$(pip show nautobot | grep "^Version" | sed "s/Version: //"); \ 65 | poetry add --lock nautobot@${INSTALLED_NAUTOBOT_VER} --python ${PYTHON_VER} || \ 66 | poetry add --lock git+https://github.com/nautobot/nautobot.git#${NAUTOBOT_VER} --python ${PYTHON_VER}; fi 67 | 68 | # Install the app 69 | RUN poetry install --extras all --with dev 70 | 71 | COPY development/nautobot_config.py ${NAUTOBOT_ROOT}/nautobot_config.py 72 | # !!! USE CAUTION WHEN MODIFYING LINES ABOVE 73 | -------------------------------------------------------------------------------- /development/app_config_schema.py: -------------------------------------------------------------------------------- 1 | """App Config Schema Generator and Validator.""" 2 | 3 | import json 4 | from importlib import import_module 5 | from os import getenv 6 | from pathlib import Path 7 | from urllib.parse import urlparse 8 | 9 | import jsonschema 10 | import toml 11 | from django.conf import settings 12 | from to_json_schema.to_json_schema import SchemaBuilder 13 | 14 | 15 | def _enrich_object_schema(schema, defaults, required): 16 | schema["additionalProperties"] = False 17 | for key, value in schema["properties"].items(): 18 | if required and key in required: 19 | value["required"] = True 20 | default_value = defaults and defaults.get(key, None) 21 | if value["type"] == "object" and "properties" in value: 22 | _enrich_object_schema(value, default_value, None) 23 | elif default_value is not None: 24 | value["default"] = default_value 25 | 26 | 27 | def _main(): 28 | pyproject = toml.loads(Path("pyproject.toml").read_text()) 29 | url = urlparse(pyproject["tool"]["poetry"]["repository"]) 30 | _, owner, repository = url.path.split("/") 31 | package_name = pyproject["tool"]["poetry"]["packages"][0]["include"] 32 | app_config = settings.PLUGINS_CONFIG[package_name] # type: ignore 33 | schema_path = Path(package_name) / "app-config-schema.json" 34 | command = getenv("APP_CONFIG_SCHEMA_COMMAND", "") 35 | if command == "generate": 36 | schema = { 37 | "$schema": "https://json-schema.org/draft/2020-12/schema", 38 | "$id": f"https://raw.githubusercontent.com/{owner}/{repository}/develop/{package_name}/app-config-schema.json", 39 | "$comment": "TBD: Update $id, replace `develop` with the future release tag", 40 | **SchemaBuilder().to_json_schema(app_config), # type: ignore 41 | } 42 | app_config = import_module(package_name).config 43 | _enrich_object_schema(schema, app_config.default_settings, app_config.required_settings) 44 | schema_path.write_text(json.dumps(schema, indent=4) + "\n") 45 | print(f"\n==================\nGenerated schema:\n\n{schema_path}\n") 46 | print( 47 | "WARNING: Review and edit the generated file before committing.\n" 48 | "\n" 49 | "Its content is inferred from:\n" 50 | "\n" 51 | "- The current configuration in `PLUGINS_CONFIG`\n" 52 | "- `NautobotAppConfig.default_settings`\n" 53 | "- `NautobotAppConfig.required_settings`" 54 | ) 55 | elif command == "validate": 56 | schema = json.loads(schema_path.read_text()) 57 | jsonschema.validate(app_config, schema) 58 | print( 59 | f"\n==================\nValidated configuration using the schema:\n{schema_path}\nConfiguration is valid." 60 | ) 61 | else: 62 | raise RuntimeError(f"Unknown command: {command}") 63 | 64 | 65 | _main() 66 | -------------------------------------------------------------------------------- /development/creds.example.env: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # CREDS File: Store private information. Copied to creds.env and always ignored 3 | ################################################################################ 4 | # Nautobot Configuration Secret Items 5 | NAUTOBOT_CREATE_SUPERUSER=true 6 | NAUTOBOT_DB_PASSWORD=changeme 7 | NAUTOBOT_NAPALM_USERNAME='' 8 | NAUTOBOT_NAPALM_PASSWORD='' 9 | NAUTOBOT_REDIS_PASSWORD=changeme 10 | NAUTOBOT_SECRET_KEY='changeme' 11 | NAUTOBOT_SUPERUSER_NAME=admin 12 | NAUTOBOT_SUPERUSER_EMAIL=admin@example.com 13 | NAUTOBOT_SUPERUSER_PASSWORD=admin 14 | NAUTOBOT_SUPERUSER_API_TOKEN=0123456789abcdef0123456789abcdef01234567 15 | 16 | # Postgres 17 | POSTGRES_PASSWORD=${NAUTOBOT_DB_PASSWORD} 18 | PGPASSWORD=${NAUTOBOT_DB_PASSWORD} 19 | 20 | # MySQL Credentials 21 | MYSQL_ROOT_PASSWORD=${NAUTOBOT_DB_PASSWORD} 22 | MYSQL_PASSWORD=${NAUTOBOT_DB_PASSWORD} 23 | 24 | # Use these to override values in development.env 25 | # NAUTOBOT_DB_HOST=localhost 26 | # NAUTOBOT_REDIS_HOST=localhost 27 | # NAUTOBOT_CONFIG=development/nautobot_config.py 28 | -------------------------------------------------------------------------------- /development/development.env: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # DEV File: Store environment information. NOTE: Secrets NOT stored here! 3 | ################################################################################ 4 | # Nautobot Configuration Environment Variables 5 | NAUTOBOT_ALLOWED_HOSTS=* 6 | NAUTOBOT_BANNER_TOP="Local" 7 | NAUTOBOT_CHANGELOG_RETENTION=0 8 | 9 | NAUTOBOT_DEBUG=True 10 | NAUTOBOT_LOG_DEPRECATION_WARNINGS=True 11 | NAUTOBOT_LOG_LEVEL=DEBUG 12 | NAUTOBOT_METRICS_ENABLED=True 13 | NAUTOBOT_NAPALM_TIMEOUT=5 14 | NAUTOBOT_MAX_PAGE_SIZE=0 15 | 16 | # Redis Configuration Environment Variables 17 | NAUTOBOT_REDIS_HOST=redis 18 | NAUTOBOT_REDIS_PORT=6379 19 | # Uncomment NAUTOBOT_REDIS_SSL if using SSL 20 | # NAUTOBOT_REDIS_SSL=True 21 | 22 | # Nautobot DB Connection Environment Variables 23 | NAUTOBOT_DB_NAME=nautobot 24 | NAUTOBOT_DB_USER=nautobot 25 | NAUTOBOT_DB_HOST=db 26 | NAUTOBOT_DB_TIMEOUT=300 27 | 28 | # Use them to overwrite the defaults in nautobot_config.py 29 | # NAUTOBOT_DB_ENGINE=django.db.backends.postgresql 30 | # NAUTOBOT_DB_PORT=5432 31 | 32 | # Needed for Postgres should match the values for Nautobot above 33 | POSTGRES_USER=${NAUTOBOT_DB_USER} 34 | POSTGRES_DB=${NAUTOBOT_DB_NAME} 35 | 36 | # Needed for MYSQL should match the values for Nautobot above 37 | MYSQL_USER=${NAUTOBOT_DB_USER} 38 | MYSQL_DATABASE=${NAUTOBOT_DB_NAME} 39 | MYSQL_ROOT_HOST=% 40 | 41 | # Use a less verbose log level for Celery Beat 42 | NAUTOBOT_BEAT_LOG_LEVEL=INFO 43 | -------------------------------------------------------------------------------- /development/development_mysql.env: -------------------------------------------------------------------------------- 1 | # Custom ENVs for Mysql 2 | # Due to docker image limitations for Mysql, we need "root" user to create more than one database table 3 | NAUTOBOT_DB_USER=root 4 | -------------------------------------------------------------------------------- /development/docker-compose.base.yml: -------------------------------------------------------------------------------- 1 | --- 2 | x-nautobot-build: &nautobot-build 3 | build: 4 | args: 5 | NAUTOBOT_VER: "${NAUTOBOT_VER}" 6 | PYTHON_VER: "${PYTHON_VER}" 7 | context: "../" 8 | dockerfile: "development/Dockerfile" 9 | x-nautobot-base: &nautobot-base 10 | image: "nautobot-data-validation-engine/nautobot:${NAUTOBOT_VER}-py${PYTHON_VER}" 11 | env_file: 12 | - "development.env" 13 | - "creds.env" 14 | tty: true 15 | 16 | services: 17 | nautobot: 18 | depends_on: 19 | redis: 20 | condition: "service_started" 21 | db: 22 | condition: "service_healthy" 23 | <<: 24 | - *nautobot-base 25 | - *nautobot-build 26 | worker: 27 | entrypoint: 28 | - "sh" 29 | - "-c" # this is to evaluate the $NAUTOBOT_LOG_LEVEL from the env 30 | - "nautobot-server celery worker -l $$NAUTOBOT_LOG_LEVEL --events" ## $$ because of docker-compose 31 | depends_on: 32 | nautobot: 33 | condition: "service_healthy" 34 | healthcheck: 35 | interval: "30s" 36 | timeout: "10s" 37 | start_period: "30s" 38 | retries: 3 39 | test: ["CMD", "bash", "-c", "nautobot-server celery inspect ping --destination celery@$$HOSTNAME"] ## $$ because of docker-compose 40 | <<: *nautobot-base 41 | beat: 42 | entrypoint: 43 | - "sh" 44 | - "-c" # this is to evaluate the $NAUTOBOT_BEAT_LOG_LEVEL from the env 45 | - "nautobot-server celery beat -l $$NAUTOBOT_BEAT_LOG_LEVEL" ## $$ because of docker-compose 46 | depends_on: 47 | nautobot: 48 | condition: "service_healthy" 49 | healthcheck: 50 | disable: true 51 | <<: *nautobot-base 52 | -------------------------------------------------------------------------------- /development/docker-compose.dev.yml: -------------------------------------------------------------------------------- 1 | # We can't remove volumes in a compose override, for the test configuration using the final containers 2 | # we don't want the volumes so this is the default override file to add the volumes in the dev case 3 | # any override will need to include these volumes to use them. 4 | # see: https://github.com/docker/compose/issues/3729 5 | --- 6 | services: 7 | nautobot: 8 | command: "nautobot-server runserver 0.0.0.0:8080" 9 | ports: 10 | - "8080:8080" 11 | volumes: 12 | - "./nautobot_config.py:/opt/nautobot/nautobot_config.py" 13 | - "../:/source" 14 | healthcheck: 15 | interval: "30s" 16 | timeout: "10s" 17 | start_period: "60s" 18 | retries: 3 19 | test: ["CMD", "true"] # Due to layering, disable: true won't work. Instead, change the test 20 | docs: 21 | entrypoint: "mkdocs serve -v -a 0.0.0.0:8080" 22 | ports: 23 | - "8001:8080" 24 | volumes: 25 | - "../:/source" 26 | image: "nautobot-data-validation-engine/nautobot:${NAUTOBOT_VER}-py${PYTHON_VER}" 27 | healthcheck: 28 | disable: true 29 | tty: true 30 | worker: 31 | entrypoint: 32 | - "sh" 33 | - "-c" # this is to evaluate the $NAUTOBOT_LOG_LEVEL from the env 34 | - "watchmedo auto-restart --directory './' --pattern '*.py' --recursive -- nautobot-server celery worker -l $$NAUTOBOT_LOG_LEVEL --events" ## $$ because of docker-compose 35 | volumes: 36 | - "./nautobot_config.py:/opt/nautobot/nautobot_config.py" 37 | - "../:/source" 38 | healthcheck: 39 | test: ["CMD", "true"] # Due to layering, disable: true won't work. Instead, change the test 40 | beat: 41 | entrypoint: 42 | - "sh" 43 | - "-c" # this is to evaluate the $NAUTOBOT_BEAT_LOG_LEVEL from the env 44 | - "watchmedo auto-restart --directory './' --pattern '*.py' --recursive -- nautobot-server celery beat -l $$NAUTOBOT_BEAT_LOG_LEVEL" ## $$ because of docker-compose 45 | volumes: 46 | - "./nautobot_config.py:/opt/nautobot/nautobot_config.py" 47 | - "../:/source" 48 | # To expose postgres (5432), myql (3306) on db service or redis (6379) to the host uncomment the 49 | # following. Ensure to match the 2 idented spaces which to have the service nested under services. 50 | # db: 51 | # ports: 52 | # - "5432:5432" 53 | # - "3306:3306" 54 | # redis: 55 | # ports: 56 | # - "6379:6379" 57 | -------------------------------------------------------------------------------- /development/docker-compose.mysql.yml: -------------------------------------------------------------------------------- 1 | --- 2 | services: 3 | nautobot: 4 | environment: 5 | - "NAUTOBOT_DB_ENGINE=django.db.backends.mysql" 6 | env_file: 7 | - "development.env" 8 | - "creds.env" 9 | - "development_mysql.env" 10 | worker: 11 | environment: 12 | - "NAUTOBOT_DB_ENGINE=django.db.backends.mysql" 13 | env_file: 14 | - "development.env" 15 | - "creds.env" 16 | - "development_mysql.env" 17 | beat: 18 | environment: 19 | - "NAUTOBOT_DB_ENGINE=django.db.backends.mysql" 20 | env_file: 21 | - "development.env" 22 | - "creds.env" 23 | - "development_mysql.env" 24 | db: 25 | image: "mysql:lts" 26 | command: 27 | - "--max_connections=1000" 28 | env_file: 29 | - "development.env" 30 | - "creds.env" 31 | - "development_mysql.env" 32 | volumes: 33 | - "mysql_data:/var/lib/mysql" 34 | healthcheck: 35 | test: 36 | - "CMD" 37 | - "mysqladmin" 38 | - "ping" 39 | - "-h" 40 | - "localhost" 41 | timeout: "20s" 42 | retries: 10 43 | volumes: 44 | mysql_data: {} 45 | -------------------------------------------------------------------------------- /development/docker-compose.postgres.yml: -------------------------------------------------------------------------------- 1 | --- 2 | services: 3 | nautobot: 4 | environment: 5 | - "NAUTOBOT_DB_ENGINE=django.db.backends.postgresql" 6 | db: 7 | image: "postgres:17-alpine" 8 | command: 9 | - "-c" 10 | - "max_connections=200" 11 | env_file: 12 | - "development.env" 13 | - "creds.env" 14 | volumes: 15 | - "postgres_data:/var/lib/postgresql/data" 16 | healthcheck: 17 | test: "pg_isready --username=$$POSTGRES_USER --dbname=$$POSTGRES_DB" 18 | interval: "10s" 19 | timeout: "5s" 20 | retries: 10 21 | 22 | volumes: 23 | postgres_data: {} 24 | -------------------------------------------------------------------------------- /development/docker-compose.redis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | services: 3 | redis: 4 | image: "redis:6-alpine" 5 | command: 6 | - "sh" 7 | - "-c" # this is to evaluate the $NAUTOBOT_REDIS_PASSWORD from the env 8 | - "redis-server --appendonly yes --requirepass $$NAUTOBOT_REDIS_PASSWORD" 9 | env_file: 10 | - "development.env" 11 | - "creds.env" 12 | -------------------------------------------------------------------------------- /development/nautobot_config.py: -------------------------------------------------------------------------------- 1 | """Nautobot development configuration file.""" 2 | 3 | import os 4 | import sys 5 | 6 | from nautobot.core.settings import * # noqa: F403 # pylint: disable=wildcard-import,unused-wildcard-import 7 | from nautobot.core.settings_funcs import is_truthy 8 | 9 | # 10 | # Debug 11 | # 12 | 13 | DEBUG = is_truthy(os.getenv("NAUTOBOT_DEBUG", "false")) 14 | _TESTING = len(sys.argv) > 1 and sys.argv[1] == "test" 15 | 16 | if DEBUG and not _TESTING: 17 | DEBUG_TOOLBAR_CONFIG = {"SHOW_TOOLBAR_CALLBACK": lambda _request: True} 18 | 19 | if "debug_toolbar" not in INSTALLED_APPS: # noqa: F405 20 | INSTALLED_APPS.append("debug_toolbar") # noqa: F405 21 | if "debug_toolbar.middleware.DebugToolbarMiddleware" not in MIDDLEWARE: # noqa: F405 22 | MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware") # noqa: F405 23 | 24 | # 25 | # Misc. settings 26 | # 27 | 28 | ALLOWED_HOSTS = os.getenv("NAUTOBOT_ALLOWED_HOSTS", "").split(" ") 29 | SECRET_KEY = os.getenv("NAUTOBOT_SECRET_KEY", "") 30 | 31 | # 32 | # Database 33 | # 34 | 35 | nautobot_db_engine = os.getenv("NAUTOBOT_DB_ENGINE", "django.db.backends.postgresql") 36 | default_db_settings = { 37 | "django.db.backends.postgresql": { 38 | "NAUTOBOT_DB_PORT": "5432", 39 | }, 40 | "django.db.backends.mysql": { 41 | "NAUTOBOT_DB_PORT": "3306", 42 | }, 43 | } 44 | DATABASES = { 45 | "default": { 46 | "NAME": os.getenv("NAUTOBOT_DB_NAME", "nautobot"), # Database name 47 | "USER": os.getenv("NAUTOBOT_DB_USER", ""), # Database username 48 | "PASSWORD": os.getenv("NAUTOBOT_DB_PASSWORD", ""), # Database password 49 | "HOST": os.getenv("NAUTOBOT_DB_HOST", "localhost"), # Database server 50 | "PORT": os.getenv( 51 | "NAUTOBOT_DB_PORT", 52 | default_db_settings[nautobot_db_engine]["NAUTOBOT_DB_PORT"], 53 | ), # Database port, default to postgres 54 | "CONN_MAX_AGE": int(os.getenv("NAUTOBOT_DB_TIMEOUT", "300")), # Database timeout 55 | "ENGINE": nautobot_db_engine, 56 | } 57 | } 58 | 59 | # Ensure proper Unicode handling for MySQL 60 | if DATABASES["default"]["ENGINE"] == "django.db.backends.mysql": 61 | DATABASES["default"]["OPTIONS"] = {"charset": "utf8mb4"} 62 | 63 | # 64 | # Redis 65 | # 66 | 67 | # The django-redis cache is used to establish concurrent locks using Redis. 68 | # Inherited from nautobot.core.settings 69 | # CACHES = {....} 70 | 71 | # 72 | # Celery settings are not defined here because they can be overloaded with 73 | # environment variables. By default they use `CACHES["default"]["LOCATION"]`. 74 | # 75 | 76 | # 77 | # Logging 78 | # 79 | 80 | LOG_LEVEL = "DEBUG" if DEBUG else "INFO" 81 | 82 | # Verbose logging during normal development operation, but quiet logging during unit test execution 83 | if not _TESTING: 84 | LOGGING = { 85 | "version": 1, 86 | "disable_existing_loggers": False, 87 | "formatters": { 88 | "normal": { 89 | "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)s : %(message)s", 90 | "datefmt": "%H:%M:%S", 91 | }, 92 | "verbose": { 93 | "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)-20s %(filename)-15s %(funcName)30s() : %(message)s", 94 | "datefmt": "%H:%M:%S", 95 | }, 96 | }, 97 | "handlers": { 98 | "normal_console": { 99 | "level": "INFO", 100 | "class": "logging.StreamHandler", 101 | "formatter": "normal", 102 | }, 103 | "verbose_console": { 104 | "level": "DEBUG", 105 | "class": "logging.StreamHandler", 106 | "formatter": "verbose", 107 | }, 108 | }, 109 | "loggers": { 110 | "django": {"handlers": ["normal_console"], "level": "INFO"}, 111 | "nautobot": { 112 | "handlers": ["verbose_console" if DEBUG else "normal_console"], 113 | "level": LOG_LEVEL, 114 | }, 115 | }, 116 | } 117 | 118 | # 119 | # Apps 120 | # 121 | 122 | # Enable installed Apps. Add the name of each App to the list. 123 | PLUGINS = ["nautobot_data_validation_engine"] 124 | 125 | # Apps configuration settings. These settings are used by various Apps that the user may have installed. 126 | # Each key in the dictionary is the name of an installed App and its value is a dictionary of settings. 127 | # PLUGINS_CONFIG = { 128 | # 'nautobot_data_validation_engine': { 129 | # 'foo': 'bar', 130 | # 'buzz': 'bazz' 131 | # } 132 | # } 133 | -------------------------------------------------------------------------------- /development/towncrier_template.j2: -------------------------------------------------------------------------------- 1 | 2 | # v{{ versiondata.version.split(".")[:2] | join(".") }} Release Notes 3 | 4 | This document describes all new features and changes in the release. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 5 | 6 | ## Release Overview 7 | 8 | - Major features or milestones 9 | - Changes to compatibility with Nautobot and/or other apps, libraries etc. 10 | 11 | {% if render_title %} 12 | ## [v{{ versiondata.version }} ({{ versiondata.date }})](https://github.com/nautobot/nautobot-app-data-validation-engine/releases/tag/v{{ versiondata.version}}) 13 | 14 | {% endif %} 15 | {% for section, _ in sections.items() %} 16 | {% if sections[section] %} 17 | {% for category, val in definitions.items() if category in sections[section] %} 18 | {% if sections[section][category]|length != 0 %} 19 | ### {{ definitions[category]['name'] }} 20 | 21 | {% if definitions[category]['showcontent'] %} 22 | {% for text, values in sections[section][category].items() %} 23 | {% for item in text.split('\n') %} 24 | {% if values %} 25 | - {{ values|join(', ') }} - {{ item.strip() }} 26 | {% else %} 27 | - {{ item.strip() }} 28 | {% endif %} 29 | {% endfor %} 30 | {% endfor %} 31 | 32 | {% else %} 33 | - {{ sections[section][category]['']|join(', ') }} 34 | 35 | {% endif %} 36 | {% endif %} 37 | {% endfor %} 38 | {% else %} 39 | No significant changes. 40 | 41 | {% endif %} 42 | {% endfor %} 43 | 44 | -------------------------------------------------------------------------------- /docs/admin/compatibility_matrix.md: -------------------------------------------------------------------------------- 1 | # Compatibility Matrix 2 | 3 | | Data Validation Engine Version | Nautobot First Support Version | Nautobot Last Support Version | 4 | | ------------------------------ | ------------------------------ | ----------------------------- | 5 | | 1.0.X | 1.0.0 | 1.3.10 | 6 | | 2.0.X | 1.5.2 | 1.5.24 | 7 | | 2.1.X | 1.5.2 | 1.5.24 | 8 | | 2.2.X | 1.6.0 | 1.99.99 | 9 | | 3.0.X | 2.0.0 | 2.99.99 | 10 | | 3.1.X | 2.0.0 | 2.99.99 | 11 | | 3.2.X | 2.1.9 | 2.99.99 | 12 | -------------------------------------------------------------------------------- /docs/admin/install.md: -------------------------------------------------------------------------------- 1 | # Installing the App in Nautobot 2 | 3 | Here you will find detailed instructions on how to **install** and **configure** the App within your Nautobot environment. 4 | 5 | ## Prerequisites 6 | 7 | - The app is compatible with Nautobot 2.1.9 and higher. 8 | - Databases supported: PostgreSQL, MySQL 9 | 10 | !!! note 11 | Please check the [dedicated page](compatibility_matrix.md) for a full compatibility matrix and the deprecation policy. 12 | 13 | ### Access Requirements 14 | 15 | ## Install Guide 16 | 17 | !!! note 18 | Apps can be installed from the [Python Package Index](https://pypi.org/) or locally. See the [Nautobot documentation](https://docs.nautobot.com/projects/core/en/stable/user-guide/administration/installation/app-install/) for more details. The pip package name for this app is [`nautobot-data-validation-engine`](https://pypi.org/project/nautobot-data-validation-engine/). 19 | 20 | The app is available as a Python package via PyPI and can be installed with `pip`: 21 | 22 | ```shell 23 | pip install nautobot-data-validation-engine 24 | ``` 25 | 26 | To ensure Data Validation Engine is automatically re-installed during future upgrades, create a file named `local_requirements.txt` (if not already existing) in the Nautobot root directory (alongside `requirements.txt`) and list the `nautobot-data-validation-engine` package: 27 | 28 | ```shell 29 | echo nautobot-data-validation-engine >> local_requirements.txt 30 | ``` 31 | 32 | Once installed, the app needs to be enabled in your Nautobot configuration. The following block of code below shows the additional configuration required to be added to your `nautobot_config.py` file: 33 | 34 | - Append `"nautobot_data_validation_engine"` to the `PLUGINS` list. 35 | - Append the `"nautobot_data_validation_engine"` dictionary to the `PLUGINS_CONFIG` dictionary and override any defaults. 36 | 37 | ```python 38 | # In your nautobot_config.py 39 | PLUGINS = ["nautobot_data_validation_engine"] 40 | 41 | # PLUGINS_CONFIG = { 42 | # "nautobot_data_validation_engine": { 43 | # ADD YOUR SETTINGS HERE 44 | # } 45 | # } 46 | ``` 47 | 48 | Once the Nautobot configuration is updated, run the Post Upgrade command (`nautobot-server post_upgrade`) to run migrations and clear any cache: 49 | 50 | ```shell 51 | nautobot-server post_upgrade 52 | ``` 53 | 54 | Then restart (if necessary) the Nautobot services which may include: 55 | 56 | - Nautobot 57 | - Nautobot Workers 58 | - Nautobot Scheduler 59 | 60 | ```shell 61 | sudo systemctl restart nautobot nautobot-worker nautobot-scheduler 62 | ``` 63 | 64 | ## App Configuration -------------------------------------------------------------------------------- /docs/admin/release_notes/index.md: -------------------------------------------------------------------------------- 1 | # Release Notes 2 | 3 | All the published release notes can be found via the navigation menu. All patch releases are included in the same minor release (e.g. `v1.2`) document. 4 | -------------------------------------------------------------------------------- /docs/admin/release_notes/version_1.0.md: -------------------------------------------------------------------------------- 1 | # v1.0 Release Notes 2 | 3 | This document describes all new features and changes in the release `1.0`. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 4 | 5 | ## Release Overview 6 | 7 | Initial release of the Nautobot Data Validation Engine App. 8 | 9 | ## [v1.0.0] - 2021-06-18 10 | 11 | ### Added 12 | 13 | - Regular Expression Rules 14 | - Min/Max Numeric Rules 15 | -------------------------------------------------------------------------------- /docs/admin/release_notes/version_2.0.md: -------------------------------------------------------------------------------- 1 | # v2.0 Release Notes 2 | 3 | This document describes all new features and changes in the release `2.0`. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 4 | 5 | ## Release Overview 6 | 7 | This release contains major new rule types and changes to UI & API behaviour 8 | 9 | ## [v2.0.1] - 2023-04-28 10 | 11 | ### Changed 12 | 13 | - Updated GitHub repo token for ci.yml 14 | - Fixed yaml error when docker-compose >= 2.17.0 15 | - Update dependencies 16 | 17 | ## [v2.0.0] - 2023-04-04 18 | 19 | ### Added 20 | 21 | - [#15] - Added the required field validation rule type 22 | - [#20] - Added the unique field validation rule type 23 | - [#28] - Added support for Jinja2 template context rendering in regular expression validation rules 24 | 25 | ### Changed 26 | 27 | - The UI navigation dropdown menu items have been moved to the Extensibility tab 28 | - The UI URL routes for regular expression and min/max rules have been changed: 29 | - Regular expression rules are now located at `/plugins/data-validation-engine/regex-rules/` 30 | - Min/max rules are now located at `/plugins/data-validation-engine/min-max-rules/` 31 | - The REST API routes for regular expression and min/max rules have been changed: 32 | - Regular expression rules are now located at `/api/plugins/data-validation-engine/regex-rules/` 33 | - Min/max rules are now located at `/api/plugins/data-validation-engine/min-max-rules/` 34 | - The app's code has been refactored to align with the `cookiecutter-ntc` template 35 | -------------------------------------------------------------------------------- /docs/admin/release_notes/version_2.1.md: -------------------------------------------------------------------------------- 1 | # v2.1 Release Notes 2 | 3 | This document describes all new features and changes in the release `2.1`. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 4 | 5 | ## Release Overview 6 | 7 | This release adds DataCompliance and DataComplianceRules to allow auditing of data within Nautobot programatically. 8 | 9 | ## [v2.1.1] - 2023-08-22 10 | 11 | ### Changed 12 | 13 | - Dependency updates 14 | - Additional Data Compliance docs and images 15 | - Pin poetry to v1.5.1 due to dropping Python 3.7 support 16 | 17 | ### Fixed 18 | 19 | - Fixed mkdocs bug encountered on Python 3.7 20 | - Fixed bug where empty values were flagged by Unique Validation Rules 21 | - Modified Unique Validation to exclude current object from unique count 22 | 23 | ## [v2.1.0] - 2023-05-17 24 | 25 | ### Added 26 | 27 | - [#50] - Creation of DataCompliance and DataComplianceRules -------------------------------------------------------------------------------- /docs/admin/release_notes/version_2.2.md: -------------------------------------------------------------------------------- 1 | # v2.2 Release Notes 2 | 3 | This document describes all new features and changes in the release `2.2`. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 4 | 5 | ## Release Overview 6 | 7 | This release adds support for Nautobot v1.6 LTM which also removes support for Python 3.7. 8 | 9 | ## [v2.2.0] - 2023-09-08 10 | 11 | ### Changed 12 | 13 | - Dependency updates 14 | - Dropping of Python 3.7 support 15 | - Bump minimum Nautobot to v1.6.0 -------------------------------------------------------------------------------- /docs/admin/release_notes/version_3.0.md: -------------------------------------------------------------------------------- 1 | # v3.0 Release Notes 2 | 3 | This document describes all new features and changes in the release `3.0`. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 4 | 5 | ## Release Overview 6 | 7 | This release adds support for Nautobot v2.0.0. 8 | 9 | ## [v3.0.2] - 2023-12-29 10 | 11 | ### Added 12 | 13 | - Additional `invoke` tasks including `export`, `backup_db`, `import_db` 14 | 15 | ### Changed 16 | 17 | - Updates from rebaked cookie using Drift Manager 18 | - Renamed `plugin` to `app` throughout code 19 | - Dependency updates 20 | 21 | ## [v3.0.1] - 2023-10-19 22 | 23 | ### Added 24 | 25 | - Added migration `0060_add_field_defaults` 26 | 27 | ### Changed 28 | 29 | - Dependency updates 30 | 31 | ## [v3.0.0] - 2023-09-29 32 | 33 | ### Added 34 | 35 | - Added Tags to validation rules 36 | - Added `pylint-nautobot` 37 | - Re-added healthcheck to `docker-compose` 38 | - Added `0004_created_datetime` migration 39 | - Added `0005_remove_slugs_alter_tags` migration 40 | - Uses natural_key in place of slugs 41 | 42 | ### Changed 43 | 44 | - Nautobot v2.0 updates following `from-v1` migration guides 45 | - Changed rule urls to use UUID instead of slug 46 | - Changed LogLevelChoices.LOG_SUCCESS to __.LOG_INFO 47 | - Updated jobs logging and restering 48 | - Moved from Site & Region to Location model 49 | - Changed filter fields to use `__all__` 50 | - Updates to form parent classes 51 | - Dependency updates 52 | 53 | ### Removed 54 | 55 | - Removed the use of slugs 56 | - Removed nested serializers 57 | -------------------------------------------------------------------------------- /docs/admin/release_notes/version_3.1.md: -------------------------------------------------------------------------------- 1 | # v3.1 Release Notes 2 | 3 | This document describes all new features and changes in the release `3.1`. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 4 | 5 | ## Release Overview 6 | 7 | This release adds additional functionality to the Data Compliance feature with the ability to now include built-in data validation rules. 8 | 9 | ## [v3.1.1 (2024-04-15)](https://github.com/nautobot/nautobot-app-data-validation-engine/releases/tag/v3.1.1) 10 | 11 | ### Security 12 | 13 | - [#144](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/144) - Updated `cryptography` dependency to 42.0.0 due to CVE-2023-50782. 14 | - [#145](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/145) - Updated `django` dependency to `3.2.24` due to CVE-2024-24680. 15 | - [#148](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/148) - Updated `cryptography` dependency to 42.0.4 due to CVE-2024-26130 and CVE-2024-0727. 16 | - [#153](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/153) - Updated `django` dependency to `3.2.25` due to CVE-2024-27351. 17 | - [#154](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/154) - Updated `black` dependency to `24.3.0` due to CVE-2024-21503. 18 | - [#157](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/157) - Updated `idna` dependency to `3.7` due to CVE-2024-3651. 19 | 20 | ### Fixed 21 | 22 | - [#155](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/155) - Fixed issues where going to "Data Compliance" tab could potentially hide other tabs. 23 | 24 | ### Housekeeping 25 | 26 | - [#150](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/150), [#152](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/152) - Re-baked from the latest template. 27 | 28 | ## v3.1.0 (2024-02-02) 29 | 30 | ### Added 31 | 32 | - Added built-in validation rules (Min/Max, Regex, Required, Unique) to Data Compliance. 33 | - Added check-box option to Data Compliance job for built-in rules. 34 | - Added link to Data Compliance results within job logging. 35 | 36 | ### Changed 37 | 38 | - [#141](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/141) - Replaced pydocstyle with ruff. 39 | - Updated compliance job logging. 40 | - Updated data compliance comments. 41 | - Updated with drift manager inconsistencies. 42 | -------------------------------------------------------------------------------- /docs/admin/release_notes/version_3.2.md: -------------------------------------------------------------------------------- 1 | # v3.2 Release Notes 2 | 3 | This document describes all new features and changes in the release. The format is based on [Keep a 4 | Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic 5 | Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## Release Overview 8 | 9 | - Now supports Python 3.12. 10 | 11 | ## [v3.2.0 (2024-11-04)](https://github.com/nautobot/nautobot-app-data-validation-engine/releases/tag/v3.2.0) 12 | 13 | ### Security 14 | 15 | - [#160](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/160) - Updated `sqlparse` dependency to `0.5.0` due to GHSA-2m57-hf25-phgg. 16 | - [#163](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/163) - Updated `jinja2` dependency to `3.1.4` due to CVE-2024-34064. 17 | - [#167](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/167) - Updated `requests` dependency to `2.32.2` due to CVE-2024-35195. 18 | - [#171](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/171) - Updated `urllib3` dependency to `2.2.2` due to CVE-2024-37891. 19 | 20 | ### Added 21 | 22 | - [#162](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/162) - Added view name to `OrderedDefaultRouter`. 23 | - [#177](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/177) - Added support for Python 3.12. 24 | - [#183](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/183) - Added support for filtering by Compliance Class Name with a name longer than twenty characters and to filter by multiple names at the same time. 25 | 26 | ### Changed 27 | 28 | - [#146](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/146) - Updated app images with screenshots from Nautobot 2.X UI. 29 | - [#146](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/146) - Changed references of `site` to `location` in docs. 30 | - [#162](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/162) - Updated minimum Nautobot version to `2.1.9`. 31 | - [#162](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/162) - Disabled specific `nb-use-fields-all` and `nb-sub-class-name` pylint rules in `tables.py`. 32 | 33 | ### Removed 34 | 35 | - [#162](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/162) - Removed `DataValidationEngineRootView` class and `APIRootView` override. 36 | - [#162](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/162) - Removed `version` from docker-compose files. 37 | - [#163](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/163) - Removed authentication/password command from MySQL Docker Compose. 38 | 39 | ### Housekeeping 40 | 41 | - [#0](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/0) - Rebaked from the cookie `nautobot-app-v2.4.0`. 42 | - [#174](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/174) - Rebake with 2.3.0 Cookiecutter. 43 | - [#177](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/177) - Rebaked with nautobot-app-v2.3.2 Cookiecutter. 44 | - [#185](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/185) - Changed model_class_name in .cookiecutter.json to a valid model to help with drift management. 45 | -------------------------------------------------------------------------------- /docs/admin/uninstall.md: -------------------------------------------------------------------------------- 1 | # Uninstall the App from Nautobot 2 | 3 | Here you will find any steps necessary to cleanly remove the App from your Nautobot environment. 4 | 5 | ## Database Cleanup 6 | 7 | Prior to removing the app from the `nautobot_config.py`, run the following command to roll back any migration specific to this app. 8 | 9 | ```shell 10 | nautobot-server migrate nautobot_data_validation_engine zero 11 | ``` 12 | 13 | ## Remove App configuration 14 | 15 | Remove the configuration you added in `nautobot_config.py` from `PLUGINS` & `PLUGINS_CONFIG`. 16 | 17 | ## Uninstall the package 18 | 19 | ```bash 20 | $ pip3 uninstall nautobot-data-validation-engine 21 | ``` 22 | -------------------------------------------------------------------------------- /docs/admin/upgrade.md: -------------------------------------------------------------------------------- 1 | # Upgrading the App 2 | 3 | Here you will find any steps necessary to upgrade the App in your Nautobot environment. 4 | 5 | ## Upgrade Guide 6 | 7 | When a new release comes out it may be necessary to run a migration of the database to account for any changes in the data models used by this app. Execute the command `nautobot-server post-upgrade` within the runtime environment of your Nautobot installation after updating the `nautobot-data-validation-engine` package via `pip`. 8 | -------------------------------------------------------------------------------- /docs/assets/extra.css: -------------------------------------------------------------------------------- 1 | :root>* { 2 | --md-accent-fg-color: #ff8504; 3 | --md-primary-fg-color: #ff8504; 4 | --md-typeset-a-color: #0097ff; 5 | } 6 | 7 | [data-md-color-scheme="slate"] { 8 | --md-default-bg-color: hsla(var(--md-hue), 0%, 15%, 1); 9 | --md-typeset-a-color: #0097ff; 10 | } 11 | 12 | /* Accessibility: Increase fonts for dark theme */ 13 | [data-md-color-scheme="slate"] .md-typeset { 14 | font-size: 0.9rem; 15 | } 16 | 17 | [data-md-color-scheme="slate"] .md-typeset table:not([class]) { 18 | font-size: 0.7rem; 19 | } 20 | 21 | /* 22 | * The default max-width is 61rem which does not provide nearly enough space to present code examples or larger tables 23 | */ 24 | .md-grid { 25 | margin-left: auto; 26 | margin-right: auto; 27 | max-width: 95%; 28 | } 29 | 30 | .md-tabs__link { 31 | font-size: 0.8rem; 32 | } 33 | 34 | .md-tabs__link--active { 35 | color: var(--md-primary-fg-color); 36 | } 37 | 38 | .md-header__button.md-logo :is(img, svg) { 39 | height: 2rem; 40 | } 41 | 42 | .md-header__button.md-logo :-webkit-any(img, svg) { 43 | height: 2rem; 44 | } 45 | 46 | .md-header__title { 47 | font-size: 1.2rem; 48 | } 49 | 50 | img.logo { 51 | height: 200px; 52 | } 53 | 54 | img.copyright-logo { 55 | height: 24px; 56 | vertical-align: middle; 57 | } 58 | 59 | [data-md-color-primary=black] .md-header { 60 | background-color: #212121; 61 | } 62 | 63 | @media screen and (min-width: 76.25em) { 64 | [data-md-color-primary=black] .md-tabs { 65 | background-color: #212121; 66 | } 67 | } 68 | 69 | /* Customization for mkdocstrings */ 70 | /* Indentation. */ 71 | div.doc-contents:not(.first) { 72 | padding-left: 25px; 73 | border-left: .2rem solid var(--md-typeset-table-color); 74 | } 75 | 76 | /* Mark external links as such. */ 77 | a.autorefs-external::after { 78 | /* https://primer.style/octicons/arrow-up-right-24 */ 79 | background-image: url('data:image/svg+xml,'); 80 | content: ' '; 81 | 82 | display: inline-block; 83 | position: relative; 84 | top: 0.1em; 85 | margin-left: 0.2em; 86 | margin-right: 0.1em; 87 | 88 | height: 1em; 89 | width: 1em; 90 | border-radius: 100%; 91 | background-color: var(--md-typeset-a-color); 92 | } 93 | 94 | a.autorefs-external:hover::after { 95 | background-color: var(--md-accent-fg-color); 96 | } 97 | 98 | 99 | /* Customization for markdown-version-annotations */ 100 | :root { 101 | /* Icon for "version-added" admonition: Material Design Icons "plus-box-outline" */ 102 | --md-admonition-icon--version-added: url('data:image/svg+xml;charset=utf-8,'); 103 | /* Icon for "version-changed" admonition: Material Design Icons "delta" */ 104 | --md-admonition-icon--version-changed: url('data:image/svg+xml;charset=utf-8,'); 105 | /* Icon for "version-removed" admonition: Material Design Icons "minus-circle-outline" */ 106 | --md-admonition-icon--version-removed: url('data:image/svg+xml;charset=utf-8,'); 107 | } 108 | 109 | /* "version-added" admonition in green */ 110 | .md-typeset .admonition.version-added, 111 | .md-typeset details.version-added { 112 | border-color: rgb(0, 200, 83); 113 | } 114 | 115 | .md-typeset .version-added>.admonition-title, 116 | .md-typeset .version-added>summary { 117 | background-color: rgba(0, 200, 83, .1); 118 | } 119 | 120 | .md-typeset .version-added>.admonition-title::before, 121 | .md-typeset .version-added>summary::before { 122 | background-color: rgb(0, 200, 83); 123 | -webkit-mask-image: var(--md-admonition-icon--version-added); 124 | mask-image: var(--md-admonition-icon--version-added); 125 | } 126 | 127 | /* "version-changed" admonition in orange */ 128 | .md-typeset .admonition.version-changed, 129 | .md-typeset details.version-changed { 130 | border-color: rgb(255, 145, 0); 131 | } 132 | 133 | .md-typeset .version-changed>.admonition-title, 134 | .md-typeset .version-changed>summary { 135 | background-color: rgba(255, 145, 0, .1); 136 | } 137 | 138 | .md-typeset .version-changed>.admonition-title::before, 139 | .md-typeset .version-changed>summary::before { 140 | background-color: rgb(255, 145, 0); 141 | -webkit-mask-image: var(--md-admonition-icon--version-changed); 142 | mask-image: var(--md-admonition-icon--version-changed); 143 | } 144 | 145 | /* "version-removed" admonition in red */ 146 | .md-typeset .admonition.version-removed, 147 | .md-typeset details.version-removed { 148 | border-color: rgb(255, 82, 82); 149 | } 150 | 151 | .md-typeset .version-removed>.admonition-title, 152 | .md-typeset .version-removed>summary { 153 | background-color: rgba(255, 82, 82, .1); 154 | } 155 | 156 | .md-typeset .version-removed>.admonition-title::before, 157 | .md-typeset .version-removed>summary::before { 158 | background-color: rgb(255, 82, 82); 159 | -webkit-mask-image: var(--md-admonition-icon--version-removed); 160 | mask-image: var(--md-admonition-icon--version-removed); 161 | } 162 | 163 | /* Do not wrap code blocks in markdown tables. */ 164 | div.md-typeset__table>table>tbody>tr>td>code { 165 | white-space: nowrap; 166 | } 167 | -------------------------------------------------------------------------------- /docs/assets/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/assets/favicon.ico -------------------------------------------------------------------------------- /docs/assets/nautobot_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/assets/nautobot_logo.png -------------------------------------------------------------------------------- /docs/assets/networktocode_bw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/assets/networktocode_bw.png -------------------------------------------------------------------------------- /docs/assets/overrides/partials/copyright.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | {% if config.copyright %} 4 |
5 | {{ config.copyright }} 6 |
7 | Apache-2.0 LICENSE 8 | {% endif %} 9 |
10 | 11 |
12 | Made with Material for MkDocs 13 | | Join #Nautobot on the Network to Code Slack 14 | {% if config.extra.ntc_sponsor == true %} 15 | | Sponsored by 17 | {% endif %} 18 |
19 | 20 | 21 |
22 | 23 | -------------------------------------------------------------------------------- /docs/dev/code_reference/api.md: -------------------------------------------------------------------------------- 1 | # Data Validation Engine API Package 2 | 3 | ::: nautobot_data_validation_engine.api 4 | options: 5 | show_submodules: True 6 | -------------------------------------------------------------------------------- /docs/dev/code_reference/index.md: -------------------------------------------------------------------------------- 1 | # Code Reference 2 | 3 | Auto-generated code reference documentation from docstrings. 4 | -------------------------------------------------------------------------------- /docs/dev/code_reference/package.md: -------------------------------------------------------------------------------- 1 | ::: nautobot_data_validation_engine 2 | -------------------------------------------------------------------------------- /docs/dev/contributing.md: -------------------------------------------------------------------------------- 1 | # Contributing to the App 2 | 3 | The project is packaged with a light [development environment](dev_environment.md) based on `docker-compose` to help with the local development of the project and to run tests. 4 | 5 | The project is following Network to Code software development guidelines and is leveraging the following: 6 | 7 | - Python linting and formatting: `pylint` and `ruff`. 8 | - YAML linting is done with `yamllint`. 9 | - Django unit test to ensure the app is working properly. 10 | 11 | Documentation is built using [mkdocs](https://www.mkdocs.org/). The [Docker based development environment](dev_environment.md#docker-development-environment) automatically starts a container hosting a live version of the documentation website on [http://localhost:8001](http://localhost:8001) that auto-refreshes when you make any changes to your local files. 12 | 13 | ## Creating Changelog Fragments 14 | 15 | All pull requests to `next` or `develop` must include a changelog fragment file in the `./changes` directory. To create a fragment, use your GitHub issue number and fragment type as the filename. For example, `2362.added`. Valid fragment types are `added`, `changed`, `deprecated`, `fixed`, `removed`, and `security`. The change summary is added to the file in plain text. Change summaries should be complete sentences, starting with a capital letter and ending with a period, and be in past tense. Each line of the change fragment will generate a single change entry in the release notes. Use multiple lines in the same file if your change needs to generate multiple release notes in the same category. If the change needs to create multiple entries in separate categories, create multiple files. 16 | 17 | !!! example 18 | 19 | **Wrong** 20 | ```plaintext title="changes/1234.fixed" 21 | fix critical bug in documentation 22 | ``` 23 | 24 | **Right** 25 | ```plaintext title="changes/1234.fixed" 26 | Fixed critical bug in documentation. 27 | ``` 28 | 29 | !!! example "Multiple Entry Example" 30 | 31 | This will generate 2 entries in the `fixed` category and one entry in the `changed` category. 32 | 33 | ```plaintext title="changes/1234.fixed" 34 | Fixed critical bug in documentation. 35 | Fixed release notes generation. 36 | ``` 37 | 38 | ```plaintext title="changes/1234.changed" 39 | Changed release notes generation. 40 | ``` 41 | 42 | ## Branching Policy 43 | 44 | The branching policy includes the following tenets: 45 | 46 | - The `develop` branch is the branch of the next major and minor paired version planned. 47 | - PRs intended to add new features should be sourced from the `develop` branch. 48 | - PRs intended to fix issues in the Nautobot LTM compatible release should be sourced from the latest `ltm-` branch instead of `develop`. 49 | 50 | Data Validation Engine will observe semantic versioning, as of 1.0. This may result in a quick turnaround in minor versions to keep pace with an ever-growing feature set. 51 | 52 | ### Backporting to Older Releases 53 | 54 | If you are backporting any fixes to a prior major or minor version of this app, please open an issue, comment on an existing issue, or post in the [Network to Code Slack](https://networktocode.slack.com/) (channel `#nautobot`). 55 | 56 | We will create a `release-X.Y` branch for you to open your PR against and cut a new release once the PR is successfully merged. 57 | 58 | ## Release Policy 59 | 60 | Data Validation Engine has currently no intended scheduled release schedule, and will release new features in minor versions. 61 | 62 | The steps taken by maintainers when creating a new release are documented in the [release checklist](./release_checklist.md). 63 | -------------------------------------------------------------------------------- /docs/dev/extending.md: -------------------------------------------------------------------------------- 1 | # Extending the App 2 | 3 | Extending the application is welcome, however it is best to open an issue first, to ensure that a PR would be accepted and makes sense in terms of features and design. 4 | 5 | -------------------------------------------------------------------------------- /docs/dev/release_checklist.md: -------------------------------------------------------------------------------- 1 | # Release Checklist 2 | 3 | This document is intended for app maintainers and outlines the steps to perform when releasing a new version of the app. 4 | 5 | !!! important 6 | Before starting, make sure your **local** `develop`, `main`, and (if applicable) the current LTM branch are all up to date with upstream! 7 | 8 | ``` 9 | git fetch 10 | git switch develop && git pull # and repeat for main/ltm 11 | ``` 12 | 13 | Choose your own adventure: 14 | 15 | - LTM release? Jump [here](#ltm-releases). 16 | - Patch release from `develop`? Jump [here](#all-releases-from-develop). 17 | - Minor release? Continue with [Minor Version Bumps](#minor-version-bumps) and then [All Releases from `develop`](#all-releases-from-develop). 18 | 19 | ## Minor Version Bumps 20 | 21 | ### Update Requirements 22 | 23 | Every minor version release should refresh `poetry.lock`, so that it lists the most recent stable release of each package. To do this: 24 | 25 | 0. Run `poetry update --dry-run` to have Poetry automatically tell you what package updates are available and the versions it would upgrade to. This requires an existing environment created from the lock file (i.e. via `poetry install`). 26 | 1. Review each requirement's release notes for any breaking or otherwise noteworthy changes. 27 | 2. Run `poetry update ` to update the package versions in `poetry.lock` as appropriate. 28 | 3. If a required package requires updating to a new release not covered in the version constraints for a package as defined in `pyproject.toml`, (e.g. `Django ~3.1.7` would never install `Django >=4.0.0`), update it manually in `pyproject.toml`. 29 | 4. Run `poetry install` to install the refreshed versions of all required packages. 30 | 5. Run all tests (`poetry run invoke tests`) and check that the UI and API function as expected. 31 | 32 | ### Update Documentation 33 | 34 | If there are any changes to the compatibility matrix (such as a bump in the minimum supported Nautobot version), update it accordingly. 35 | 36 | Commit any resulting changes from the following sections to the documentation before proceeding with the release. 37 | 38 | !!! tip 39 | Fire up the documentation server in your development environment with `poetry run mkdocs serve`! This allows you to view the documentation site locally (the link is in the output of the command) and automatically rebuilds it as you make changes. 40 | 41 | ### Verify the Installation and Upgrade Steps 42 | 43 | Follow the [installation instructions](../admin/install.md) to perform a new production installation of the app. If possible, also test the [upgrade process](../admin/upgrade.md) from the previous released version. 44 | 45 | The goal of this step is to walk through the entire install process *as documented* to make sure nothing there needs to be changed or updated, to catch any errors or omissions in the documentation, and to ensure that it is current with each release. 46 | 47 | --- 48 | 49 | ## All Releases from `develop` 50 | 51 | ### Verify CI Build Status 52 | 53 | Ensure that continuous integration testing on the `develop` branch is completing successfully. 54 | 55 | ### Bump the Version 56 | 57 | Update the package version using `poetry version` if necessary. This command shows the current version of the project or bumps the version of the project and writes the new version back to `pyproject.toml` if a valid bump rule is provided. 58 | 59 | The new version must be a valid semver string or a valid bump rule: `patch`, `minor`, `major`, `prepatch`, `preminor`, `premajor`, `prerelease`. Always try to use a bump rule when you can. 60 | 61 | Display the current version with no arguments: 62 | 63 | ```no-highlight 64 | > poetry version 65 | nautobot-data-validation-engine 1.0.0-beta.2 66 | ``` 67 | 68 | Bump pre-release versions using `prerelease`: 69 | 70 | ```no-highlight 71 | > poetry version prerelease 72 | Bumping version from 1.0.0-beta.2 to 1.0.0-beta.3 73 | ``` 74 | 75 | For major versions, use `major`: 76 | 77 | ```no-highlight 78 | > poetry version major 79 | Bumping version from 1.0.0-beta.2 to 1.0.0 80 | ``` 81 | 82 | For patch versions, use `minor`: 83 | 84 | ```no-highlight 85 | > poetry version minor 86 | Bumping version from 1.0.0 to 1.1.0 87 | ``` 88 | 89 | And lastly, for patch versions, you guessed it, use `patch`: 90 | 91 | ```no-highlight 92 | > poetry version patch 93 | Bumping version from 1.1.0 to 1.1.1 94 | ``` 95 | 96 | Please see the [official Poetry documentation on `version`](https://python-poetry.org/docs/cli/#version) for more information. 97 | 98 | ### Update the Changelog 99 | 100 | !!! important 101 | The changelog must adhere to the [Keep a Changelog](https://keepachangelog.com/) style guide. 102 | 103 | This guide uses `1.4.2` as the new version in its examples, so change it to match the version you bumped to in the previous step! Every. single. time. you. copy/paste commands :) 104 | 105 | First, create a release branch off of `develop` (`git switch -c release-1.4.2 develop`). 106 | 107 | > You will need to have the project's poetry environment built at this stage, as the towncrier command runs **locally only**. If you don't have it, run `poetry install` first. 108 | 109 | Generate release notes with `invoke generate-release-notes --version 1.4.2` and answer `yes` to the prompt `Is it okay if I remove those files? [Y/n]:`. This will update the release notes in `docs/admin/release_notes/version_X.Y.md`, stage that file in git, and `git rm` all the fragments that have now been incorporated into the release notes. 110 | 111 | There are two possibilities: 112 | 113 | 1. If you're releasing a new major or minor version, rename the `version_X.Y.md` file accordingly (e.g. rename to `docs/admin/release_notes/version_1.4.md`). Update the `Release Overview` and add this new page to the table of contents within `mkdocs.yml`. 114 | 2. If you're releasing a patch version, copy your version's section from the `version_X.Y.md` file into the already existing `docs/admin/release_notes/version_1.4.md` file. Delete the `version_X.Y.md` file. 115 | 116 | Stage all the changes (`git add`) and check the diffs to verify all of the changes are correct (`git diff --cached`). 117 | 118 | Commit `git commit -m "Release v1.4.2"` and `git push` the staged changes. 119 | 120 | ### Submit Release Pull Request 121 | 122 | Submit a pull request titled `Release v1.4.2` to merge your release branch into `main`. Copy the documented release notes into the pull request's body. 123 | 124 | !!! important 125 | Do not squash merge this branch into `main`. Make sure to select `Create a merge commit` when merging in GitHub. 126 | 127 | Once CI has completed on the PR, merge it. 128 | 129 | ### Create a New Release in GitHub 130 | 131 | Draft a [new release](https://github.com/nautobot/nautobot-app-data-validation-engine/releases/new) with the following parameters. 132 | 133 | * **Tag:** Input current version (e.g. `v1.4.2`) and select `Create new tag: v1.4.2 on publish` 134 | * **Target:** `main` 135 | * **Title:** Version and date (e.g. `v1.4.2 - 2024-04-02`) 136 | 137 | Click "Generate Release Notes" and edit the auto-generated content as follows: 138 | 139 | - Change the entries generated by GitHub to only the usernames of the contributors. e.g. `* Updated dockerfile by @nautobot_user in https://github.com/nautobot/nautobot-app-data-validation-engine/pull/123` -> `* @nautobot_user`. 140 | - This should give you the list for the new `Contributors` section. 141 | - Make sure there are no duplicated entries. 142 | - Replace the content of the `What's Changed` section with the description of changes from the release PR (what towncrier generated). 143 | - If it exists, leave the `New Contributors` list as it is. 144 | 145 | The release notes should look as follows: 146 | 147 | ```markdown 148 | ## What's Changed 149 | 150 | **Towncrier generated Changed/Fixed/Housekeeping etc. sections here** 151 | 152 | ## Contributors 153 | 154 | * @alice 155 | * @bob 156 | 157 | ## New Contributors 158 | 159 | * @bob 160 | 161 | **Full Changelog**: https://github.com/nautobot/nautobot-app-data-validation-engine/compare/v1.4.1...v1.4.2 162 | ``` 163 | 164 | Publish the release! 165 | 166 | ### Create a PR from `main` back to `develop` 167 | 168 | First, sync your `main` branch with upstream changes: `git switch main && git pull`. 169 | 170 | Create a new branch from `main` called `release-1.4.2-to-develop` and use `poetry version prepatch` to bump the development version to the next release. 171 | 172 | For example, if you just released `v1.4.2`: 173 | 174 | ```no-highlight 175 | > git switch -c release-1.4.2-to-develop main 176 | Switched to a new branch 'release-1.4.2-to-develop' 177 | 178 | > poetry version prepatch 179 | Bumping version from 1.4.2 to 1.4.3a1 180 | 181 | > git add pyproject.toml && git commit -m "Bump version" 182 | 183 | > git push 184 | ``` 185 | 186 | !!! important 187 | Do not squash merge this branch into `develop`. Make sure to select `Create a merge commit` when merging in GitHub. 188 | 189 | Open a new PR from `release-1.4.2-to-develop` against `develop`, wait for CI to pass, and merge it. 190 | 191 | ### Final checks 192 | 193 | At this stage, the CI should be running or finished for the `v1.4.2` tag and a package successfully published to PyPI and added into the GitHub Release. Double check that's the case. 194 | 195 | Documentation should also have been built for the tag on ReadTheDocs and if you're reading this page online, refresh it and look for the new version in the little version fly-out menu down at the bottom right of the page. 196 | 197 | All done! 198 | 199 | 200 | ## LTM Releases 201 | 202 | For projects maintaining a Nautobot LTM compatible release, all development and release management is done through the `ltm-x.y` branch. The `x.y` relates to the LTM version of Nautobot it's compatible with, for example `1.6`. 203 | 204 | The process is similar to releasing from `develop`, but there is no need for post-release branch syncing because you'll release directly from the LTM branch: 205 | 206 | 1. Make sure your `ltm-1.6` branch is passing CI. 207 | 2. Create a release branch from the `ltm-1.6` branch: `git switch -c release-1.2.3 ltm-1.6`. 208 | 3. Bump up the patch version `poetry version patch`. If you're backporting a feature instead of bugfixes, bump the minor version instead with `poetry version minor`. 209 | 4. Generate the release notes: `invoke generate-release-notes --version 1.2.3`. 210 | 5. Move the release notes from the generated `docs/admin/release_notes/version_X.Y.md` to `docs/admin/release_notes/version_1.2.md`. 211 | 6. Add all the changes and `git commit -m "Release v1.2.3"`, then `git push`. 212 | 7. Open a new PR against `ltm-1.6`. Once CI is passing in the PR, `Create a merge commit` (don't squash!). 213 | 8. Create a New Release in GitHub - use the same steps documented [here](#create-a-new-release-in-github). 214 | 9. Open a separate PR against `develop` to synchronize all LTM release changelogs into the latest version of the docs for visibility. 215 | -------------------------------------------------------------------------------- /docs/images/data-compliance-filtered-results-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/data-compliance-filtered-results-list.png -------------------------------------------------------------------------------- /docs/images/data-compliance-object-tab-invalid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/data-compliance-object-tab-invalid.png -------------------------------------------------------------------------------- /docs/images/data-compliance-object-tab-valid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/data-compliance-object-tab-valid.png -------------------------------------------------------------------------------- /docs/images/data-compliance-results-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/data-compliance-results-list.png -------------------------------------------------------------------------------- /docs/images/data-compliance-run-registered-data-compliance-rules-job.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/data-compliance-run-registered-data-compliance-rules-job.png -------------------------------------------------------------------------------- /docs/images/dropdown.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/dropdown.png -------------------------------------------------------------------------------- /docs/images/icon-DataValidationEngine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/icon-DataValidationEngine.png -------------------------------------------------------------------------------- /docs/images/min-max-rules-edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/min-max-rules-edit.png -------------------------------------------------------------------------------- /docs/images/min-max-rules-enforcement.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/min-max-rules-enforcement.png -------------------------------------------------------------------------------- /docs/images/min-max-rules-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/min-max-rules-list.png -------------------------------------------------------------------------------- /docs/images/regex-rules-edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/regex-rules-edit.png -------------------------------------------------------------------------------- /docs/images/regex-rules-enforcement.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/regex-rules-enforcement.png -------------------------------------------------------------------------------- /docs/images/regex-rules-jinja2-context-processing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/regex-rules-jinja2-context-processing.png -------------------------------------------------------------------------------- /docs/images/regex-rules-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/regex-rules-list.png -------------------------------------------------------------------------------- /docs/images/required-rules-edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/required-rules-edit.png -------------------------------------------------------------------------------- /docs/images/required-rules-enforcement.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/required-rules-enforcement.png -------------------------------------------------------------------------------- /docs/images/required-rules-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/required-rules-list.png -------------------------------------------------------------------------------- /docs/images/unique-rules-edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/unique-rules-edit.png -------------------------------------------------------------------------------- /docs/images/unique-rules-enforcement.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/unique-rules-enforcement.png -------------------------------------------------------------------------------- /docs/images/unique-rules-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/docs/images/unique-rules-list.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | hide: 3 | 4 | - navigation 5 | 6 | --- 7 | 8 | --8<-- "README.md" 9 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | mkdocs==1.6.0 2 | mkdocs-material==9.5.32 3 | markdown-version-annotations==1.0.1 4 | griffe==1.1.1 5 | mkdocstrings-python==1.10.8 6 | mkdocstrings==0.25.2 7 | mkdocs-autorefs==1.2.0 8 | -------------------------------------------------------------------------------- /docs/user/app_getting_started.md: -------------------------------------------------------------------------------- 1 | # Getting Started with the App 2 | 3 | This document provides a step-by-step tutorial on how to get the App going and how to use it. 4 | 5 | ## Install the App 6 | 7 | To install the App, please follow the instructions detailed in the [Installation Guide](../admin/install.md). 8 | 9 | ## First steps with the App 10 | 11 | Once the App is installed, under the "Extensibility" tab, you will find the supported Data Validation Engine rules in the "Data Validation" section. For example, "Min/Max Rules" or "Regex Rules". 12 | 13 | There you can list the existing validation rules of each type, or create them (one by one, or dumping them). 14 | 15 | !!! note 16 | The validation rules only take effect for new data entries, not for previous existing data. So, when you create a new object, for instance, an `ipam.VLAN`, that is when the existing validation rules will be enforced. Validation rules will be enforced when explicitly editing existing data. 17 | 18 | ## What are the next steps? 19 | 20 | You can check out the [Use Cases](app_use_cases.md) section for more examples. 21 | -------------------------------------------------------------------------------- /docs/user/app_overview.md: -------------------------------------------------------------------------------- 1 | # App Overview 2 | 3 | This document provides an overview of the App including critical information and important considerations when applying it to your Nautobot environment. 4 | 5 | !!! note 6 | Throughout this documentation, the terms "app" and "plugin" will be used interchangeably. 7 | 8 | ## Description 9 | 10 | The data validation engine app offers a set of user definable rules which are used to enforce business constraints on the data in Nautobot. These rules are tied to particular models and each rule is meant to enforce one aspect of a business use case. 11 | 12 | Supported rule types include: 13 | 14 | - Regular expression 15 | - Min/max value 16 | - Required fields 17 | - Unique values 18 | 19 | ## Audience (User Personas) - Who should use this App? 20 | 21 | Network Engineers interested in Network Automation, Infrastructure as Code, etc., that need to add some custom validation to their data input process. 22 | 23 | ## Authors and Maintainers 24 | 25 | - John Anderson (@lampwins) 26 | -------------------------------------------------------------------------------- /docs/user/app_use_cases.md: -------------------------------------------------------------------------------- 1 | # Using the App 2 | 3 | This document describes common use-cases and scenarios for this App. 4 | 5 | ## General Usage 6 | 7 | The data validation engine app offers a set of user definable rules which are used to enforce business constraints on the data in Nautobot. These rules are tied to particular models and each rule is meant to enforce one aspect of a business use case. 8 | 9 | Supported rule types include: 10 | 11 | - Regular expression 12 | - Min/max value 13 | - Required fields 14 | - Unique values 15 | 16 | ![Dropdown](../images/dropdown.png) 17 | 18 | ## Use-cases and common workflows 19 | 20 | ### Min/Max Rules 21 | 22 | ![Min/Max Rules List](../images/min-max-rules-list.png) 23 | 24 | Each rule is defined with these fields: 25 | 26 | * name - A unique name for the rule. 27 | * enabled - A boolean to toggle enforcement of the rule on and off. 28 | * content type - The Nautobot model to which the rule should apply (e.g. device, location, etc.). 29 | * field - The name of the numeric based field on the model to which the min/max value is validated. 30 | * min - The min value to validate value against (greater than or equal). 31 | * max - The max value to validate value against (less than or equal). 32 | * error message - An optional error message to display to the user when validation fails. By default, a message indicating validation against the defined min/max value has failed is shown. 33 | 34 | ![Min/Max Rules Edit](../images/min-max-rules-edit.png) 35 | 36 | In this example, a max value for VLAN IDs has been configured, preventing VLANs greater than 3999 from being created. 37 | 38 | ![Min/Max Rules Enforcement](../images/min-max-rules-enforcement.png) 39 | 40 | ### Regular Expression Rules 41 | 42 | ![Regex Rules List](../images/regex-rules-list.png) 43 | 44 | Each rule is defined with these fields: 45 | 46 | * name - A unique name for the rule. 47 | * enabled - A boolean to toggle enforcement of the rule on and off. 48 | * content type - The Nautobot model to which the rule should apply (e.g. device, location, etc.). 49 | * field - The name of the character based field on the model to which the regular expression is validated. 50 | * regular expression - The body of the regular expression used for validation. 51 | * context processing - A boolean to toggle Jinja2 context processing of the regular expression prior to evaluation 52 | * error message - An optional error message to display to the use when validation fails. By default, a message indicating validation against the defined regular expression has failed is shown. 53 | 54 | ![Regex Rules Edit](../images/regex-rules-edit.png) 55 | 56 | In this example, a device hostname validation rule has been created and prevents device records from being created or updated that do not conform to the naming standard. 57 | 58 | ![Regex Rules Enforcement](../images/regex-rules-enforcement.png) 59 | 60 | Regex rules may also support complex Jinja2 rendering called context processing which allows for the regular expression itself to by dynamically generated based on the context of the data it is validating. 61 | 62 | In this example the name of a device must start with the first three characters of the name of the location to which the device belongs. The dynamic nature of the Jinja2 rendering means that the location name can be anything, the enforcement action is simply that the given device name matches its assigned location. 63 | 64 | ![Regex Rules Jinja2 Context Processing](../images/regex-rules-jinja2-context-processing.png) 65 | 66 | !!! warning 67 | If there is an exception while rendering the Jinja2 template or the resulting regular expression string is invalid, data validation against the rule will fail and users will be instructed to either fix the rule or disable it before the data may be saved. 68 | 69 | ### Required Rules 70 | 71 | ![Required List](../images/required-rules-list.png) 72 | 73 | Each rule is defined with these fields: 74 | 75 | * name - A unique name for the rule. 76 | * enabled - A boolean to toggle enforcement of the rule on and off. 77 | * content type - The Nautobot model to which the rule should apply (e.g. device, location, etc.). 78 | * field - The name of the field on the Nautobot model which should always be required. 79 | * error message - An optional error message to display to the user when validation fails. By default, a message indicating the field may not be left blank is shown. 80 | 81 | ![Required Rules Edit](../images/required-rules-edit.png) 82 | 83 | In this example, a rule is enforcing that location objects must always have a description populated. 84 | 85 | ![Required Rules Enforcement](../images/required-rules-enforcement.png) 86 | 87 | ### Unique Rules 88 | 89 | ![Unique List](../images/unique-rules-list.png) 90 | 91 | Each rule is defined with these fields: 92 | 93 | * name - A unique name for the rule. 94 | * enabled - A boolean to toggle enforcement of the rule on and off. 95 | * content type - The Nautobot model to which the rule should apply (e.g. device, location, etc.). 96 | * field - The name of the field on the Nautobot model which should always be required. 97 | * max instances - The total number of records that may have the same unique value for the given field. Default of 1. 98 | * error message - An optional error message to display to the user when validation fails. By default, a message indicating the value already exists on another record or set of records, as determined by max instances. 99 | 100 | ![Unique Rules Edit](../images/unique-rules-edit.png) 101 | 102 | In this example, the rule enforces that the assigned ASN for a location is unique across all other locations. 103 | 104 | ![Unique Rules Enforcement](../images/unique-rules-enforcement.png) 105 | -------------------------------------------------------------------------------- /docs/user/faq.md: -------------------------------------------------------------------------------- 1 | # Frequently Asked Questions 2 | -------------------------------------------------------------------------------- /invoke.example.yml: -------------------------------------------------------------------------------- 1 | --- 2 | nautobot_data_validation_engine: 3 | nautobot_ver: "2.1.9" 4 | python_ver: "3.11" 5 | # local: false 6 | # compose_dir: "/full/path/to/nautobot-app-data-validation-engine/development" 7 | 8 | # The following is an example of using MySQL as the database backend 9 | # --- 10 | # nautobot_data_validation_engine: 11 | # compose_files: 12 | # - "docker-compose.base.yml" 13 | # - "docker-compose.redis.yml" 14 | # - "docker-compose.mysql.yml" 15 | # - "docker-compose.dev.yml" 16 | -------------------------------------------------------------------------------- /invoke.mysql.yml: -------------------------------------------------------------------------------- 1 | --- 2 | nautobot_data_validation_engine: 3 | project_name: "nautobot-data-validation-engine" 4 | nautobot_ver: "2.1.9" 5 | local: false 6 | python_ver: "3.11" 7 | compose_dir: "development" 8 | compose_files: 9 | - "docker-compose.base.yml" 10 | - "docker-compose.redis.yml" 11 | - "docker-compose.mysql.yml" 12 | - "docker-compose.dev.yml" 13 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dev_addr: "127.0.0.1:8001" 3 | edit_uri: "edit/main/docs" 4 | site_dir: "nautobot_data_validation_engine/static/nautobot_data_validation_engine/docs" 5 | site_name: "Data Validation Engine Documentation" 6 | site_url: "https://docs.nautobot.com/projects/data-validation/en/latest/" 7 | repo_url: "https://github.com/nautobot/nautobot-app-data-validation-engine" 8 | copyright: "Copyright © The Authors" 9 | theme: 10 | name: "material" 11 | navigation_depth: 4 12 | custom_dir: "docs/assets/overrides" 13 | hljs_languages: 14 | - "django" 15 | - "yaml" 16 | features: 17 | - "content.action.edit" 18 | - "content.action.view" 19 | - "content.code.copy" 20 | - "navigation.footer" 21 | - "navigation.indexes" 22 | - "navigation.tabs" 23 | - "navigation.tabs.sticky" 24 | - "navigation.tracking" 25 | - "search.highlight" 26 | - "search.share" 27 | - "search.suggest" 28 | favicon: "assets/favicon.ico" 29 | logo: "assets/nautobot_logo.svg" 30 | palette: 31 | # Palette toggle for light mode 32 | - media: "(prefers-color-scheme: light)" 33 | scheme: "default" 34 | primary: "black" 35 | toggle: 36 | icon: "material/weather-sunny" 37 | name: "Switch to dark mode" 38 | 39 | # Palette toggle for dark mode 40 | - media: "(prefers-color-scheme: dark)" 41 | scheme: "slate" 42 | primary: "black" 43 | toggle: 44 | icon: "material/weather-night" 45 | name: "Switch to light mode" 46 | extra_css: 47 | - "assets/extra.css" 48 | 49 | extra: 50 | generator: false 51 | ntc_sponsor: true 52 | social: 53 | - icon: "fontawesome/solid/rss" 54 | link: "https://blog.networktocode.com/blog/tags/nautobot" 55 | name: "Network to Code Blog" 56 | - icon: "fontawesome/brands/youtube" 57 | link: "https://www.youtube.com/playlist?list=PLjA0bhxgryJ2Ts4GJMDA-tPzVWEncv4pb" 58 | name: "Nautobot Videos" 59 | - icon: "fontawesome/brands/slack" 60 | link: "https://www.networktocode.com/community/" 61 | name: "Network to Code Community" 62 | - icon: "fontawesome/brands/github" 63 | link: "https://github.com/nautobot/nautobot" 64 | name: "GitHub Repo" 65 | - icon: "fontawesome/brands/twitter" 66 | link: "https://twitter.com/networktocode" 67 | name: "Network to Code Twitter" 68 | markdown_extensions: 69 | - "markdown_version_annotations": 70 | admonition_tag: "???" 71 | - "admonition" 72 | - "toc": 73 | permalink: true 74 | - "attr_list" 75 | - "md_in_html" 76 | - "pymdownx.highlight": 77 | anchor_linenums: true 78 | - "pymdownx.inlinehilite" 79 | - "pymdownx.snippets" 80 | - "pymdownx.superfences": 81 | custom_fences: 82 | - name: "mermaid" 83 | class: "mermaid" 84 | format: !!python/name:pymdownx.superfences.fence_code_format 85 | - "footnotes" 86 | plugins: 87 | - "search" 88 | - "mkdocstrings": 89 | default_handler: "python" 90 | handlers: 91 | python: 92 | paths: ["."] 93 | options: 94 | show_root_heading: true 95 | watch: 96 | - "README.md" 97 | 98 | validation: 99 | omitted_files: "warn" 100 | absolute_links: "warn" 101 | unrecognized_links: "warn" 102 | anchors: "warn" 103 | 104 | nav: 105 | - Overview: "index.md" 106 | - User Guide: 107 | - App Overview: "user/app_overview.md" 108 | - Getting Started: "user/app_getting_started.md" 109 | - Using the App: "user/app_use_cases.md" 110 | - Data Compliance: "user/app_data_compliance.md" 111 | - Frequently Asked Questions: "user/faq.md" 112 | - Administrator Guide: 113 | - Install and Configure: "admin/install.md" 114 | - Upgrade: "admin/upgrade.md" 115 | - Uninstall: "admin/uninstall.md" 116 | - Compatibility Matrix: "admin/compatibility_matrix.md" 117 | - Release Notes: 118 | - "admin/release_notes/index.md" 119 | - v3.2: "admin/release_notes/version_3.2.md" 120 | - v3.1: "admin/release_notes/version_3.1.md" 121 | - v3.0: "admin/release_notes/version_3.0.md" 122 | - v2.2: "admin/release_notes/version_2.2.md" 123 | - v2.1: "admin/release_notes/version_2.1.md" 124 | - v2.0: "admin/release_notes/version_2.0.md" 125 | - v1.0: "admin/release_notes/version_1.0.md" 126 | - Developer Guide: 127 | - Extending the App: "dev/extending.md" 128 | - Contributing to the App: "dev/contributing.md" 129 | - Development Environment: "dev/dev_environment.md" 130 | - Release Checklist: "dev/release_checklist.md" 131 | - Code Reference: 132 | - "dev/code_reference/index.md" 133 | - Package: "dev/code_reference/package.md" 134 | - API: "dev/code_reference/api.md" 135 | - Nautobot Docs Home ↗︎: "https://docs.nautobot.com" 136 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/__init__.py: -------------------------------------------------------------------------------- 1 | """App declaration for nautobot_data_validation_engine.""" 2 | 3 | # Metadata is inherited from Nautobot. If not including Nautobot in the environment, this should be added 4 | from importlib import metadata 5 | 6 | from nautobot.apps import NautobotAppConfig 7 | 8 | __version__ = metadata.version(__name__) 9 | 10 | 11 | class NautobotDataValidationEngineConfig(NautobotAppConfig): 12 | """App configuration for the nautobot_data_validation_engine app.""" 13 | 14 | name = "nautobot_data_validation_engine" 15 | verbose_name = "Data Validation Engine" 16 | version = __version__ 17 | author = "Network to Code, LLC" 18 | description = "Provides UI to build custom data validation rules for data in Nautobot." 19 | base_url = "nautobot-data-validation-engine" 20 | required_settings = [] 21 | min_version = "2.1.9" 22 | max_version = "2.9999" 23 | default_settings = {} 24 | caching_config = {} 25 | docs_view_name = "plugins:nautobot_data_validation_engine:docs" 26 | 27 | 28 | config = NautobotDataValidationEngineConfig # pylint:disable=invalid-name 29 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/api/__init__.py: -------------------------------------------------------------------------------- 1 | """REST API module for nautobot_data_validation_engine app.""" 2 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/api/serializers.py: -------------------------------------------------------------------------------- 1 | """API serializers for nautobot_data_validation_engine.""" 2 | 3 | from django.contrib.contenttypes.models import ContentType 4 | from nautobot.apps.api import NautobotModelSerializer, TaggedModelSerializerMixin 5 | from nautobot.core.api import ContentTypeField 6 | from nautobot.extras.utils import FeatureQuery 7 | from rest_framework import serializers 8 | 9 | from nautobot_data_validation_engine.models import ( 10 | DataCompliance, 11 | MinMaxValidationRule, 12 | RegularExpressionValidationRule, 13 | RequiredValidationRule, 14 | UniqueValidationRule, 15 | ) 16 | 17 | 18 | class RegularExpressionValidationRuleSerializer(NautobotModelSerializer, TaggedModelSerializerMixin): 19 | """Serializer for `RegularExpressionValidationRule` objects.""" 20 | 21 | url = serializers.HyperlinkedIdentityField( 22 | view_name="plugins-api:nautobot_data_validation_engine-api:regularexpressionvalidationrule-detail" 23 | ) 24 | content_type = ContentTypeField( 25 | queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()), 26 | ) 27 | 28 | class Meta: 29 | """Serializer metadata for RegularExpressionValidationRule objects.""" 30 | 31 | model = RegularExpressionValidationRule 32 | fields = "__all__" 33 | 34 | 35 | class MinMaxValidationRuleSerializer(NautobotModelSerializer, TaggedModelSerializerMixin): 36 | """Serializer for `MinMaxValidationRule` objects.""" 37 | 38 | url = serializers.HyperlinkedIdentityField( 39 | view_name="plugins-api:nautobot_data_validation_engine-api:minmaxvalidationrule-detail" 40 | ) 41 | content_type = ContentTypeField( 42 | queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()), 43 | ) 44 | 45 | class Meta: 46 | """Serializer metadata for MinMaxValidationRule objects.""" 47 | 48 | model = MinMaxValidationRule 49 | fields = "__all__" 50 | 51 | 52 | class RequiredValidationRuleSerializer(NautobotModelSerializer, TaggedModelSerializerMixin): 53 | """Serializer for `RequiredValidationRule` objects.""" 54 | 55 | url = serializers.HyperlinkedIdentityField( 56 | view_name="plugins-api:nautobot_data_validation_engine-api:requiredvalidationrule-detail" 57 | ) 58 | content_type = ContentTypeField( 59 | queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()), 60 | ) 61 | 62 | class Meta: 63 | """Serializer metadata for RequiredValidationRule objects.""" 64 | 65 | model = RequiredValidationRule 66 | fields = "__all__" 67 | 68 | 69 | class UniqueValidationRuleSerializer(NautobotModelSerializer, TaggedModelSerializerMixin): 70 | """Serializer for `UniqueValidationRule` objects.""" 71 | 72 | url = serializers.HyperlinkedIdentityField( 73 | view_name="plugins-api:nautobot_data_validation_engine-api:uniquevalidationrule-detail" 74 | ) 75 | content_type = ContentTypeField( 76 | queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()), 77 | ) 78 | 79 | class Meta: 80 | """Serializer metadata for UniqueValidationRule objects.""" 81 | 82 | model = UniqueValidationRule 83 | fields = "__all__" 84 | 85 | 86 | class DataComplianceSerializer(NautobotModelSerializer): 87 | """Serializer for DataCompliance.""" 88 | 89 | class Meta: 90 | """Meta class for serializer.""" 91 | 92 | model = DataCompliance 93 | fields = "__all__" 94 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/api/urls.py: -------------------------------------------------------------------------------- 1 | """Django API urlpatterns declaration for nautobot_data_validation_engine app.""" 2 | 3 | from nautobot.apps.api import OrderedDefaultRouter 4 | 5 | from nautobot_data_validation_engine.api import views 6 | 7 | router = OrderedDefaultRouter(view_name="Data Validation Engine") 8 | # add the name of your api endpoint, usually hyphenated model name in plural, e.g. "my-model-classes" 9 | # Regular expression rules 10 | router.register("regex-rules", views.RegularExpressionValidationRuleViewSet) 11 | 12 | # Min/max rules 13 | router.register("min-max-rules", views.MinMaxValidationRuleViewSet) 14 | 15 | # Required rules 16 | router.register("required-rules", views.RequiredValidationRuleViewSet) 17 | 18 | # Unique rules 19 | router.register("unique-rules", views.UniqueValidationRuleViewSet) 20 | 21 | # Data Compliance 22 | router.register("data-compliance", views.DataComplianceAPIView) 23 | 24 | 25 | app_name = "nautobot_data_validation_engine-api" 26 | urlpatterns = router.urls 27 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/api/views.py: -------------------------------------------------------------------------------- 1 | """API views for nautobot_data_validation_engine.""" 2 | 3 | from nautobot.apps.api import NautobotModelViewSet 4 | 5 | from nautobot_data_validation_engine import filters, models 6 | from nautobot_data_validation_engine.api import serializers 7 | 8 | 9 | class RegularExpressionValidationRuleViewSet(NautobotModelViewSet): 10 | """View to manage regular expression validation rules.""" 11 | 12 | queryset = models.RegularExpressionValidationRule.objects.all() 13 | serializer_class = serializers.RegularExpressionValidationRuleSerializer 14 | filterset_class = filters.RegularExpressionValidationRuleFilterSet 15 | 16 | 17 | class MinMaxValidationRuleViewSet(NautobotModelViewSet): 18 | """View to manage min max expression validation rules.""" 19 | 20 | queryset = models.MinMaxValidationRule.objects.all() 21 | serializer_class = serializers.MinMaxValidationRuleSerializer 22 | filterset_class = filters.MinMaxValidationRuleFilterSet 23 | 24 | 25 | class RequiredValidationRuleViewSet(NautobotModelViewSet): 26 | """View to manage min max expression validation rules.""" 27 | 28 | queryset = models.RequiredValidationRule.objects.all() 29 | serializer_class = serializers.RequiredValidationRuleSerializer 30 | filterset_class = filters.RequiredValidationRuleFilterSet 31 | 32 | 33 | class UniqueValidationRuleViewSet(NautobotModelViewSet): 34 | """View to manage min max expression validation rules.""" 35 | 36 | queryset = models.UniqueValidationRule.objects.all() 37 | serializer_class = serializers.UniqueValidationRuleSerializer 38 | filterset_class = filters.UniqueValidationRuleFilterSet 39 | 40 | 41 | class DataComplianceAPIView(NautobotModelViewSet): 42 | """API Views for DataCompliance.""" 43 | 44 | queryset = models.DataCompliance.objects.all() 45 | serializer_class = serializers.DataComplianceSerializer 46 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/app-config-schema.json: -------------------------------------------------------------------------------- 1 | true 2 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/datasources.py: -------------------------------------------------------------------------------- 1 | """Datasource definitions.""" 2 | 3 | from nautobot.extras.choices import LogLevelChoices 4 | from nautobot.extras.registry import DatasourceContent 5 | 6 | from nautobot_data_validation_engine.custom_validators import get_classes_from_git_repo 7 | 8 | 9 | def refresh_git_data_compliance_rules(repository_record, job_result, delete=False): # pylint: disable=W0613 10 | """Callback for repo refresh.""" 11 | job_result.log("Successfully pulled git repo", level_choice=LogLevelChoices.LOG_INFO) 12 | for compliance_class in get_classes_from_git_repo(repository_record): 13 | job_result.log(f"Found class {str(compliance_class.__name__)}", level_choice=LogLevelChoices.LOG_INFO) 14 | 15 | 16 | datasource_contents = [ 17 | ( 18 | "extras.gitrepository", 19 | DatasourceContent( 20 | name="data compliance rules", 21 | content_identifier="nautobot_data_validation_engine.data_compliance_rules", 22 | icon="mdi-file-document-outline", 23 | callback=refresh_git_data_compliance_rules, 24 | ), 25 | ) 26 | ] 27 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/filters.py: -------------------------------------------------------------------------------- 1 | """Filtering for nautobot_data_validation_engine.""" 2 | 3 | import django_filters as filters 4 | from django.db import models 5 | from nautobot.apps.filters import NautobotFilterSet 6 | from nautobot.core.filters import ContentTypeMultipleChoiceFilter, SearchFilter 7 | from nautobot.extras.utils import FeatureQuery 8 | 9 | from nautobot_data_validation_engine.models import ( 10 | DataCompliance, 11 | MinMaxValidationRule, 12 | RegularExpressionValidationRule, 13 | RequiredValidationRule, 14 | UniqueValidationRule, 15 | ) 16 | 17 | 18 | class RegularExpressionValidationRuleFilterSet(NautobotFilterSet): 19 | """Base filterset for the RegularExpressionValidationRule model.""" 20 | 21 | q = SearchFilter( 22 | filter_predicates={ 23 | "name": "icontains", 24 | "error_message": "icontains", 25 | "content_type__app_label": "iexact", 26 | "content_type__model": "iexact", 27 | "field": "iexact", 28 | "regular_expression": "icontains", 29 | } 30 | ) 31 | content_type = ContentTypeMultipleChoiceFilter( 32 | choices=FeatureQuery("custom_validators").get_choices, 33 | conjoined=False, # Make this an OR with multi-values 34 | ) 35 | 36 | class Meta: 37 | """Filterset metadata for the RegularExpressionValidationRule model.""" 38 | 39 | model = RegularExpressionValidationRule 40 | fields = "__all__" 41 | 42 | 43 | class MinMaxValidationRuleFilterSet(NautobotFilterSet): 44 | """Base filterset for the MinMaxValidationRule model.""" 45 | 46 | q = SearchFilter( 47 | filter_predicates={ 48 | "name": "icontains", 49 | "error_message": "icontains", 50 | "content_type__app_label": "iexact", 51 | "content_type__model": "iexact", 52 | "field": "iexact", 53 | } 54 | ) 55 | content_type = ContentTypeMultipleChoiceFilter( 56 | choices=FeatureQuery("custom_validators").get_choices, 57 | conjoined=False, # Make this an OR with multi-values 58 | ) 59 | 60 | class Meta: 61 | """Filterset metadata for the MinMaxValidationRuleFilterSet model.""" 62 | 63 | model = MinMaxValidationRule 64 | fields = "__all__" 65 | 66 | 67 | class RequiredValidationRuleFilterSet(NautobotFilterSet): 68 | """Base filterset for the RequiredValidationRule model.""" 69 | 70 | q = SearchFilter( 71 | filter_predicates={ 72 | "name": "icontains", 73 | "error_message": "icontains", 74 | "content_type__app_label": "iexact", 75 | "content_type__model": "iexact", 76 | "field": "iexact", 77 | } 78 | ) 79 | content_type = ContentTypeMultipleChoiceFilter( 80 | choices=FeatureQuery("custom_validators").get_choices, 81 | conjoined=False, # Make this an OR with multi-values 82 | ) 83 | 84 | class Meta: 85 | """Filterset metadata for the RequiredValidationRuleFilterSet model.""" 86 | 87 | model = RequiredValidationRule 88 | fields = "__all__" 89 | 90 | 91 | class UniqueValidationRuleFilterSet(NautobotFilterSet): 92 | """Base filterset for the UniqueValidationRule model.""" 93 | 94 | q = SearchFilter( 95 | filter_predicates={ 96 | "name": "icontains", 97 | "error_message": "icontains", 98 | "content_type__app_label": "icontains", 99 | "content_type__model": "icontains", 100 | "field": "iexact", 101 | } 102 | ) 103 | content_type = ContentTypeMultipleChoiceFilter( 104 | choices=FeatureQuery("custom_validators").get_choices, 105 | conjoined=False, # Make this an OR with multi-values 106 | ) 107 | 108 | class Meta: 109 | """Filterset metadata for the UniqueValidationRuleFilterSet model.""" 110 | 111 | model = UniqueValidationRule 112 | fields = "__all__" 113 | 114 | 115 | # 116 | # DataCompliance 117 | # 118 | 119 | 120 | class CustomContentTypeFilter(filters.MultipleChoiceFilter): 121 | """Filter for ContentType that doesn't rely on the model's plural name to be in the registry.""" 122 | 123 | def filter(self, qs, value): 124 | """Filter on value, which should be list of content-type names. 125 | 126 | e.g. `['dcim.device', 'dcim.rack']` 127 | """ 128 | q = models.Q() 129 | for v in value: 130 | try: 131 | app_label, model = v.lower().split(".") 132 | except ValueError: 133 | continue 134 | q |= models.Q( 135 | **{ 136 | f"{self.field_name}__app_label": app_label, 137 | f"{self.field_name}__model": model, 138 | } 139 | ) 140 | qs = qs.filter(q) 141 | return qs 142 | 143 | 144 | class DataComplianceFilterSet(NautobotFilterSet): 145 | """Base filterset for DataComplianceRule model.""" 146 | 147 | q = SearchFilter( 148 | filter_predicates={ 149 | "compliance_class_name": "icontains", 150 | "message": "icontains", 151 | "content_type__app_label": "icontains", 152 | "content_type__model": "icontains", 153 | "object_id": "icontains", 154 | } 155 | ) 156 | content_type = CustomContentTypeFilter( 157 | choices=FeatureQuery("custom_validators").get_choices, 158 | ) 159 | 160 | class Meta: 161 | """Meta class for DataComplianceFilterSet.""" 162 | 163 | model = DataCompliance 164 | fields = "__all__" 165 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/forms.py: -------------------------------------------------------------------------------- 1 | """Forms for nautobot_data_validation_engine.""" 2 | 3 | from django import forms 4 | from django.contrib.contenttypes.models import ContentType 5 | 6 | try: 7 | from nautobot.apps.constants import CHARFIELD_MAX_LENGTH 8 | except ImportError: 9 | CHARFIELD_MAX_LENGTH = 255 10 | from nautobot.core.forms import ( 11 | BootstrapMixin, 12 | BulkEditNullBooleanSelect, 13 | CSVMultipleContentTypeField, 14 | DynamicModelChoiceField, 15 | DynamicModelMultipleChoiceField, 16 | MultipleContentTypeField, 17 | MultiValueCharField, 18 | StaticSelect2, 19 | TagFilterField, 20 | ) 21 | from nautobot.core.forms.constants import BOOLEAN_WITH_BLANK_CHOICES 22 | from nautobot.extras.forms import ( 23 | NautobotBulkEditForm, 24 | NautobotFilterForm, 25 | NautobotModelForm, 26 | TagsBulkEditFormMixin, 27 | ) 28 | from nautobot.extras.utils import FeatureQuery 29 | 30 | from nautobot_data_validation_engine.models import ( 31 | DataCompliance, 32 | MinMaxValidationRule, 33 | RegularExpressionValidationRule, 34 | RequiredValidationRule, 35 | UniqueValidationRule, 36 | ) 37 | 38 | # 39 | # RegularExpressionValidationRules 40 | # 41 | 42 | 43 | class RegularExpressionValidationRuleForm(NautobotModelForm): 44 | """Base model form for the RegularExpressionValidationRule model.""" 45 | 46 | content_type = DynamicModelChoiceField( 47 | queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()).order_by( 48 | "app_label", "model" 49 | ), 50 | ) 51 | 52 | class Meta: 53 | """Form metadata for the RegularExpressionValidationRule model.""" 54 | 55 | model = RegularExpressionValidationRule 56 | fields = "__all__" 57 | 58 | 59 | class RegularExpressionValidationRuleBulkEditForm(TagsBulkEditFormMixin, NautobotBulkEditForm): 60 | """Base bulk edit form for the RegularExpressionValidationRule model.""" 61 | 62 | pk = DynamicModelMultipleChoiceField( 63 | queryset=RegularExpressionValidationRule.objects.all(), widget=forms.MultipleHiddenInput 64 | ) 65 | enabled = forms.NullBooleanField( 66 | required=False, 67 | widget=BulkEditNullBooleanSelect(), 68 | ) 69 | regular_expression = forms.CharField(required=False) 70 | error_message = forms.CharField(required=False) 71 | context_processing = forms.NullBooleanField( 72 | required=False, 73 | widget=BulkEditNullBooleanSelect(), 74 | ) 75 | 76 | class Meta: 77 | """Bulk edit form metadata for the RegularExpressionValidationRule model.""" 78 | 79 | fields = ["tags"] 80 | nullable_fields = ["error_message"] 81 | 82 | 83 | class RegularExpressionValidationRuleFilterForm(NautobotFilterForm): 84 | """Base filter form for the RegularExpressionValidationRule model.""" 85 | 86 | model = RegularExpressionValidationRule 87 | field_order = [ 88 | "q", 89 | "name", 90 | "enabled", 91 | "content_type", 92 | "field", 93 | "regular_expression", 94 | "context_processing", 95 | "error_message", 96 | ] 97 | q = forms.CharField(required=False, label="Search") 98 | # "CSV" field is being used here because it is using the slug-form input for 99 | # content-types, which improves UX. 100 | content_type = CSVMultipleContentTypeField( 101 | queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()).order_by( 102 | "app_label", "model" 103 | ), 104 | required=False, 105 | ) 106 | tags = TagFilterField(model) 107 | 108 | 109 | # 110 | # MinMaxValidationRules 111 | # 112 | 113 | 114 | class MinMaxValidationRuleForm(NautobotModelForm): 115 | """Base model form for the MinMaxValidationRule model.""" 116 | 117 | content_type = DynamicModelChoiceField( 118 | queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()).order_by( 119 | "app_label", "model" 120 | ), 121 | ) 122 | 123 | class Meta: 124 | """Form metadata for the MinMaxValidationRule model.""" 125 | 126 | model = MinMaxValidationRule 127 | fields = "__all__" 128 | 129 | 130 | class MinMaxValidationRuleBulkEditForm(TagsBulkEditFormMixin, NautobotBulkEditForm): 131 | """Base bulk edit form for the MinMaxValidationRule model.""" 132 | 133 | pk = DynamicModelMultipleChoiceField(queryset=MinMaxValidationRule.objects.all(), widget=forms.MultipleHiddenInput) 134 | enabled = forms.NullBooleanField( 135 | required=False, 136 | widget=BulkEditNullBooleanSelect(), 137 | ) 138 | min = forms.IntegerField(required=False) 139 | max = forms.IntegerField(required=False) 140 | error_message = forms.CharField(required=False) 141 | 142 | class Meta: 143 | """Bulk edit form metadata for the MinMaxValidationRule model.""" 144 | 145 | fields = ["tags"] 146 | nullable_fields = ["error_message"] 147 | 148 | 149 | class MinMaxValidationRuleFilterForm(NautobotFilterForm): 150 | """Base filter form for the MinMaxValidationRule model.""" 151 | 152 | model = MinMaxValidationRule 153 | field_order = ["q", "name", "enabled", "content_type", "field", "min", "max", "error_message"] 154 | q = forms.CharField(required=False, label="Search") 155 | # "CSV" field is being used here because it is using the slug-form input for 156 | # content-types, which improves UX. 157 | content_type = CSVMultipleContentTypeField( 158 | queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()).order_by( 159 | "app_label", "model" 160 | ), 161 | required=False, 162 | ) 163 | min = forms.IntegerField(required=False) 164 | max = forms.IntegerField(required=False) 165 | tags = TagFilterField(model) 166 | 167 | 168 | # 169 | # RequiredValidationRules 170 | # 171 | 172 | 173 | class RequiredValidationRuleForm(NautobotModelForm): 174 | """Base model form for the RequiredValidationRule model.""" 175 | 176 | content_type = DynamicModelChoiceField( 177 | queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()).order_by( 178 | "app_label", "model" 179 | ), 180 | ) 181 | 182 | class Meta: 183 | """Form metadata for the RequiredValidationRule model.""" 184 | 185 | model = RequiredValidationRule 186 | fields = "__all__" 187 | 188 | 189 | class RequiredValidationRuleBulkEditForm(TagsBulkEditFormMixin, NautobotBulkEditForm): 190 | """Base bulk edit form for the RequiredValidationRule model.""" 191 | 192 | pk = DynamicModelMultipleChoiceField( 193 | queryset=RequiredValidationRule.objects.all(), widget=forms.MultipleHiddenInput 194 | ) 195 | enabled = forms.NullBooleanField( 196 | required=False, 197 | widget=BulkEditNullBooleanSelect(), 198 | ) 199 | error_message = forms.CharField(required=False) 200 | 201 | class Meta: 202 | """Bulk edit form metadata for the RequiredValidationRule model.""" 203 | 204 | fields = ["tags"] 205 | nullable_fields = ["error_message"] 206 | 207 | 208 | class RequiredValidationRuleFilterForm(NautobotFilterForm): 209 | """Base filter form for the RequiredValidationRule model.""" 210 | 211 | model = RequiredValidationRule 212 | field_order = [ 213 | "q", 214 | "name", 215 | "enabled", 216 | "content_type", 217 | "field", 218 | "error_message", 219 | ] 220 | q = forms.CharField(required=False, label="Search") 221 | # "CSV" field is being used here because it is using the slug-form input for 222 | # content-types, which improves UX. 223 | content_type = CSVMultipleContentTypeField( 224 | queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()).order_by( 225 | "app_label", "model" 226 | ), 227 | required=False, 228 | ) 229 | tags = TagFilterField(model) 230 | 231 | 232 | # 233 | # UniqueValidationRules 234 | # 235 | 236 | 237 | class UniqueValidationRuleForm(NautobotModelForm): 238 | """Base model form for the UniqueValidationRule model.""" 239 | 240 | content_type = DynamicModelChoiceField( 241 | queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()).order_by( 242 | "app_label", "model" 243 | ), 244 | ) 245 | 246 | class Meta: 247 | """Form metadata for the UniqueValidationRule model.""" 248 | 249 | model = UniqueValidationRule 250 | fields = "__all__" 251 | 252 | 253 | class UniqueValidationRuleBulkEditForm(TagsBulkEditFormMixin, NautobotBulkEditForm): 254 | """Base bulk edit form for the UniqueValidationRule model.""" 255 | 256 | pk = DynamicModelMultipleChoiceField(queryset=UniqueValidationRule.objects.all(), widget=forms.MultipleHiddenInput) 257 | enabled = forms.NullBooleanField( 258 | required=False, 259 | widget=BulkEditNullBooleanSelect(), 260 | ) 261 | max_instances = forms.IntegerField(required=False) 262 | error_message = forms.CharField(required=False) 263 | 264 | class Meta: 265 | """Bulk edit form metadata for the UniqueValidationRule model.""" 266 | 267 | fields = ["tags"] 268 | nullable_fields = ["error_message"] 269 | 270 | 271 | class UniqueValidationRuleFilterForm(NautobotFilterForm): 272 | """Base filter form for the UniqueValidationRule model.""" 273 | 274 | model = UniqueValidationRule 275 | field_order = [ 276 | "q", 277 | "name", 278 | "enabled", 279 | "content_type", 280 | "field", 281 | "max_instances", 282 | "error_message", 283 | ] 284 | q = forms.CharField(required=False, label="Search") 285 | # "CSV" field is being used here because it is using the slug-form input for 286 | # content-types, which improves UX. 287 | content_type = CSVMultipleContentTypeField( 288 | queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()).order_by( 289 | "app_label", "model" 290 | ), 291 | required=False, 292 | ) 293 | max_instances = forms.IntegerField(required=False) 294 | tags = TagFilterField(model) 295 | 296 | 297 | # 298 | # DataCompliance 299 | # 300 | 301 | 302 | class DataComplianceFilterForm(BootstrapMixin, forms.Form): 303 | """Form for DataCompliance instances.""" 304 | 305 | model = DataCompliance 306 | compliance_class_name = MultiValueCharField(max_length=CHARFIELD_MAX_LENGTH, required=False) 307 | validated_attribute = MultiValueCharField(max_length=CHARFIELD_MAX_LENGTH, required=False) 308 | valid = forms.NullBooleanField(required=False, widget=StaticSelect2(choices=BOOLEAN_WITH_BLANK_CHOICES)) 309 | content_type = MultipleContentTypeField( 310 | feature=None, 311 | queryset=ContentType.objects.all().order_by("app_label", "model"), 312 | choices_as_strings=True, 313 | required=False, 314 | ) 315 | q = forms.CharField(required=False, label="Search") 316 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/jobs.py: -------------------------------------------------------------------------------- 1 | """Jobs for nautobot_data_validation_engine.""" 2 | 3 | from django.apps import apps as global_apps 4 | from django.contrib.contenttypes.models import ContentType 5 | from django.db.models import Q 6 | from nautobot.core.celery import register_jobs 7 | from nautobot.extras.jobs import BooleanVar, Job, MultiChoiceVar, get_task_logger 8 | from nautobot.extras.models import GitRepository 9 | from nautobot.extras.plugins import CustomValidator, ValidationError 10 | from nautobot.extras.registry import registry 11 | 12 | from nautobot_data_validation_engine.custom_validators import get_classes_from_git_repo, get_data_compliance_rules_map 13 | from nautobot_data_validation_engine.models import DataCompliance 14 | 15 | logger = get_task_logger(__name__) 16 | 17 | 18 | def get_data_compliance_rules(): 19 | """Generate a list of Audit Ruleset classes that exist from the registry.""" 20 | validators = [] 21 | for rule_sets in get_data_compliance_rules_map().values(): 22 | validators.extend(rule_sets) 23 | return validators 24 | 25 | 26 | def get_choices(): 27 | """Get choices from registry.""" 28 | choices = [] 29 | for ruleset_class in get_data_compliance_rules(): 30 | choices.append((ruleset_class.__name__, ruleset_class.__name__)) 31 | for repo in GitRepository.objects.all(): 32 | if "nautobot_data_validation_engine.data_compliance_rules" in repo.provided_contents: 33 | for compliance_class in get_classes_from_git_repo(repo): 34 | choices.append((compliance_class.__name__, compliance_class.__name__)) 35 | 36 | choices.sort() 37 | return choices 38 | 39 | 40 | def clean_compliance_rules_results_for_instance(instance, excluded_pks): 41 | """Clean compliance results.""" 42 | excluded_pks = excluded_pks or [] 43 | DataCompliance.objects.filter( 44 | object_id=instance.id, 45 | content_type=ContentType.objects.get_for_model(instance), 46 | compliance_class_name__endswith="CustomValidator", 47 | ).exclude(pk__in=excluded_pks).delete() 48 | 49 | 50 | class RunRegisteredDataComplianceRules(Job): 51 | """Run the validate function on all registered DataComplianceRule classes and, optionally, the built-in data validation rules.""" 52 | 53 | name = "Run Registered Data Compliance Rules" 54 | description = "Runs selected Data Compliance rule classes." 55 | 56 | selected_data_compliance_rules = MultiChoiceVar( 57 | choices=get_choices, 58 | label="Select Data Compliance Rules", 59 | required=False, 60 | description="Not selecting any rules will run all rules listed.", 61 | ) 62 | 63 | run_builtin_rules_in_report = BooleanVar( 64 | label="Run built-in validation rules?", description="Include created, built-in data validation rules in report." 65 | ) 66 | 67 | def run(self, *args, **kwargs): 68 | """Run the validate function on all given DataComplianceRule classes.""" 69 | selected_data_compliance_rules = kwargs.get("selected_data_compliance_rules", None) 70 | 71 | compliance_classes = [] 72 | compliance_classes.extend(get_data_compliance_rules()) 73 | 74 | for repo in GitRepository.objects.all(): 75 | if "nautobot_data_validation_engine.data_compliance_rules" in repo.provided_contents: 76 | compliance_classes.extend(get_classes_from_git_repo(repo)) 77 | 78 | for compliance_class in compliance_classes: 79 | if selected_data_compliance_rules and compliance_class.__name__ not in selected_data_compliance_rules: 80 | continue 81 | logger.info(f"Running {compliance_class.__name__}") 82 | app_label, model = compliance_class.model.split(".") 83 | for obj in global_apps.get_model(app_label, model).objects.all(): 84 | ins = compliance_class(obj) 85 | ins.enforce = False 86 | ins.clean() 87 | 88 | run_builtin_rules_in_report = kwargs.get("run_builtin_rules_in_report", False) 89 | if run_builtin_rules_in_report: 90 | logger.info("Running built-in data validation rules") 91 | self.report_for_validation_rules() 92 | 93 | logger.info("View Data Compliance results [here](/plugins/nautobot-data-validation-engine/data-compliance/)") 94 | 95 | @staticmethod 96 | def report_for_validation_rules(): 97 | """Run built-in data validation rules and add to report.""" 98 | query = ( 99 | Q(uniquevalidationrule__isnull=False) 100 | | Q(regularexpressionvalidationrule__isnull=False) 101 | | Q(minmaxvalidationrule__isnull=False) 102 | | Q(requiredvalidationrule__isnull=False) 103 | ) 104 | 105 | model_classes = [ct.model_class() for ct in ContentType.objects.filter(query).distinct()] 106 | 107 | # Gather custom validators of built-in rules 108 | validator_dicts = [] 109 | for model_class in model_classes: 110 | model_custom_validators = registry["plugin_custom_validators"][model_class._meta.label_lower] 111 | # Get only DataValidationCustomValidators 112 | # otherwise, we would get all validators (more than those dynamically created) 113 | validator_dicts.extend( 114 | [ 115 | {cv: model_class} 116 | for cv in model_custom_validators 117 | if cv.__name__ 118 | == f"{model_class._meta.app_label.capitalize()}{model_class._meta.model_name.capitalize()}CustomValidator" 119 | ] 120 | ) 121 | 122 | # Run validation on existing objects and add to report 123 | for validator_dict in validator_dicts: 124 | for validator, class_name in validator_dict.items(): 125 | if getattr(validator, "clean") == getattr(CustomValidator, "clean"): 126 | continue 127 | 128 | for validated_object in class_name.objects.all(): 129 | try: 130 | validator(validated_object).clean(exclude_disabled_rules=False) 131 | clean_compliance_rules_results_for_instance(instance=validated_object, excluded_pks=[]) 132 | except ValidationError as error: 133 | result = validator.get_compliance_result( 134 | validator, 135 | instance=validated_object, 136 | message=error.messages[0], 137 | attribute=list(error.message_dict.keys())[0], 138 | valid=False, 139 | ) 140 | clean_compliance_rules_results_for_instance(instance=validated_object, excluded_pks=[result.pk]) 141 | 142 | 143 | class DeleteOrphanedDataComplianceData(Job): 144 | """Utility job to delete any Data Compliance objects where the validated object no longer exists.""" 145 | 146 | name = "Delete Orphaned Data Compliance Data" 147 | description = "Delete any Data Compliance objects where its validated object no longer exists." 148 | 149 | def run(self, *args, **kwargs): 150 | """Delete DataCompliance objects where its validated_object no longer exists.""" 151 | number_deleted = 0 152 | for obj in DataCompliance.objects.all(): 153 | if obj.validated_object is None: 154 | logger.info("Deleting %s.", obj) 155 | obj.delete() 156 | number_deleted += 1 157 | logger.info("Deleted %s orphaned DataCompliance objects.", number_deleted) 158 | 159 | 160 | jobs = ( 161 | RunRegisteredDataComplianceRules, 162 | DeleteOrphanedDataComplianceData, 163 | ) 164 | register_jobs(*jobs) 165 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.1.11 on 2021-05-26 06:04 2 | 3 | import uuid 4 | 5 | import django.db.models.deletion 6 | import nautobot.extras.utils 7 | from django.db import migrations, models 8 | 9 | import nautobot_data_validation_engine.models 10 | 11 | 12 | class Migration(migrations.Migration): 13 | initial = True 14 | 15 | dependencies = [ 16 | ("contenttypes", "0002_remove_content_type_name"), 17 | ] 18 | 19 | operations = [ 20 | migrations.CreateModel( 21 | name="RegularExpressionValidationRule", 22 | fields=[ 23 | ( 24 | "id", 25 | models.UUIDField( 26 | default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True 27 | ), 28 | ), 29 | ("created", models.DateField(auto_now_add=True, null=True)), 30 | ("last_updated", models.DateTimeField(auto_now=True, null=True)), 31 | ("name", models.CharField(max_length=100, unique=True)), 32 | ("slug", models.SlugField(max_length=100, unique=True)), 33 | ("enabled", models.BooleanField(default=True)), 34 | ("error_message", models.CharField(blank=True, max_length=255, null=True)), 35 | ("field", models.CharField(max_length=50)), 36 | ( 37 | "regular_expression", 38 | models.TextField(validators=[nautobot_data_validation_engine.models.validate_regex]), 39 | ), 40 | ( 41 | "content_type", 42 | models.ForeignKey( 43 | limit_choices_to=nautobot.extras.utils.FeatureQuery("custom_validators"), 44 | on_delete=django.db.models.deletion.CASCADE, 45 | to="contenttypes.contenttype", 46 | ), 47 | ), 48 | ], 49 | options={ 50 | "ordering": ("name",), 51 | "unique_together": {("content_type", "field")}, 52 | }, 53 | ), 54 | migrations.CreateModel( 55 | name="MinMaxValidationRule", 56 | fields=[ 57 | ( 58 | "id", 59 | models.UUIDField( 60 | default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True 61 | ), 62 | ), 63 | ("created", models.DateField(auto_now_add=True, null=True)), 64 | ("last_updated", models.DateTimeField(auto_now=True, null=True)), 65 | ("name", models.CharField(max_length=100, unique=True)), 66 | ("slug", models.SlugField(max_length=100, unique=True)), 67 | ("enabled", models.BooleanField(default=True)), 68 | ("error_message", models.CharField(blank=True, max_length=255, null=True)), 69 | ("field", models.CharField(max_length=50)), 70 | ("min", models.FloatField(blank=True, null=True)), 71 | ("max", models.FloatField(blank=True, null=True)), 72 | ( 73 | "content_type", 74 | models.ForeignKey( 75 | limit_choices_to=nautobot.extras.utils.FeatureQuery("custom_validators"), 76 | on_delete=django.db.models.deletion.CASCADE, 77 | to="contenttypes.contenttype", 78 | ), 79 | ), 80 | ], 81 | options={ 82 | "ordering": ("name",), 83 | "unique_together": {("content_type", "field")}, 84 | }, 85 | ), 86 | ] 87 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/migrations/0002_required_unique_types_regex_context.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.16 on 2022-11-05 04:42 2 | 3 | import uuid 4 | 5 | import django.core.serializers.json 6 | import django.core.validators 7 | import django.db.models.deletion 8 | import nautobot.extras.models.mixins 9 | import nautobot.extras.utils 10 | import taggit.managers 11 | from django.db import migrations, models 12 | 13 | 14 | class Migration(migrations.Migration): 15 | dependencies = [ 16 | ("contenttypes", "0002_remove_content_type_name"), 17 | ("extras", "0048_alter_objectchange_change_context_detail"), 18 | ("nautobot_data_validation_engine", "0001_initial"), 19 | ] 20 | 21 | operations = [ 22 | migrations.AddField( 23 | model_name="minmaxvalidationrule", 24 | name="_custom_field_data", 25 | field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), 26 | ), 27 | migrations.AddField( 28 | model_name="minmaxvalidationrule", 29 | name="tags", 30 | field=taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag"), 31 | ), 32 | migrations.AddField( 33 | model_name="regularexpressionvalidationrule", 34 | name="_custom_field_data", 35 | field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), 36 | ), 37 | migrations.AddField( 38 | model_name="regularexpressionvalidationrule", 39 | name="context_processing", 40 | field=models.BooleanField(default=False), 41 | ), 42 | migrations.AddField( 43 | model_name="regularexpressionvalidationrule", 44 | name="tags", 45 | field=taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag"), 46 | ), 47 | migrations.AlterField( 48 | model_name="regularexpressionvalidationrule", 49 | name="regular_expression", 50 | field=models.TextField(), 51 | ), 52 | migrations.CreateModel( 53 | name="UniqueValidationRule", 54 | fields=[ 55 | ( 56 | "id", 57 | models.UUIDField( 58 | default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True 59 | ), 60 | ), 61 | ("created", models.DateField(auto_now_add=True, null=True)), 62 | ("last_updated", models.DateTimeField(auto_now=True, null=True)), 63 | ( 64 | "_custom_field_data", 65 | models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), 66 | ), 67 | ("name", models.CharField(max_length=100, unique=True)), 68 | ("slug", models.SlugField(max_length=100, unique=True)), 69 | ("enabled", models.BooleanField(default=True)), 70 | ("error_message", models.CharField(blank=True, max_length=255, null=True)), 71 | ("field", models.CharField(max_length=50)), 72 | ( 73 | "max_instances", 74 | models.PositiveIntegerField(default=1, validators=[django.core.validators.MinValueValidator(1)]), 75 | ), 76 | ( 77 | "content_type", 78 | models.ForeignKey( 79 | limit_choices_to=nautobot.extras.utils.FeatureQuery("custom_validators"), 80 | on_delete=django.db.models.deletion.CASCADE, 81 | to="contenttypes.contenttype", 82 | ), 83 | ), 84 | ("tags", taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag")), 85 | ], 86 | options={ 87 | "ordering": ("name",), 88 | "unique_together": {("content_type", "field")}, 89 | }, 90 | bases=( 91 | models.Model, 92 | nautobot.extras.models.mixins.DynamicGroupMixin, 93 | nautobot.extras.models.mixins.NotesMixin, 94 | ), 95 | ), 96 | migrations.CreateModel( 97 | name="RequiredValidationRule", 98 | fields=[ 99 | ( 100 | "id", 101 | models.UUIDField( 102 | default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True 103 | ), 104 | ), 105 | ("created", models.DateField(auto_now_add=True, null=True)), 106 | ("last_updated", models.DateTimeField(auto_now=True, null=True)), 107 | ( 108 | "_custom_field_data", 109 | models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), 110 | ), 111 | ("name", models.CharField(max_length=100, unique=True)), 112 | ("slug", models.SlugField(max_length=100, unique=True)), 113 | ("enabled", models.BooleanField(default=True)), 114 | ("error_message", models.CharField(blank=True, max_length=255, null=True)), 115 | ("field", models.CharField(max_length=50)), 116 | ( 117 | "content_type", 118 | models.ForeignKey( 119 | limit_choices_to=nautobot.extras.utils.FeatureQuery("custom_validators"), 120 | on_delete=django.db.models.deletion.CASCADE, 121 | to="contenttypes.contenttype", 122 | ), 123 | ), 124 | ("tags", taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag")), 125 | ], 126 | options={ 127 | "ordering": ("name",), 128 | "unique_together": {("content_type", "field")}, 129 | }, 130 | bases=( 131 | models.Model, 132 | nautobot.extras.models.mixins.DynamicGroupMixin, 133 | nautobot.extras.models.mixins.NotesMixin, 134 | ), 135 | ), 136 | ] 137 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/migrations/0003_datacompliance.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.18 on 2023-05-09 20:08 2 | 3 | import uuid 4 | 5 | import django.core.serializers.json 6 | import django.db.models.deletion 7 | import nautobot.extras.models.mixins 8 | import taggit.managers 9 | from django.db import migrations, models 10 | 11 | 12 | class Migration(migrations.Migration): 13 | dependencies = [ 14 | ("extras", "0053_relationship_required_on"), 15 | ("contenttypes", "0002_remove_content_type_name"), 16 | ("nautobot_data_validation_engine", "0002_required_unique_types_regex_context"), 17 | ] 18 | 19 | operations = [ 20 | migrations.CreateModel( 21 | name="DataCompliance", 22 | fields=[ 23 | ( 24 | "id", 25 | models.UUIDField( 26 | default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True 27 | ), 28 | ), 29 | ("created", models.DateField(auto_now_add=True, null=True)), 30 | ("last_updated", models.DateTimeField(auto_now=True, null=True)), 31 | ( 32 | "_custom_field_data", 33 | models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), 34 | ), 35 | ("compliance_class_name", models.CharField(max_length=100)), 36 | ("last_validation_date", models.DateTimeField(auto_now=True)), 37 | ("object_id", models.CharField(max_length=200)), 38 | ("validated_object_str", models.CharField(blank=True, max_length=200, null=True)), 39 | ("validated_attribute", models.CharField(blank=True, max_length=100, null=True)), 40 | ("validated_attribute_value", models.CharField(blank=True, max_length=200, null=True)), 41 | ("valid", models.BooleanField()), 42 | ("message", models.TextField(blank=True, null=True)), 43 | ( 44 | "content_type", 45 | models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to="contenttypes.contenttype"), 46 | ), 47 | ("tags", taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag")), 48 | ], 49 | options={ 50 | "verbose_name_plural": "Data Compliance", 51 | "unique_together": {("compliance_class_name", "content_type", "object_id", "validated_attribute")}, 52 | }, 53 | bases=( 54 | models.Model, 55 | nautobot.extras.models.mixins.DynamicGroupMixin, 56 | nautobot.extras.models.mixins.NotesMixin, 57 | ), 58 | ), 59 | ] 60 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/migrations/0004_created_datetime.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations, models 2 | 3 | 4 | class Migration(migrations.Migration): 5 | dependencies = [ 6 | ("nautobot_data_validation_engine", "0003_datacompliance"), 7 | ] 8 | 9 | operations = [ 10 | migrations.AlterField( 11 | model_name="datacompliance", 12 | name="created", 13 | field=models.DateTimeField(auto_now_add=True, null=True), 14 | ), 15 | migrations.AlterField( 16 | model_name="regularexpressionvalidationrule", 17 | name="created", 18 | field=models.DateTimeField(auto_now_add=True, null=True), 19 | ), 20 | migrations.AlterField( 21 | model_name="minmaxvalidationrule", 22 | name="created", 23 | field=models.DateTimeField(auto_now_add=True, null=True), 24 | ), 25 | migrations.AlterField( 26 | model_name="requiredvalidationrule", 27 | name="created", 28 | field=models.DateTimeField(auto_now_add=True, null=True), 29 | ), 30 | migrations.AlterField( 31 | model_name="uniquevalidationrule", 32 | name="created", 33 | field=models.DateTimeField(auto_now_add=True, null=True), 34 | ), 35 | ] 36 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/migrations/0005_remove_slugs_alter_tags.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.20 on 2023-08-23 19:02 2 | 3 | import nautobot.core.models.fields 4 | from django.db import migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | dependencies = [ 9 | ("extras", "0098_rename_data_jobresult_result"), 10 | ("nautobot_data_validation_engine", "0004_created_datetime"), 11 | ] 12 | 13 | operations = [ 14 | migrations.RemoveField( 15 | model_name="minmaxvalidationrule", 16 | name="slug", 17 | ), 18 | migrations.RemoveField( 19 | model_name="regularexpressionvalidationrule", 20 | name="slug", 21 | ), 22 | migrations.RemoveField( 23 | model_name="requiredvalidationrule", 24 | name="slug", 25 | ), 26 | migrations.RemoveField( 27 | model_name="uniquevalidationrule", 28 | name="slug", 29 | ), 30 | migrations.AlterField( 31 | model_name="datacompliance", 32 | name="tags", 33 | field=nautobot.core.models.fields.TagsField(through="extras.TaggedItem", to="extras.Tag"), 34 | ), 35 | migrations.AlterField( 36 | model_name="minmaxvalidationrule", 37 | name="tags", 38 | field=nautobot.core.models.fields.TagsField(through="extras.TaggedItem", to="extras.Tag"), 39 | ), 40 | migrations.AlterField( 41 | model_name="regularexpressionvalidationrule", 42 | name="tags", 43 | field=nautobot.core.models.fields.TagsField(through="extras.TaggedItem", to="extras.Tag"), 44 | ), 45 | migrations.AlterField( 46 | model_name="requiredvalidationrule", 47 | name="tags", 48 | field=nautobot.core.models.fields.TagsField(through="extras.TaggedItem", to="extras.Tag"), 49 | ), 50 | migrations.AlterField( 51 | model_name="uniquevalidationrule", 52 | name="tags", 53 | field=nautobot.core.models.fields.TagsField(through="extras.TaggedItem", to="extras.Tag"), 54 | ), 55 | ] 56 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/migrations/0006_add_field_defaults.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.21 on 2023-10-17 14:13 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("nautobot_data_validation_engine", "0005_remove_slugs_alter_tags"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AlterField( 13 | model_name="datacompliance", 14 | name="message", 15 | field=models.TextField(blank=True, default=""), 16 | ), 17 | migrations.AlterField( 18 | model_name="datacompliance", 19 | name="validated_attribute", 20 | field=models.CharField(blank=True, default="", max_length=100), 21 | ), 22 | migrations.AlterField( 23 | model_name="datacompliance", 24 | name="validated_attribute_value", 25 | field=models.CharField(blank=True, default="", max_length=200), 26 | ), 27 | migrations.AlterField( 28 | model_name="datacompliance", 29 | name="validated_object_str", 30 | field=models.CharField(blank=True, default="", max_length=200), 31 | ), 32 | migrations.AlterField( 33 | model_name="minmaxvalidationrule", 34 | name="error_message", 35 | field=models.CharField(blank=True, default="", max_length=255), 36 | ), 37 | migrations.AlterField( 38 | model_name="regularexpressionvalidationrule", 39 | name="error_message", 40 | field=models.CharField(blank=True, default="", max_length=255), 41 | ), 42 | migrations.AlterField( 43 | model_name="requiredvalidationrule", 44 | name="error_message", 45 | field=models.CharField(blank=True, default="", max_length=255), 46 | ), 47 | migrations.AlterField( 48 | model_name="uniquevalidationrule", 49 | name="error_message", 50 | field=models.CharField(blank=True, default="", max_length=255), 51 | ), 52 | ] 53 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/migrations/0007_alter_datacompliance_compliance_class_name_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.16 on 2024-10-23 20:57 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("nautobot_data_validation_engine", "0006_add_field_defaults"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AlterField( 13 | model_name="datacompliance", 14 | name="compliance_class_name", 15 | field=models.CharField(max_length=255), 16 | ), 17 | migrations.AlterField( 18 | model_name="datacompliance", 19 | name="object_id", 20 | field=models.CharField(max_length=255), 21 | ), 22 | migrations.AlterField( 23 | model_name="datacompliance", 24 | name="validated_attribute", 25 | field=models.CharField(blank=True, default="", max_length=255), 26 | ), 27 | migrations.AlterField( 28 | model_name="datacompliance", 29 | name="validated_attribute_value", 30 | field=models.CharField(blank=True, default="", max_length=255), 31 | ), 32 | migrations.AlterField( 33 | model_name="datacompliance", 34 | name="validated_object_str", 35 | field=models.CharField(blank=True, default="", max_length=255), 36 | ), 37 | ] 38 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nautobot/nautobot-app-data-validation-engine/9b45481673b9a9cb503427554c5ecfe4d0e57a48/nautobot_data_validation_engine/migrations/__init__.py -------------------------------------------------------------------------------- /nautobot_data_validation_engine/navigation.py: -------------------------------------------------------------------------------- 1 | """App navigation menu items.""" 2 | 3 | from nautobot.apps.ui import NavMenuAddButton, NavMenuGroup, NavMenuImportButton, NavMenuItem, NavMenuTab 4 | 5 | menu_items = ( 6 | NavMenuTab( 7 | name="Extensibility", 8 | groups=( 9 | NavMenuGroup( 10 | name="Data Validation Engine", 11 | weight=200, 12 | items=( 13 | NavMenuItem( 14 | link="plugins:nautobot_data_validation_engine:minmaxvalidationrule_list", 15 | name="Min/Max Rules", 16 | permissions=["nautobot_data_validation_engine.view_minmaxvalidationrule"], 17 | buttons=( 18 | NavMenuAddButton( 19 | link="plugins:nautobot_data_validation_engine:minmaxvalidationrule_add", 20 | permissions=["nautobot_data_validation_engine.add_minmaxvalidationrule"], 21 | ), 22 | NavMenuImportButton( 23 | link="plugins:nautobot_data_validation_engine:minmaxvalidationrule_import", 24 | permissions=["nautobot_data_validation_engine.add_minmaxvalidationrule"], 25 | ), 26 | ), 27 | ), 28 | NavMenuItem( 29 | link="plugins:nautobot_data_validation_engine:regularexpressionvalidationrule_list", 30 | name="Regex Rules", 31 | permissions=["nautobot_data_validation_engine.view_regularexpressionvalidationrule"], 32 | buttons=( 33 | NavMenuAddButton( 34 | link="plugins:nautobot_data_validation_engine:regularexpressionvalidationrule_add", 35 | permissions=["nautobot_data_validation_engine.add_regularexpressionvalidationrule"], 36 | ), 37 | NavMenuImportButton( 38 | link="plugins:nautobot_data_validation_engine:regularexpressionvalidationrule_import", 39 | permissions=["nautobot_data_validation_engine.add_regularexpressionvalidationrule"], 40 | ), 41 | ), 42 | ), 43 | NavMenuItem( 44 | link="plugins:nautobot_data_validation_engine:requiredvalidationrule_list", 45 | name="Required Rules", 46 | permissions=["nautobot_data_validation_engine.view_requiredvalidationrule"], 47 | buttons=( 48 | NavMenuAddButton( 49 | link="plugins:nautobot_data_validation_engine:requiredvalidationrule_add", 50 | permissions=["nautobot_data_validation_engine.add_requiredvalidationrule"], 51 | ), 52 | NavMenuImportButton( 53 | link="plugins:nautobot_data_validation_engine:requiredvalidationrule_import", 54 | permissions=["nautobot_data_validation_engine.add_requiredvalidationrule"], 55 | ), 56 | ), 57 | ), 58 | NavMenuItem( 59 | link="plugins:nautobot_data_validation_engine:uniquevalidationrule_list", 60 | name="Unique Rules", 61 | permissions=["nautobot_data_validation_engine.view_uniquevalidationrule"], 62 | buttons=( 63 | NavMenuAddButton( 64 | link="plugins:nautobot_data_validation_engine:uniquevalidationrule_add", 65 | permissions=["nautobot_data_validation_engine.add_uniquevalidationrule"], 66 | ), 67 | NavMenuImportButton( 68 | link="plugins:nautobot_data_validation_engine:uniquevalidationrule_import", 69 | permissions=["nautobot_data_validation_engine.add_uniquevalidationrule"], 70 | ), 71 | ), 72 | ), 73 | NavMenuItem( 74 | link="plugins:nautobot_data_validation_engine:datacompliance_list", 75 | name="Data Compliance", 76 | permissions=["nautobot_data_validation_engine.view_datacompliance"], 77 | ), 78 | ), 79 | ), 80 | ), 81 | ), 82 | ) 83 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/tables.py: -------------------------------------------------------------------------------- 1 | """Tables for nautobot_data_validation_engine.""" 2 | 3 | import django_tables2 as tables 4 | from django.utils.html import format_html 5 | from nautobot.core.tables import BaseTable, TagColumn, ToggleColumn 6 | 7 | from nautobot_data_validation_engine.models import ( 8 | DataCompliance, 9 | MinMaxValidationRule, 10 | RegularExpressionValidationRule, 11 | RequiredValidationRule, 12 | UniqueValidationRule, 13 | ) 14 | 15 | # 16 | # RegularExpressionValidationRules 17 | # 18 | 19 | 20 | class RegularExpressionValidationRuleTable(BaseTable): 21 | """Base table for the RegularExpressionValidationRule model.""" 22 | 23 | pk = ToggleColumn() 24 | name = tables.LinkColumn(order_by=("name",)) 25 | tags = TagColumn() 26 | 27 | class Meta(BaseTable.Meta): 28 | """Table metadata for the RegularExpressionValidationRule model.""" 29 | 30 | model = RegularExpressionValidationRule 31 | fields = ( # pylint: disable=nb-use-fields-all 32 | "pk", 33 | "name", 34 | "enabled", 35 | "content_type", 36 | "field", 37 | "regular_expression", 38 | "context_processing", 39 | "error_message", 40 | "tags", 41 | ) 42 | default_columns = ( 43 | "pk", 44 | "name", 45 | "enabled", 46 | "content_type", 47 | "field", 48 | "regular_expression", 49 | "context_processing", 50 | "error_message", 51 | ) 52 | 53 | 54 | # 55 | # MinMaxValidationRules 56 | # 57 | 58 | 59 | class MinMaxValidationRuleTable(BaseTable): 60 | """Base table for the MinMaxValidationRule model.""" 61 | 62 | pk = ToggleColumn() 63 | name = tables.LinkColumn(order_by=("name",)) 64 | tags = TagColumn() 65 | 66 | class Meta(BaseTable.Meta): 67 | """Table metadata for the MinMaxValidationRuleTable model.""" 68 | 69 | model = MinMaxValidationRule 70 | fields = ( # pylint: disable=nb-use-fields-all 71 | "pk", 72 | "name", 73 | "enabled", 74 | "content_type", 75 | "field", 76 | "min", 77 | "max", 78 | "error_message", 79 | "tags", 80 | ) 81 | default_columns = ( 82 | "pk", 83 | "name", 84 | "enabled", 85 | "content_type", 86 | "field", 87 | "min", 88 | "max", 89 | "error_message", 90 | ) 91 | 92 | 93 | # 94 | # RequiredValidationRules 95 | # 96 | 97 | 98 | class RequiredValidationRuleTable(BaseTable): 99 | """Base table for the RequiredValidationRule model.""" 100 | 101 | pk = ToggleColumn() 102 | name = tables.LinkColumn(order_by=("name",)) 103 | tags = TagColumn() 104 | 105 | class Meta(BaseTable.Meta): 106 | """Table metadata for the RequiredValidationRuleTable model.""" 107 | 108 | model = RequiredValidationRule 109 | fields = ( # pylint: disable=nb-use-fields-all 110 | "pk", 111 | "name", 112 | "enabled", 113 | "content_type", 114 | "field", 115 | "error_message", 116 | "tags", 117 | ) 118 | default_columns = ( 119 | "pk", 120 | "name", 121 | "enabled", 122 | "content_type", 123 | "field", 124 | "error_message", 125 | ) 126 | 127 | 128 | # 129 | # UniqueValidationRules 130 | # 131 | 132 | 133 | class UniqueValidationRuleTable(BaseTable): 134 | """Base table for the UniqueValidationRule model.""" 135 | 136 | pk = ToggleColumn() 137 | name = tables.LinkColumn(order_by=("name",)) 138 | tags = TagColumn() 139 | 140 | class Meta(BaseTable.Meta): 141 | """Table metadata for the UniqueValidationRuleTable model.""" 142 | 143 | model = UniqueValidationRule 144 | fields = ( # pylint: disable=nb-use-fields-all 145 | "pk", 146 | "name", 147 | "enabled", 148 | "content_type", 149 | "field", 150 | "max_instances", 151 | "error_message", 152 | "tags", 153 | ) 154 | default_columns = ( 155 | "pk", 156 | "name", 157 | "enabled", 158 | "content_type", 159 | "field", 160 | "max_instances", 161 | "error_message", 162 | ) 163 | 164 | 165 | # 166 | # DataCompliance 167 | # 168 | 169 | 170 | class ValidatedAttributeColumn(tables.Column): 171 | """Column that links to the object's attribute if it is linkable.""" 172 | 173 | def render(self, value, record): # pylint: disable=W0221 174 | """Generate a link to a validated attribute if it is linkable, otherwise return the attribute.""" 175 | if hasattr(record.validated_object, value) and hasattr( 176 | getattr(record.validated_object, value), "get_absolute_url" 177 | ): 178 | return format_html('{}', getattr(record.validated_object, value).get_absolute_url(), value) 179 | return value 180 | 181 | 182 | class DataComplianceTable(BaseTable): 183 | """Base table for viewing all DataCompliance objects.""" 184 | 185 | pk = ToggleColumn() 186 | id = tables.Column(linkify=True, verbose_name="ID") 187 | validated_object = tables.RelatedLinkColumn() 188 | validated_attribute = ValidatedAttributeColumn() 189 | 190 | def order_validated_object(self, queryset, is_descending): 191 | """Reorder table by string representation of validated_object.""" 192 | qs = queryset.order_by(("-" if is_descending else "") + "validated_object_str") 193 | return (qs, True) 194 | 195 | class Meta(BaseTable.Meta): 196 | """Meta class for DataComplianceTable.""" 197 | 198 | model = DataCompliance 199 | fields = [ # pylint: disable=nb-use-fields-all 200 | "pk", 201 | "id", 202 | "content_type", 203 | "compliance_class_name", 204 | "last_validation_date", 205 | "validated_object", 206 | "validated_attribute", 207 | "validated_attribute_value", 208 | "valid", 209 | "message", 210 | ] 211 | default_columns = [ 212 | "pk", 213 | "id", 214 | "content_type", 215 | "compliance_class_name", 216 | "last_validation_date", 217 | "validated_object", 218 | "validated_attribute", 219 | "validated_attribute_value", 220 | "valid", 221 | "message", 222 | ] 223 | 224 | 225 | class DataComplianceTableTab(BaseTable): # pylint: disable=nb-sub-class-name 226 | """Base table for viewing the DataCompliance related to a single object.""" 227 | 228 | validated_attribute = ValidatedAttributeColumn() 229 | 230 | class Meta(BaseTable.Meta): 231 | """Meta class for DataComplianceTableTab.""" 232 | 233 | model = DataCompliance 234 | order_by = ("compliance_class_name", "validated_attribute") 235 | fields = [ # pylint: disable=nb-use-fields-all 236 | "content_type", 237 | "compliance_class_name", 238 | "last_validation_date", 239 | "validated_attribute", 240 | "validated_attribute_value", 241 | "valid", 242 | "message", 243 | ] 244 | default_columns = [ 245 | "content_type", 246 | "compliance_class_name", 247 | "last_validation_date", 248 | "validated_attribute", 249 | "validated_attribute_value", 250 | "valid", 251 | "message", 252 | ] 253 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/template_content.py: -------------------------------------------------------------------------------- 1 | """Template content for nautobot_data_validation_engine.""" 2 | 3 | from django.urls import reverse 4 | from nautobot.extras.plugins import TemplateExtension 5 | from nautobot.extras.utils import registry 6 | 7 | 8 | def tab_factory(content_type_label): 9 | """Generate a DataComplianceTab object for a given content type.""" 10 | 11 | class DataComplianceTab(TemplateExtension): # pylint: disable=W0223 12 | """Dynamically generated DataComplianceTab class.""" 13 | 14 | model = content_type_label 15 | 16 | def detail_tabs(self): 17 | return [ 18 | { 19 | "title": "Data Compliance", 20 | "url": reverse( 21 | "plugins:nautobot_data_validation_engine:data-compliance-tab", 22 | kwargs={"id": self.context["object"].id, "model": self.model}, 23 | ), 24 | }, 25 | ] 26 | 27 | return DataComplianceTab 28 | 29 | 30 | class ComplianceTemplateIterator: 31 | """Iterator that generates PluginCustomValidator classes for each model registered in the extras feature query registry 'custom_validators'.""" 32 | 33 | def __iter__(self): 34 | """Return a generator of PluginCustomValidator classes for each registered model.""" 35 | for app_label, models in registry["model_features"]["custom_validators"].items(): 36 | for model in models: 37 | label = f"{app_label}.{model}" 38 | yield tab_factory(label) 39 | 40 | 41 | template_extensions = ComplianceTemplateIterator() 42 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/templates/nautobot_data_validation_engine/datacompliance_retrieve.html: -------------------------------------------------------------------------------- 1 | {% extends 'generic/object_retrieve.html' %} 2 | {% load helpers %} 3 | {% load tz %} 4 | {% load static %} 5 | 6 | {% block content_left_page %} 7 |
8 |
9 | Data Compliance 10 |
11 | 12 | 13 | 14 | 17 | 18 | 19 | 20 | 21 | 24 | 25 | 26 | 27 | 30 | 31 | 32 | 33 | 36 | 37 | 38 | 39 | 42 | 43 | 44 | 45 | 48 | 49 | 50 | 51 | 54 | 55 |
Compliance Class Name 15 | {{ object.compliance_class_name }} 16 |
Last Validation Date 22 | {{ object.last_validation_date }} 23 |
Validated Object 28 | {{ object.validated_object|hyperlinked_object }} 29 |
Validated Attribute 34 | {{ object.validated_attribute }} 35 |
Validated Attribute Value 40 | {{ object.validated_attribute_value }} 41 |
Valid 46 | {{ object.valid }} 47 |
Message 52 | {{ object.message }} 53 |
56 |
57 | {% endblock %} -------------------------------------------------------------------------------- /nautobot_data_validation_engine/templates/nautobot_data_validation_engine/datacompliance_tab.html: -------------------------------------------------------------------------------- 1 | {% extends base_template %} 2 | {% load helpers %} 3 | {% load tz %} 4 | {% load static %} 5 | 6 | {% block title %} {{ object }} - Data Compliance {% endblock %} 7 | 8 | {% block content %} 9 | {% include 'responsive_table.html' %} 10 | {% endblock %} -------------------------------------------------------------------------------- /nautobot_data_validation_engine/templates/nautobot_data_validation_engine/minmaxvalidationrule_retrieve.html: -------------------------------------------------------------------------------- 1 | {% extends 'generic/object_detail.html' %} 2 | {% load helpers %} 3 | 4 | {% block content_left_page %} 5 |
6 |
7 | Min/Max Rule 8 |
9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 |
Name{{ object.name }}
Enabled 17 | {{ object.enabled | render_boolean }} 18 |
Content type{{ object.content_type.app_label }}.{{ object.content_type.model }}
Field{{ object.field }}
Min{{ object.min|placeholder }}
Max{{ object.max|placeholder }}
Error message{{ object.error_message|placeholder }}
41 |
42 | {% endblock %} 43 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/templates/nautobot_data_validation_engine/regularexpressionvalidationrule_retrieve.html: -------------------------------------------------------------------------------- 1 | {% extends 'generic/object_detail.html' %} 2 | {% load helpers %} 3 | 4 | {% block content_left_page %} 5 |
6 |
7 | Regular Expression Rule 8 |
9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 37 | 38 | 39 | 40 | 41 | 42 |
Name{{ object.name }}
Enabled 17 | {{ object.enabled | render_boolean }} 18 |
Content type{{ object.content_type.app_label }}.{{ object.content_type.model }}
Field{{ object.field }}
Regular expression{{ object.regular_expression }}
Context processing 35 | {{ object.context_processing | render_boolean }} 36 |
Error message{{ object.error_message|placeholder }}
43 |
44 | {% endblock %} 45 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/templates/nautobot_data_validation_engine/requiredvalidationrule_retrieve.html: -------------------------------------------------------------------------------- 1 | {% extends 'generic/object_detail.html' %} 2 | {% load helpers %} 3 | 4 | {% block content_left_page %} 5 |
6 |
7 | Required Rule 8 |
9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 |
Name{{ object.name }}
Enabled 17 | {{ object.enabled | render_boolean }} 18 |
Content type{{ object.content_type.app_label }}.{{ object.content_type.model }}
Field{{ object.field }}
Error message{{ object.error_message|placeholder }}
33 |
34 | {% endblock %} 35 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/templates/nautobot_data_validation_engine/uniquevalidationrule_retrieve.html: -------------------------------------------------------------------------------- 1 | {% extends 'generic/object_detail.html' %} 2 | {% load helpers %} 3 | 4 | {% block content_left_page %} 5 |
6 |
7 | Required Rule 8 |
9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 |
Name{{ object.name }}
Enabled 17 | {{ object.enabled | render_boolean }} 18 |
Content type{{ object.content_type.app_label }}.{{ object.content_type.model }}
Field{{ object.field }}
Max instances{{ object.max_instances }}
Error message{{ object.error_message|placeholder }}
37 |
38 | {% endblock %} 39 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Unit tests for nautobot_data_validation_engine app.""" 2 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/tests/test_api.py: -------------------------------------------------------------------------------- 1 | """Unit tests for nautobot_data_validation_engine.""" 2 | 3 | from django.contrib.contenttypes.models import ContentType 4 | from django.urls import reverse 5 | from nautobot.core.testing import APITestCase, APIViewTestCases 6 | from nautobot.dcim.models import Location, Manufacturer, Platform, PowerFeed 7 | 8 | from nautobot_data_validation_engine.models import ( 9 | MinMaxValidationRule, 10 | RegularExpressionValidationRule, 11 | RequiredValidationRule, 12 | UniqueValidationRule, 13 | ) 14 | 15 | 16 | class AppTest(APITestCase): 17 | """ 18 | Test base path for app 19 | """ 20 | 21 | def test_root(self): 22 | """ 23 | Test the root view 24 | """ 25 | url = reverse("plugins-api:nautobot_data_validation_engine-api:api-root") 26 | response = self.client.get(f"{url}?format=api", **self.header) 27 | 28 | self.assertEqual(response.status_code, 200) 29 | 30 | 31 | class RegularExpressionValidationRuleTest(APIViewTestCases.APIViewTestCase): 32 | """ 33 | API view test cases for the RegularExpressionValidationRule model 34 | """ 35 | 36 | model = RegularExpressionValidationRule 37 | brief_fields = [ 38 | "display", 39 | "id", 40 | "name", 41 | "url", 42 | ] 43 | choices_fields = {"content_type"} 44 | 45 | create_data = [ 46 | { 47 | "name": "Regex rule 4", 48 | "content_type": "dcim.location", 49 | "field": "contact_name", 50 | "regular_expression": "^.*$", 51 | }, 52 | { 53 | "name": "Regex rule 5", 54 | "content_type": "dcim.location", 55 | "field": "physical_address", 56 | "regular_expression": "^.*$", 57 | }, 58 | { 59 | "name": "Regex rule 6", 60 | "content_type": "dcim.location", 61 | "field": "shipping_address", 62 | "regular_expression": "^.*$", 63 | }, 64 | ] 65 | bulk_update_data = { 66 | "enabled": False, 67 | } 68 | 69 | @classmethod 70 | def setUpTestData(cls): 71 | """ 72 | Create test data 73 | """ 74 | RegularExpressionValidationRule.objects.create( 75 | name="Regex rule 1", 76 | content_type=ContentType.objects.get_for_model(Location), 77 | field="name", 78 | regular_expression="^.*$", 79 | ) 80 | RegularExpressionValidationRule.objects.create( 81 | name="Regex rule 2", 82 | content_type=ContentType.objects.get_for_model(Location), 83 | field="description", 84 | regular_expression="^.*$", 85 | ) 86 | RegularExpressionValidationRule.objects.create( 87 | name="Regex rule 3", 88 | content_type=ContentType.objects.get_for_model(Location), 89 | field="comments", 90 | regular_expression="^.*$", 91 | ) 92 | 93 | 94 | class MinMaxValidationRuleTest(APIViewTestCases.APIViewTestCase): 95 | """ 96 | API view test cases for the MinMaxValidationRule model 97 | """ 98 | 99 | model = MinMaxValidationRule 100 | brief_fields = [ 101 | "display", 102 | "id", 103 | "name", 104 | "url", 105 | ] 106 | choices_fields = {"content_type"} 107 | 108 | create_data = [ 109 | { 110 | "name": "Min max rule 4", 111 | "content_type": "dcim.device", 112 | "field": "vc_position", 113 | "min": 0, 114 | "max": 1, 115 | }, 116 | { 117 | "name": "Min max rule 5", 118 | "content_type": "dcim.device", 119 | "field": "vc_priority", 120 | "min": -5.6, 121 | "max": 0, 122 | }, 123 | { 124 | "name": "Min max rule 6", 125 | "content_type": "dcim.device", 126 | "field": "position", 127 | "min": 5, 128 | "max": 6, 129 | }, 130 | ] 131 | bulk_update_data = { 132 | "enabled": False, 133 | } 134 | 135 | @classmethod 136 | def setUpTestData(cls): 137 | """ 138 | Create test data 139 | """ 140 | MinMaxValidationRule.objects.create( 141 | name="Min max rule 1", 142 | content_type=ContentType.objects.get_for_model(PowerFeed), 143 | field="amperage", 144 | min=1, 145 | ) 146 | MinMaxValidationRule.objects.create( 147 | name="Min max rule 2", 148 | content_type=ContentType.objects.get_for_model(PowerFeed), 149 | field="max_utilization", 150 | min=1, 151 | ) 152 | MinMaxValidationRule.objects.create( 153 | name="Min max rule 3", 154 | content_type=ContentType.objects.get_for_model(PowerFeed), 155 | field="voltage", 156 | min=1, 157 | ) 158 | 159 | 160 | class RequiredValidationRuleTest(APIViewTestCases.APIViewTestCase): 161 | """ 162 | API view test cases for the RequiredValidationRule model 163 | """ 164 | 165 | model = RequiredValidationRule 166 | brief_fields = [ 167 | "display", 168 | "id", 169 | "name", 170 | "url", 171 | ] 172 | choices_fields = {"content_type"} 173 | 174 | create_data = [ 175 | { 176 | "name": "Required rule 4", 177 | "content_type": "dcim.location", 178 | "field": "physical_address", 179 | }, 180 | { 181 | "name": "Required rule 5", 182 | "content_type": "dcim.location", 183 | "field": "asn", 184 | }, 185 | { 186 | "name": "Required rule 6", 187 | "content_type": "dcim.location", 188 | "field": "facility", 189 | }, 190 | ] 191 | bulk_update_data = { 192 | "enabled": False, 193 | } 194 | 195 | @classmethod 196 | def setUpTestData(cls): 197 | """ 198 | Create test data 199 | """ 200 | RequiredValidationRule.objects.create( 201 | name="Required rule 1", 202 | content_type=ContentType.objects.get_for_model(Location), 203 | field="description", 204 | ) 205 | RequiredValidationRule.objects.create( 206 | name="Required rule 2", 207 | content_type=ContentType.objects.get_for_model(Platform), 208 | field="description", 209 | ) 210 | RequiredValidationRule.objects.create( 211 | name="Required rule 3", 212 | content_type=ContentType.objects.get_for_model(Manufacturer), 213 | field="description", 214 | ) 215 | 216 | 217 | class UniqueValidationRuleTest(APIViewTestCases.APIViewTestCase): 218 | """ 219 | API view test cases for the UniqueValidationRule model 220 | """ 221 | 222 | model = UniqueValidationRule 223 | brief_fields = [ 224 | "display", 225 | "id", 226 | "name", 227 | "url", 228 | ] 229 | choices_fields = {"content_type"} 230 | 231 | create_data = [ 232 | { 233 | "name": "Unique rule 4", 234 | "content_type": "dcim.location", 235 | "field": "physical_address", 236 | "max_instances": 1, 237 | }, 238 | { 239 | "name": "Unique rule 5", 240 | "content_type": "dcim.location", 241 | "field": "asn", 242 | "max_instances": 2, 243 | }, 244 | { 245 | "name": "Unique rule 6", 246 | "content_type": "dcim.location", 247 | "field": "facility", 248 | "max_instances": 3, 249 | }, 250 | ] 251 | bulk_update_data = { 252 | "enabled": False, 253 | } 254 | 255 | @classmethod 256 | def setUpTestData(cls): 257 | """ 258 | Create test data 259 | """ 260 | UniqueValidationRule.objects.create( 261 | name="Unique rule 1", 262 | content_type=ContentType.objects.get_for_model(Location), 263 | field="description", 264 | max_instances=1, 265 | ) 266 | UniqueValidationRule.objects.create( 267 | name="Unique rule 2", 268 | content_type=ContentType.objects.get_for_model(Platform), 269 | field="description", 270 | max_instances=2, 271 | ) 272 | UniqueValidationRule.objects.create( 273 | name="Unique rule 3", 274 | content_type=ContentType.objects.get_for_model(Manufacturer), 275 | field="description", 276 | max_instances=3, 277 | ) 278 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/tests/test_basic.py: -------------------------------------------------------------------------------- 1 | """Basic tests that do not require Django.""" 2 | 3 | import os 4 | import unittest 5 | 6 | import toml 7 | 8 | 9 | class TestDocsPackaging(unittest.TestCase): 10 | """Test Version in doc requirements is the same pyproject.""" 11 | 12 | def test_version(self): 13 | """Verify that pyproject.toml dev dependencies have the same versions as in the docs requirements.txt.""" 14 | parent_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) 15 | poetry_path = os.path.join(parent_path, "pyproject.toml") 16 | poetry_details = toml.load(poetry_path)["tool"]["poetry"]["group"]["dev"]["dependencies"] 17 | with open(f"{parent_path}/docs/requirements.txt", "r", encoding="utf-8") as file: 18 | requirements = [line for line in file.read().splitlines() if (len(line) > 0 and not line.startswith("#"))] 19 | for pkg in requirements: 20 | package_name = pkg 21 | if len(pkg.split("==")) == 2: # noqa: PLR2004 22 | package_name, version = pkg.split("==") 23 | else: 24 | version = "*" 25 | self.assertEqual(poetry_details[package_name], version) 26 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/tests/test_data_compliance_rules.py: -------------------------------------------------------------------------------- 1 | """DataComplianceRule test cases.""" 2 | 3 | from django.test import TestCase 4 | from nautobot.dcim.models import Location, LocationType 5 | from nautobot.extras.models import Status 6 | 7 | from nautobot_data_validation_engine.custom_validators import ComplianceError, DataComplianceRule 8 | from nautobot_data_validation_engine.models import DataCompliance 9 | 10 | 11 | class TestFailedDataComplianceRule(DataComplianceRule): 12 | """Test implementation of DataComplianceRule.""" 13 | 14 | model = "dcim.location" 15 | 16 | def audit(self): 17 | """Raises an AuditError.""" 18 | # this should create 4 different Audits, one for each 19 | # attribute 20 | raise ComplianceError( 21 | { 22 | "tenant": "Tenant", 23 | "description": "Description", 24 | "name": "Name", 25 | "status": "Status", 26 | } 27 | ) 28 | 29 | 30 | class TestPassedDataComplianceRule(DataComplianceRule): 31 | """Test implementation of DataComplianceRule.""" 32 | 33 | model = "dcim.location" 34 | 35 | def audit(self): 36 | """No exception means the audit passes.""" 37 | 38 | 39 | class TestCompliance(TestCase): 40 | """Test DataComplianceRule methods.""" 41 | 42 | def setUp(self): 43 | self.location_type = LocationType(name="Region") 44 | self.location_type.save() 45 | self.s = Location( 46 | name="Test 1", 47 | location_type=LocationType.objects.get_by_natural_key("Region"), 48 | status=Status.objects.get_by_natural_key("Active"), 49 | ) 50 | self.s.save() 51 | TestFailedDataComplianceRule(self.s).clean() 52 | TestPassedDataComplianceRule(self.s).clean() 53 | 54 | def test_audit_success(self): 55 | result = DataCompliance.objects.filter(valid=True).all() 56 | self.assertEqual(len(result), 1) 57 | result = result[0] 58 | self.assertEqual(result.compliance_class_name, "TestPassedDataComplianceRule") 59 | self.assertEqual(result.validated_object, self.s) 60 | self.assertEqual(result.validated_attribute, "__all__") 61 | self.assertEqual(result.validated_attribute_value, "") 62 | 63 | def test_audit_fail(self): 64 | result = DataCompliance.objects.filter(valid=False).all() 65 | self.assertEqual(len(result), 5) 66 | result = DataCompliance.objects.get(validated_attribute="tenant") 67 | self.assertEqual(result.compliance_class_name, "TestFailedDataComplianceRule") 68 | self.assertEqual(result.validated_object, self.s) 69 | self.assertIn(result.validated_attribute, "tenant") 70 | self.assertEqual(result.validated_attribute_value, "") 71 | 72 | def test_validate_replaces_results(self): 73 | self.assertEqual( 74 | len(DataCompliance.objects.filter(compliance_class_name=TestFailedDataComplianceRule.__name__)), 5 75 | ) 76 | TestFailedDataComplianceRule(self.s).clean() 77 | self.assertEqual( 78 | len(DataCompliance.objects.filter(compliance_class_name=TestFailedDataComplianceRule.__name__)), 79 | 5, 80 | ) 81 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/urls.py: -------------------------------------------------------------------------------- 1 | """Django urlpatterns declaration for nautobot_data_validation_engine app.""" 2 | 3 | from django.templatetags.static import static 4 | from django.urls import path 5 | from django.views.generic import RedirectView 6 | from nautobot.core.views.routers import NautobotUIViewSetRouter 7 | from nautobot.extras.views import ObjectChangeLogView, ObjectNotesView 8 | 9 | from nautobot_data_validation_engine import models, views 10 | 11 | app_name = "nautobot_data_validation_engine" 12 | router = NautobotUIViewSetRouter() 13 | router.register("data-compliance", views.DataComplianceListView) 14 | router.register("regex-rules", views.RegularExpressionValidationRuleUIViewSet) 15 | router.register("min-max-rules", views.MinMaxValidationRuleUIViewSet) 16 | router.register("required-rules", views.RequiredValidationRuleUIViewSet) 17 | router.register("unique-rules", views.UniqueValidationRuleUIViewSet) 18 | 19 | urlpatterns = [ 20 | path( 21 | "data-compliance//changelog/", 22 | ObjectChangeLogView.as_view(), 23 | name="datacompliance_changelog", 24 | kwargs={"model": models.DataCompliance}, 25 | ), 26 | path( 27 | "data-compliance//notes/", 28 | ObjectNotesView.as_view(), 29 | name="datacompliance_notes", 30 | kwargs={"model": models.DataCompliance}, 31 | ), 32 | path( 33 | "data-compliance///", 34 | views.DataComplianceObjectView.as_view(), 35 | name="data-compliance-tab", 36 | ), 37 | path( 38 | "regex-rules//changelog/", 39 | ObjectChangeLogView.as_view(), 40 | name="regularexpressionvalidationrule_changelog", 41 | kwargs={"model": models.RegularExpressionValidationRule}, 42 | ), 43 | path( 44 | "regex-rules//notes/", 45 | ObjectNotesView.as_view(), 46 | name="regularexpressionvalidationrule_notes", 47 | kwargs={"model": models.RegularExpressionValidationRule}, 48 | ), 49 | path( 50 | "min-max-rules//changelog/", 51 | ObjectChangeLogView.as_view(), 52 | name="minmaxvalidationrule_changelog", 53 | kwargs={"model": models.MinMaxValidationRule}, 54 | ), 55 | path( 56 | "min-max-rules//notes/", 57 | ObjectNotesView.as_view(), 58 | name="minmaxvalidationrule_notes", 59 | kwargs={"model": models.MinMaxValidationRule}, 60 | ), 61 | path( 62 | "required-rules//changelog/", 63 | ObjectChangeLogView.as_view(), 64 | name="requiredvalidationrule_changelog", 65 | kwargs={"model": models.RequiredValidationRule}, 66 | ), 67 | path( 68 | "required-rules//notes/", 69 | ObjectNotesView.as_view(), 70 | name="requiredvalidationrule_notes", 71 | kwargs={"model": models.RequiredValidationRule}, 72 | ), 73 | path( 74 | "unique-rules//changelog/", 75 | ObjectChangeLogView.as_view(), 76 | name="uniquevalidationrule_changelog", 77 | kwargs={"model": models.UniqueValidationRule}, 78 | ), 79 | path( 80 | "unique-rules//notes/", 81 | ObjectNotesView.as_view(), 82 | name="uniquevalidationrule_notes", 83 | kwargs={"model": models.UniqueValidationRule}, 84 | ), 85 | path("docs/", RedirectView.as_view(url=static("nautobot_data_validation_engine/docs/index.html")), name="docs"), 86 | ] + router.urls 87 | -------------------------------------------------------------------------------- /nautobot_data_validation_engine/views.py: -------------------------------------------------------------------------------- 1 | """Views for nautobot_data_validation_engine.""" 2 | 3 | from django.apps import apps as global_apps 4 | from django.contrib.contenttypes.models import ContentType 5 | from django_tables2 import RequestConfig 6 | from nautobot.apps.views import ( 7 | ObjectBulkDestroyViewMixin, 8 | ObjectDestroyViewMixin, 9 | ObjectDetailViewMixin, 10 | ObjectListViewMixin, 11 | ) 12 | from nautobot.core.views.generic import ObjectView 13 | from nautobot.core.views.paginator import EnhancedPaginator, get_paginate_count 14 | from nautobot.core.views.viewsets import NautobotUIViewSet 15 | from nautobot.extras.utils import get_base_template 16 | 17 | from nautobot_data_validation_engine import filters, forms, tables 18 | from nautobot_data_validation_engine.api import serializers 19 | from nautobot_data_validation_engine.models import ( 20 | DataCompliance, 21 | MinMaxValidationRule, 22 | RegularExpressionValidationRule, 23 | RequiredValidationRule, 24 | UniqueValidationRule, 25 | ) 26 | 27 | # 28 | # RegularExpressionValidationRules 29 | # 30 | 31 | 32 | class RegularExpressionValidationRuleUIViewSet(NautobotUIViewSet): 33 | """Views for the RegularExpressionValidationRule model.""" 34 | 35 | bulk_update_form_class = forms.RegularExpressionValidationRuleBulkEditForm 36 | filterset_class = filters.RegularExpressionValidationRuleFilterSet 37 | filterset_form_class = forms.RegularExpressionValidationRuleFilterForm 38 | form_class = forms.RegularExpressionValidationRuleForm 39 | queryset = RegularExpressionValidationRule.objects.all() 40 | serializer_class = serializers.RegularExpressionValidationRuleSerializer 41 | table_class = tables.RegularExpressionValidationRuleTable 42 | 43 | 44 | # 45 | # MinMaxValidationRules 46 | # 47 | 48 | 49 | class MinMaxValidationRuleUIViewSet(NautobotUIViewSet): 50 | """Views for the MinMaxValidationRuleUIViewSet model.""" 51 | 52 | bulk_update_form_class = forms.MinMaxValidationRuleBulkEditForm 53 | filterset_class = filters.MinMaxValidationRuleFilterSet 54 | filterset_form_class = forms.MinMaxValidationRuleFilterForm 55 | form_class = forms.MinMaxValidationRuleForm 56 | queryset = MinMaxValidationRule.objects.all() 57 | serializer_class = serializers.MinMaxValidationRuleSerializer 58 | table_class = tables.MinMaxValidationRuleTable 59 | 60 | 61 | # 62 | # RequiredValidationRules 63 | # 64 | 65 | 66 | class RequiredValidationRuleUIViewSet(NautobotUIViewSet): 67 | """Views for the RequiredValidationRuleUIViewSet model.""" 68 | 69 | bulk_update_form_class = forms.RequiredValidationRuleBulkEditForm 70 | filterset_class = filters.RequiredValidationRuleFilterSet 71 | filterset_form_class = forms.RequiredValidationRuleFilterForm 72 | form_class = forms.RequiredValidationRuleForm 73 | queryset = RequiredValidationRule.objects.all() 74 | serializer_class = serializers.RequiredValidationRuleSerializer 75 | table_class = tables.RequiredValidationRuleTable 76 | 77 | 78 | # 79 | # UniqueValidationRules 80 | # 81 | 82 | 83 | class UniqueValidationRuleUIViewSet(NautobotUIViewSet): 84 | """Views for the UniqueValidationRuleUIViewSet model.""" 85 | 86 | bulk_update_form_class = forms.UniqueValidationRuleBulkEditForm 87 | filterset_class = filters.UniqueValidationRuleFilterSet 88 | filterset_form_class = forms.UniqueValidationRuleFilterForm 89 | form_class = forms.UniqueValidationRuleForm 90 | queryset = UniqueValidationRule.objects.all() 91 | serializer_class = serializers.UniqueValidationRuleSerializer 92 | table_class = tables.UniqueValidationRuleTable 93 | 94 | 95 | # 96 | # DataCompliance 97 | # 98 | 99 | 100 | class DataComplianceListView( # pylint: disable=W0223 101 | ObjectListViewMixin, ObjectDetailViewMixin, ObjectDestroyViewMixin, ObjectBulkDestroyViewMixin 102 | ): 103 | """Views for the DataComplianceListView model.""" 104 | 105 | lookup_field = "pk" 106 | queryset = DataCompliance.objects.all() 107 | table_class = tables.DataComplianceTable 108 | filterset_class = filters.DataComplianceFilterSet 109 | filterset_form_class = forms.DataComplianceFilterForm 110 | serializer_class = serializers.DataComplianceSerializer 111 | action_buttons = ("export",) 112 | 113 | 114 | class DataComplianceObjectView(ObjectView): 115 | """View for the Audit Results tab dynamically generated on specific object detail views.""" 116 | 117 | template_name = "nautobot_data_validation_engine/datacompliance_tab.html" 118 | queryset = None 119 | 120 | def dispatch(self, request, *args, **kwargs): 121 | """Set the queryset for the given object and call the inherited dispatch method.""" 122 | model = kwargs.pop("model") 123 | if not self.queryset: 124 | self.queryset = global_apps.get_model(model).objects.all() 125 | return super().dispatch(request, *args, **kwargs) 126 | 127 | def get_extra_context(self, request, instance): 128 | """Generate extra context for rendering the DataComplianceObjectView template.""" 129 | compliance_objects = DataCompliance.objects.filter( 130 | content_type=ContentType.objects.get_for_model(instance), object_id=instance.id 131 | ) 132 | compliance_table = tables.DataComplianceTableTab(compliance_objects) 133 | base_template = get_base_template(None, instance) 134 | 135 | paginate = {"paginator_class": EnhancedPaginator, "per_page": get_paginate_count(request)} 136 | RequestConfig(request, paginate).configure(compliance_table) 137 | return {"active_tab": request.GET["tab"], "table": compliance_table, "base_template": base_template} 138 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "nautobot-data-validation-engine" 3 | version = "3.2.0" 4 | description = "Provides UI to build custom data validation rules for data in Nautobot" 5 | authors = ["Network to Code, LLC "] 6 | license = "Apache-2.0" 7 | readme = "README.md" 8 | homepage = "https://github.com/nautobot/nautobot-app-data-validation-engine" 9 | repository = "https://github.com/nautobot/nautobot-app-data-validation-engine" 10 | documentation = "https://docs.nautobot.com/projects/data-validation/en/latest/" 11 | keywords = ["nautobot", "nautobot-app", "nautobot-plugin"] 12 | classifiers = [ 13 | "Intended Audience :: Developers", 14 | "Development Status :: 5 - Production/Stable", 15 | "Programming Language :: Python :: 3", 16 | "Programming Language :: Python :: 3.8", 17 | "Programming Language :: Python :: 3.9", 18 | "Programming Language :: Python :: 3.10", 19 | "Programming Language :: Python :: 3.11", 20 | "Programming Language :: Python :: 3.12", 21 | ] 22 | packages = [ 23 | { include = "nautobot_data_validation_engine" }, 24 | ] 25 | include = [ 26 | # Poetry by default will exclude files that are in .gitignore 27 | { path = "nautobot_data_validation_engine/static/nautobot_data_validation_engine/docs/**/*", format = ["sdist", "wheel"] } 28 | ] 29 | 30 | [tool.poetry.dependencies] 31 | python = ">=3.8,<3.13" 32 | # Used for local development 33 | nautobot = "^2.1.9" 34 | 35 | [tool.poetry.group.dev.dependencies] 36 | coverage = "*" 37 | django-debug-toolbar = "*" 38 | invoke = "*" 39 | ipython = "*" 40 | pylint = "*" 41 | pylint-django = "*" 42 | pylint-nautobot = "*" 43 | ruff = "0.5.5" 44 | yamllint = "*" 45 | toml = "*" 46 | # Python implementation of markdownlint 47 | pymarkdownlnt = [ 48 | {version = "0.9.26", python = "~3.8.0"}, 49 | {version = "~0.9.29", python = ">=3.9,<3.13"}, 50 | ] 51 | Markdown = "*" 52 | # Render custom markdown for version added/changed/remove notes 53 | markdown-version-annotations = "1.0.1" 54 | # Rendering docs to HTML 55 | mkdocs = "1.6.0" 56 | # Material for MkDocs theme 57 | mkdocs-material = "9.5.32" 58 | # Automatic documentation from sources, for MkDocs 59 | mkdocstrings = "0.25.2" 60 | mkdocstrings-python = "1.10.8" 61 | mkdocs-autorefs = "1.2.0" 62 | griffe = "1.1.1" 63 | towncrier = ">=23.6.0,<=24.8.0" 64 | to-json-schema = "*" 65 | jsonschema = "*" 66 | 67 | [tool.poetry.extras] 68 | all = [ 69 | ] 70 | 71 | [tool.pylint.master] 72 | # Include the pylint_django plugin to avoid spurious warnings about Django patterns 73 | load-plugins = "pylint_django, pylint_nautobot" 74 | ignore = ".venv" 75 | 76 | [tool.pylint.basic] 77 | # No docstrings required for private methods (Pylint default), or for test_ functions, or for inner Meta classes. 78 | no-docstring-rgx = "^(_|test_|Meta$)" 79 | 80 | [tool.pylint.messages_control] 81 | disable = """, 82 | duplicate-code, 83 | invalid-name, 84 | line-too-long, 85 | too-few-public-methods, 86 | too-many-ancestors 87 | """ 88 | 89 | [tool.pylint.miscellaneous] 90 | # Don't flag TODO as a failure, let us commit with things that still need to be done in the code 91 | notes = """, 92 | FIXME, 93 | XXX, 94 | """ 95 | 96 | [tool.pylint-nautobot] 97 | supported_nautobot_versions = [ 98 | "2.1.9" 99 | ] 100 | 101 | [tool.ruff] 102 | line-length = 120 103 | target-version = "py38" 104 | 105 | [tool.ruff.lint] 106 | select = [ 107 | "D", # pydocstyle 108 | "F", "E", "W", # flake8 109 | "S", # bandit 110 | "I", # isort 111 | ] 112 | ignore = [ 113 | # warning: `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible. 114 | "D203", # 1 blank line required before class docstring 115 | 116 | # D212 is enabled by default in google convention, and complains if we have a docstring like: 117 | # """ 118 | # My docstring is on the line after the opening quotes instead of on the same line as them. 119 | # """ 120 | # We've discussed and concluded that we consider this to be a valid style choice. 121 | "D212", # Multi-line docstring summary should start at the first line 122 | "D213", # Multi-line docstring summary should start at the second line 123 | 124 | # Produces a lot of issues in the current codebase. 125 | "D401", # First line of docstring should be in imperative mood 126 | "D407", # Missing dashed underline after section 127 | "D416", # Section name ends in colon 128 | "E501", # Line too long 129 | ] 130 | 131 | [tool.ruff.lint.pydocstyle] 132 | convention = "google" 133 | 134 | [tool.ruff.lint.per-file-ignores] 135 | "nautobot_data_validation_engine/migrations/*" = [ 136 | "D", 137 | ] 138 | "nautobot_data_validation_engine/tests/*" = [ 139 | "D", 140 | "S" 141 | ] 142 | 143 | [tool.coverage.run] 144 | disable_warnings = ["already-imported"] 145 | relative_files = true 146 | omit = [ 147 | # Skip Tests 148 | "*/tests/*", 149 | ] 150 | include = [ 151 | "nautobot_data_validation_engine/*", 152 | ] 153 | 154 | [tool.pymarkdown] 155 | # Seems to be not support for whitelisting rules: https://github.com/jackdewinter/pymarkdown/issues/1396 156 | plugins.md001.enabled = false 157 | plugins.md002.enabled = false 158 | plugins.md003.enabled = false 159 | plugins.md004.enabled = false 160 | plugins.md005.enabled = false 161 | plugins.md006.enabled = false 162 | plugins.md007.enabled = false 163 | plugins.md008.enabled = false 164 | plugins.md009.enabled = false 165 | plugins.md010.enabled = false 166 | plugins.md011.enabled = false 167 | plugins.md012.enabled = false 168 | plugins.md013.enabled = false 169 | plugins.md014.enabled = false 170 | plugins.md015.enabled = false 171 | plugins.md016.enabled = false 172 | plugins.md017.enabled = false 173 | plugins.md018.enabled = false 174 | plugins.md019.enabled = false 175 | plugins.md020.enabled = false 176 | plugins.md021.enabled = false 177 | plugins.md022.enabled = false 178 | plugins.md023.enabled = false 179 | plugins.md024.enabled = false 180 | plugins.md025.enabled = false 181 | plugins.md026.enabled = false 182 | plugins.md027.enabled = false 183 | plugins.md028.enabled = false 184 | plugins.md029.enabled = false 185 | plugins.md030.enabled = false 186 | plugins.md031.enabled = false 187 | plugins.md032.enabled = true # blanks-around-lists 188 | plugins.md033.enabled = false 189 | plugins.md034.enabled = false 190 | plugins.md035.enabled = false 191 | plugins.md036.enabled = false 192 | plugins.md037.enabled = false 193 | plugins.md038.enabled = false 194 | plugins.md039.enabled = false 195 | plugins.md040.enabled = false 196 | plugins.md041.enabled = false 197 | plugins.md042.enabled = false 198 | plugins.md043.enabled = false 199 | plugins.md044.enabled = false 200 | plugins.md045.enabled = false 201 | plugins.md046.enabled = false 202 | plugins.md047.enabled = false 203 | plugins.md048.enabled = false 204 | plugins.md049.enabled = false 205 | plugins.md050.enabled = false 206 | plugins.pml100.enabled = false 207 | plugins.pml101.enabled = true # list-anchored-indent 208 | plugins.pml102.enabled = false 209 | plugins.pml103.enabled = false 210 | 211 | [build-system] 212 | requires = ["poetry_core>=1.0.0"] 213 | build-backend = "poetry.core.masonry.api" 214 | 215 | [tool.towncrier] 216 | package = "nautobot_data_validation_engine" 217 | directory = "changes" 218 | filename = "docs/admin/release_notes/version_X.Y.md" 219 | template = "development/towncrier_template.j2" 220 | start_string = "" 221 | issue_format = "[#{issue}](https://github.com/nautobot/nautobot-app-data-validation-engine/issues/{issue})" 222 | 223 | [[tool.towncrier.type]] 224 | directory = "security" 225 | name = "Security" 226 | showcontent = true 227 | 228 | [[tool.towncrier.type]] 229 | directory = "added" 230 | name = "Added" 231 | showcontent = true 232 | 233 | [[tool.towncrier.type]] 234 | directory = "changed" 235 | name = "Changed" 236 | showcontent = true 237 | 238 | [[tool.towncrier.type]] 239 | directory = "deprecated" 240 | name = "Deprecated" 241 | showcontent = true 242 | 243 | [[tool.towncrier.type]] 244 | directory = "removed" 245 | name = "Removed" 246 | showcontent = true 247 | 248 | [[tool.towncrier.type]] 249 | directory = "fixed" 250 | name = "Fixed" 251 | showcontent = true 252 | 253 | [[tool.towncrier.type]] 254 | directory = "dependencies" 255 | name = "Dependencies" 256 | showcontent = true 257 | 258 | [[tool.towncrier.type]] 259 | directory = "documentation" 260 | name = "Documentation" 261 | showcontent = true 262 | 263 | [[tool.towncrier.type]] 264 | directory = "housekeeping" 265 | name = "Housekeeping" 266 | showcontent = true 267 | --------------------------------------------------------------------------------