├── .dockerignore ├── .github └── workflows │ └── testrun.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README.md ├── RUNNING_TESTS.md ├── config ├── config.example.json ├── config.json ├── defaults.json ├── log-config.yml └── redis.conf ├── docker-compose.ci.yml ├── docker-compose.dev.yml ├── docker-compose.yml ├── dockerfiles ├── dev_dockerfile ├── netpalm_controller_dockerfile ├── netpalm_fifo_worker_dockerfile ├── netpalm_pinned_worker_dockerfile └── netpalm_redis_dockerfile ├── gen_service_package.py ├── gunicorn.conf.py ├── netpalm.postman_collection.json ├── netpalm ├── __init__.py ├── backend │ ├── core │ │ ├── calls │ │ │ ├── dryrun │ │ │ │ └── dryrun.py │ │ │ ├── getconfig │ │ │ │ ├── __init__.py │ │ │ │ ├── exec_command.py │ │ │ │ └── ncclient_get.py │ │ │ ├── scriptrunner │ │ │ │ ├── __init__.py │ │ │ │ └── script.py │ │ │ ├── service │ │ │ │ ├── __init__.py │ │ │ │ ├── netpalmservice.py │ │ │ │ └── procedures.py │ │ │ └── setconfig │ │ │ │ ├── __init__.py │ │ │ │ └── exec_config.py │ │ ├── confload │ │ │ ├── __init__.py │ │ │ └── confload.py │ │ ├── driver │ │ │ ├── __init__.py │ │ │ ├── driver_auto_loader.py │ │ │ └── netpalm_driver.py │ │ ├── manager │ │ │ ├── __init__.py │ │ │ └── netpalm_manager.py │ │ ├── models │ │ │ ├── __init__.py │ │ │ ├── models.py │ │ │ ├── napalm.py │ │ │ ├── ncclient.py │ │ │ ├── netmiko.py │ │ │ ├── puresnmp.py │ │ │ ├── restconf.py │ │ │ ├── service.py │ │ │ ├── task.py │ │ │ └── transaction_log.py │ │ ├── redis │ │ │ ├── __init__.py │ │ │ └── rediz.py │ │ ├── routes │ │ │ ├── __init__.py │ │ │ └── routes.py │ │ ├── schedule │ │ │ ├── __init__.py │ │ │ └── schedule.py │ │ ├── security │ │ │ ├── __init__.py │ │ │ ├── cert │ │ │ │ └── tls │ │ │ │ │ ├── ca.crt │ │ │ │ │ ├── ca.key │ │ │ │ │ ├── ca.txt │ │ │ │ │ ├── redis.crt │ │ │ │ │ ├── redis.dh │ │ │ │ │ └── redis.key │ │ │ ├── get_api_key.py │ │ │ └── whitelist.py │ │ └── utilities │ │ │ ├── __init__.py │ │ │ ├── extensibles_reload.py │ │ │ ├── jinja2 │ │ │ ├── __init__.py │ │ │ └── j2.py │ │ │ ├── ls │ │ │ ├── __init__.py │ │ │ └── ls.py │ │ │ ├── rediz_kill_worker.py │ │ │ ├── rediz_meta.py │ │ │ ├── rediz_worker_controller.py │ │ │ ├── textfsm │ │ │ ├── __init__.py │ │ │ └── template.py │ │ │ ├── universal_template_mgr │ │ │ ├── __init__.py │ │ │ └── unvrsl.py │ │ │ └── webhook │ │ │ ├── __init__.py │ │ │ └── webhook.py │ └── plugins │ │ ├── drivers │ │ ├── napalm │ │ │ ├── __init__.py │ │ │ └── napalm_drvr.py │ │ ├── ncclient │ │ │ ├── __init__.py │ │ │ └── ncclient_drvr.py │ │ ├── netmiko │ │ │ ├── __init__.py │ │ │ └── netmiko_drvr.py │ │ ├── puresnmp │ │ │ ├── __init__.py │ │ │ └── puresnmp_drvr.py │ │ └── restconf │ │ │ ├── __init__.py │ │ │ └── restconf.py │ │ └── extensibles │ │ ├── custom_scripts │ │ ├── __init__.py │ │ ├── hello_world.py │ │ ├── hello_world_advanced_using_netpalm_manager.py │ │ ├── hello_world_embedded_pydanticmodel.py │ │ ├── hello_world_model.py │ │ └── hello_world_test.py │ │ ├── custom_webhooks │ │ ├── __init__.py │ │ ├── default_webhook.py │ │ ├── elastic.py │ │ └── servicenow_request_item_patch_netpalm_webhook.py │ │ ├── j2_config_templates │ │ ├── __init__.py │ │ ├── cisco_ios_add_ip_int.j2 │ │ ├── cisco_ios_add_snmp_config.j2 │ │ ├── cisco_ios_create_vlans.j2 │ │ ├── cisco_ios_remove_ip_int.j2 │ │ ├── cisco_ios_remove_snmp_config.j2 │ │ ├── cisco_ios_remove_vlans.j2 │ │ ├── cisco_ios_shut_interfaces.j2 │ │ ├── cisco_ios_un_shut_interfaces.j2 │ │ ├── ncclient_test.j2 │ │ └── test.j2 │ │ ├── j2_webhook_templates │ │ ├── __init__.py │ │ └── example.j2 │ │ ├── services │ │ ├── __init__.py │ │ └── example_simple.py │ │ └── ttp_templates │ │ └── example.ttp ├── controller_addtl_requirements.txt ├── exceptions.py ├── netpalm_controller.py ├── netpalm_fifo_worker.py ├── netpalm_pinned_worker.py ├── netpalm_worker_common.py ├── requirements.txt ├── routers │ ├── getconfig.py │ ├── public.py │ ├── route_utils.py │ ├── schedule.py │ ├── script.py │ ├── service.py │ ├── setconfig.py │ ├── task.py │ ├── template.py │ └── util.py └── static │ ├── css │ └── swagger-ui.css │ ├── images │ └── netpalm_white.png │ └── js │ └── swagger-ui-bundle.min.js ├── pytest.ini ├── redis_gen_new_certs.sh ├── static └── images │ ├── arch.png │ ├── netpalm.png │ ├── netpalm_concept.png │ ├── netpalm_eg_1.png │ ├── netpalm_eg_2.png │ ├── netpalm_eg_3.png │ ├── netpalm_eg_4.png │ ├── netpalm_ingest.gif │ ├── netpalm_postman.png │ ├── netpalm_service_closed_loop_demo.gif │ ├── netpalm_white.png │ ├── np-basic-new1.png │ ├── np-basic-new2.png │ ├── np-basic-overview.png │ ├── np-basic-q.png │ ├── np-component.png │ ├── np-scale-out.png │ ├── np_new.png │ ├── np_service.gif │ └── oapi.png ├── tests ├── __init__.py ├── integration │ ├── __init__.py │ ├── helper.py │ ├── test_getconfig.py │ ├── test_getconfig_cisgo.py │ ├── test_misc.py │ ├── test_script.py │ ├── test_service.py │ ├── test_setconfig.py │ ├── test_setconfig_cisgo.py │ └── test_worker.py └── unit │ ├── __init__.py │ ├── test_confload.py │ ├── test_device_whitelist.py │ ├── test_napalm_driver.py │ ├── test_ncclient_driver.py │ ├── test_netmiko_driver.py │ ├── test_router_utils.py │ ├── test_tfsm_templates.py │ └── test_update_log.py └── worker.py /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .github 3 | .gitignore 4 | .idea 5 | .vscode 6 | venv 7 | backend/plugins/extensibles/ntc-templates 8 | static 9 | docker-compose*.yml 10 | dockerfiles -------------------------------------------------------------------------------- /.github/workflows/testrun.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: tests 5 | 6 | on: [ push, pull_request ] 7 | 8 | jobs: 9 | build: 10 | 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | # - run: docker-compose -f docker-compose.ci.yml pull # don't cache this 17 | 18 | # - run: docker-compose -f docker-compose.ci.yml build cisgo redis # don't cache this 19 | 20 | - run: docker pull python:3.8-slim 21 | - run: docker pull apcela/cisshgo:v0.1.0 22 | - run: docker pull redis:6.0.7-alpine 23 | 24 | - uses: satackey/action-docker-layer-caching@v0.0.11 25 | continue-on-error: true 26 | with: 27 | key: ci-tests-{hash} 28 | restore-keys: | 29 | ci-tests- 30 | 31 | - name: Build the stack 32 | # run: docker-compose -f ./docker-compose.dev.yml up -d 33 | run: docker-compose -f docker-compose.ci.yml build --parallel 34 | 35 | - run: docker-compose -f docker-compose.ci.yml up -d 36 | 37 | - id: test_nolab 38 | name: integration tests 39 | run: docker-compose -f docker-compose.ci.yml exec -T netpalm-controller pytest -m "not fulllab" -vv tests/integration 40 | 41 | - id: test_cisgo 42 | name: unit tests 43 | run: docker-compose -f docker-compose.ci.yml exec -T netpalm-controller pytest -vv tests/unit 44 | # run: docker-compose -f ./docker-compose.dev.yml run controller echo "asdf" 45 | # - name: notify slack 46 | # if: ${{ always() }} 47 | # env: 48 | # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} 49 | # uses: abinoda/slack-action@master 50 | # with: 51 | # args: '{\"channel\":\"CUCQA382D\",\"blocks\": [ { \"type\": \"section\", \"text\": { \"type\": \"mrkdwn\", \"text\": \"Hey! ${{ github.actor }} just pushed to ${{ github.base_ref }} @ ${{ github.repositoryUrl }}. The Job status is ${{ job.status }}, the step outcome is ${{ steps.test.outcome }}, and the step conclusion is ${{ steps.test.conclusion }}!\" } } ]}' -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | .vscode/settings.json 3 | .idea 4 | *.pyc 5 | venv 6 | backend/plugins/extensibles/ntc-templates/ 7 | backend/plugins/extensibles/ntc-templates 8 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at eckelcu@cisco.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Guidance on how to contribute 2 | 3 | Contributions to this code are welcome and appreciated. 4 | Please adhere to our [Code of Conduct](./CODE_OF_CONDUCT.md) at all times. 5 | 6 | > All contributions to this code will be released under the terms of the [LICENSE](./LICENSE) of this code. By submitting a pull request or filing a bug, issue, or feature request, you are agreeing to comply with this waiver of copyright interest. Details can be found in our [LICENSE](./LICENSE). 7 | 8 | There are two primary ways to contribute: 9 | 10 | 1. Using the issue tracker 11 | 2. Changing the codebase 12 | 13 | 14 | ## Using the issue tracker 15 | 16 | Use the issue tracker to suggest feature requests, report bugs, and ask questions. This is also a great way to connect with the developers of the project as well as others who are interested in this solution. 17 | 18 | Use the issue tracker to find ways to contribute. Find a bug or a feature, mention in the issue that you will take on that effort, then follow the _Changing the codebase_ guidance below. 19 | 20 | 21 | ## Changing the codebase 22 | 23 | Generally speaking, you should fork this repository, make changes in your own fork, and then submit a pull request. All new code should have associated unit tests (if applicable) that validate implemented features and the presence or lack of defects. 24 | 25 | Additionally, the code should follow any stylistic and architectural guidelines prescribed by the project. In the absence of such guidelines, mimic the styles and patterns in the existing codebase. 26 | 27 | ## Hacking on Netpalm 28 | 29 | #### netpalm support 30 | 31 | We maintain an active community on the networktocode slack channel. `#netpalm` on `networktocode.slack.com`. Come say 'Hi' and Drop a PR! 32 | 33 | ## Contributing 34 | ### Testing 35 | Testing is vital to the mission of Netpalm. The goal of this project is to serve as the "glue" that binds many 36 | different mission critical systems. If you're going to trust Netpalm to access and even push configuration changes to 37 | vital infrastructure, you MUST be certain Netpalm is going to behave as-advertised. Rigorous testing is how we ensure 38 | the features we implement work as-designed both now, and after inevitable changes later on. 39 | 40 | In all cases all we seek to validate is Netpalms behavior itself. In the unlikely event Netmiko mishandles something 41 | for instance, then we'll rely on the maintainers of that project to identify and address it. 42 | 43 | Tests in Netpalm fall into 3 broad categories 44 | 45 | * Full integration tests. These begin with an HTTP request from your local system to a running Netpalm Instance, and 46 | from there reach out to some router, etc. Some of these depend on something like a GNS3 lab, others depend on publicly 47 | reachable devices from Cisco or other vendors. 48 | * These are "deep and narrow". They each test one exact set of conditions fully end-to-end. They are slow to run and 49 | comparatively difficult to write, as such these are the last line of defense. 50 | * Running this test suite with 30 or so tests selected takes 10 or so minutes. The intent is to run these as final 51 | validation before merges. 52 | 53 | * "Cisgo" integration tests. These also begin with an HTTP request to a Netpalm instance, but here netpalm should be 54 | deployed with access to a `cisgo` instance for much faster test runs. Right now only a hand full of tests use this model 55 | * These are still "deep and narrow" but because of the reduced runtime it's feasible to run them far more often. 56 | * accessed with `pytest -m cisgo` 57 | * current runtime on reference machine with 12 tests selected is ~40s 58 | 59 | * code-only unit tests. These tests validate core internal Netpalm logic and functionality. These are meant to be ran 60 | in-container and require have all the same requirements as the full netpalm execution environment. Everything listed in `requirements.txt` essentially. 61 | * accessed with `pytest -m nolab`. Or even `docker-compose -f .\docker-compose.dev.yml run --rm controller pytest -m nolab` 62 | * current runtime on reference machine with 31 tests selected is ~1.6s 63 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | code-exchange-repo-template 2 | 3 | Copyright (c) 2020, Cisco Systems, Inc. and/or its affiliates 4 | 5 | This project includes software developed at Cisco Systems, Inc. and/or its affiliates. 6 | -------------------------------------------------------------------------------- /config/config.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "__comment__": "Values given here will override the values given in 'defaults.json'", 3 | "api_key": "2a84465a-cf38-46b2-9d86-b84Q7d57f288", 4 | "redis_key": "Red1zp4ww0rd_", 5 | "device_whitelist": [], 6 | "redis_tls_enabled": true, 7 | "redis_tls_cert_file": "netpalm/backend/core/security/cert/tls/redis.crt", 8 | "redis_tls_key_file": "netpalm/backend/core/security/cert/tls/redis.key", 9 | "redis_tls_ca_cert_file": "netpalm/backend/core/security/cert/tls/ca.crt", 10 | 11 | "txtfsm_template_server": "http://textfsm.nornir.tech", 12 | "default_webhook_url": "https://9d4f355779c960d7509368ad5a7e3503.m.pipedream.net", 13 | 14 | "gunicorn_workers": 3, 15 | "pinned_process_per_node": 40, 16 | "fifo_process_per_node": 10, 17 | "apscheduler_num_processes": 1, 18 | "apscheduler_num_threads": 5, 19 | 20 | "redis_task_ttl": 500, 21 | "redis_task_timeout": 500, 22 | "redis_task_result_ttl": 500, 23 | "redis_cache_default_timeout": 300, 24 | "self_api_call_timeout": 15, 25 | 26 | "default_webhook_ssl_verify": true, 27 | "default_webhook_timeout": 5, 28 | "default_webhook_name": "default_webhook", 29 | "default_webhook_headers": { 30 | "Content-Type": "application/json" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /config/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "__comment__": "Values given here will override the values given in 'defaults.json'" 3 | } 4 | -------------------------------------------------------------------------------- /config/defaults.json: -------------------------------------------------------------------------------- 1 | { 2 | "api_key": "2a84465a-cf38-46b2-9d86-b84Q7d57f288", 3 | "netpalm_container_name": "netpalm-controller", 4 | "netpalm_callback_http_mode": "http", 5 | "api_key_name": "x-api-key", 6 | "cookie_domain": "netpalm.local", 7 | "listen_port": 9000, 8 | "listen_ip": "0.0.0.0", 9 | "gunicorn_workers": 3, 10 | "redis_task_ttl": 500, 11 | "redis_task_timeout": 500, 12 | "redis_task_result_ttl": 500, 13 | "redis_server": "redis", 14 | "redis_port": 6379, 15 | "redis_key": "Red1zp4ww0rd_", 16 | "redis_core_q": "process", 17 | "redis_fifo_q": "fifo", 18 | "redis_broadcast_q": "broadcast", 19 | "redis_queue_store": "netpalm_queue_store", 20 | "redis_pinned_store": "netpalm_pinned_store", 21 | "redis_schedule_store": "netpalm_schedule_store", 22 | "redis_schedule_store_stats": "netpalm_schedule_store_stats", 23 | "redis_cache_enabled": true, 24 | "redis_cache_default_timeout": 300, 25 | "redis_cache_key_prefix": "NETPALM_RESULT_CACHE", 26 | "redis_update_log": "netpalm_extensibles_update_log", 27 | "redis_tls_enabled": true, 28 | "redis_tls_cert_file": "netpalm/backend/core/security/cert/tls/redis.crt", 29 | "redis_tls_key_file": "netpalm/backend/core/security/cert/tls/redis.key", 30 | "redis_tls_ca_cert_file": "netpalm/backend/core/security/cert/tls/ca.crt", 31 | "redis_socket_connect_timeout": 30, 32 | "redis_socket_keepalive": 30, 33 | "pinned_process_per_node": 40, 34 | "fifo_process_per_node": 10, 35 | "txtfsm_index_file": "netpalm/backend/plugins/extensibles/ntc-templates/index", 36 | "txtfsm_template_server": "http://textfsm.nornir.tech", 37 | "custom_scripts": "netpalm/backend/plugins/extensibles/custom_scripts/", 38 | "jinja2_config_templates": "netpalm/backend/plugins/extensibles/j2_config_templates/", 39 | "python_service_templates": "netpalm/backend/plugins/extensibles/services/", 40 | "ttp_templates": "netpalm/backend/plugins/extensibles/ttp_templates/", 41 | "self_api_call_timeout": 15, 42 | "default_webhook_url": "https://9d4f355779c960d7509368ad5a7e3503.m.pipedream.net", 43 | "default_webhook_ssl_verify": true, 44 | "default_webhook_timeout": 5, 45 | "webhook_jinja2_templates": "netpalm/backend/plugins/extensibles/j2_webhook_templates/", 46 | "default_webhook_name": "default_webhook", 47 | "default_webhook_headers": { 48 | "Content-Type": "application/json" 49 | }, 50 | "custom_webhooks": "netpalm/backend/plugins/extensibles/custom_webhooks/", 51 | "drivers": "netpalm/backend/plugins/drivers/", 52 | "log_config_filename": "config/log-config.yml", 53 | "apscheduler_num_processes": 1, 54 | "apscheduler_num_threads": 5, 55 | "device_whitelist": [] 56 | } 57 | -------------------------------------------------------------------------------- /config/log-config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 1 3 | filters: 4 | secret_scrub: 5 | (): netpalm.backend.core.confload.confload.ScrubFilter 6 | 7 | formatters: 8 | simple: 9 | # format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' 10 | format: '[%(asctime)s:%(name)s:%(funcName)s:%(levelname)s] %(message)s' 11 | handlers: 12 | console: 13 | class: logging.StreamHandler 14 | level: DEBUG 15 | formatter: simple 16 | stream: ext://sys.stdout 17 | filters: [secret_scrub] 18 | loggers: 19 | netmiko: 20 | level: DEBUG 21 | paramiko: 22 | level: DEBUG 23 | # rq.worker: # setting these levels appears to have no effect :( 24 | # level: WARN 25 | # rq: 26 | # level: WARN 27 | netpalm_worker_common: 28 | level: DEBUG 29 | netpalm_worker_pinned: 30 | level: DEBUG 31 | netpalm_worker_fifo: 32 | level: DEBUG 33 | backend: 34 | level: DEBUG 35 | routers: 36 | level: DEBUG 37 | 38 | 39 | root: 40 | # level: DEBUG 41 | handlers: [console] 42 | 43 | disable_existing_loggers: False 44 | -------------------------------------------------------------------------------- /config/redis.conf: -------------------------------------------------------------------------------- 1 | protected-mode yes 2 | tcp-backlog 511 3 | timeout 0 4 | tcp-keepalive 300 5 | daemonize no 6 | supervised no 7 | pidfile /var/run/redis_6379.pid 8 | loglevel notice 9 | logfile "" 10 | databases 16 11 | always-show-logo yes 12 | save 900 1 13 | save 300 10 14 | save 60 10000 15 | stop-writes-on-bgsave-error yes 16 | rdbcompression yes 17 | rdbchecksum yes 18 | dbfilename dump.rdb 19 | rdb-del-sync-files no 20 | dir ./ 21 | replica-serve-stale-data yes 22 | replica-read-only yes 23 | repl-diskless-sync no 24 | repl-diskless-sync-delay 5 25 | repl-diskless-load disabled 26 | repl-disable-tcp-nodelay no 27 | replica-priority 100 28 | acllog-max-len 128 29 | lazyfree-lazy-eviction no 30 | lazyfree-lazy-expire no 31 | lazyfree-lazy-server-del no 32 | replica-lazy-flush no 33 | lazyfree-lazy-user-del no 34 | appendonly no 35 | appendfilename "appendonly.aof" 36 | appendfsync everysec 37 | no-appendfsync-on-rewrite no 38 | auto-aof-rewrite-percentage 100 39 | auto-aof-rewrite-min-size 64mb 40 | aof-load-truncated yes 41 | aof-use-rdb-preamble yes 42 | lua-time-limit 5000 43 | slowlog-log-slower-than 10000 44 | slowlog-max-len 128 45 | latency-monitor-threshold 0 46 | notify-keyspace-events "" 47 | hash-max-ziplist-entries 512 48 | hash-max-ziplist-value 64 49 | list-max-ziplist-size -2 50 | list-compress-depth 0 51 | set-max-intset-entries 512 52 | zset-max-ziplist-entries 128 53 | zset-max-ziplist-value 64 54 | hll-sparse-max-bytes 3000 55 | stream-node-max-bytes 4096 56 | stream-node-max-entries 100 57 | activerehashing yes 58 | client-output-buffer-limit normal 0 0 0 59 | client-output-buffer-limit replica 256mb 64mb 60 60 | client-output-buffer-limit pubsub 32mb 8mb 60 61 | hz 10 62 | dynamic-hz yes 63 | aof-rewrite-incremental-fsync yes 64 | rdb-save-incremental-fsync yes 65 | jemalloc-bg-thread yes 66 | 67 | #netpalm TLS ENABLED CONFIG 68 | bind redis 69 | port 0 70 | tls-port 6379 71 | tls-cert-file /etc/redis_certs/tls/redis.crt 72 | tls-key-file /etc/redis_certs/tls/redis.key 73 | tls-ca-cert-file /etc/redis_certs/tls/ca.crt 74 | tls-ca-cert-dir /etc/redis_certs 75 | tls-protocols "TLSv1.2 TLSv1.3" 76 | requirepass Red1zp4ww0rd_ 77 | 78 | ## netpalm non-TLS ENABLED CONFIG 79 | ## uncomment if you are cool and just want to go fast 80 | # port 6379 81 | # requirepass Red1zp4ww0rd_ -------------------------------------------------------------------------------- /docker-compose.ci.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | services: 4 | 5 | netpalm-controller: 6 | build: 7 | context: . 8 | dockerfile: ./dockerfiles/netpalm_controller_dockerfile 9 | environment: 10 | - NET_TEXTFSM=/usr/local/lib/python3.8/site-packages/ntc_templates/templates/ 11 | - NETPALM_CONFIG=/code/config/config.json 12 | - NETPALM_LOG_CONFIG_FILENAME=/code/config/log-config.yml 13 | ports: 14 | - "9000:9000" 15 | networks: 16 | - "netpalm-network" 17 | depends_on: 18 | - redis 19 | - cisgo 20 | 21 | worker-pinned: 22 | image: netpalm_netpalm-controller 23 | command: python3 worker.py pinned 24 | environment: 25 | - NET_TEXTFSM=/usr/local/lib/python3.8/site-packages/ntc_templates/templates/ 26 | - NETPALM_CONFIG=/code/config/config.json 27 | - NETPALM_LOG_CONFIG_FILENAME=/code/config/log-config.yml 28 | depends_on: 29 | - redis 30 | networks: 31 | - "netpalm-network" 32 | # deploy: 33 | # replicas: 2 34 | 35 | worker-fifo: 36 | image: netpalm_netpalm-controller 37 | command: python3 worker.py fifo 38 | environment: 39 | - NET_TEXTFSM=/usr/local/lib/python3.8/site-packages/ntc_templates/templates/ 40 | - NETPALM_CONFIG=/code/config/config.json 41 | - NETPALM_LOG_CONFIG_FILENAME=/code/config/log-config.yml 42 | depends_on: 43 | - redis 44 | networks: 45 | - "netpalm-network" 46 | 47 | redis: 48 | build: 49 | context: . 50 | dockerfile: ./dockerfiles/netpalm_redis_dockerfile 51 | networks: 52 | - "netpalm-network" 53 | 54 | cisgo: 55 | image: apcela/cisshgo:v0.1.1 56 | ports: 57 | - "10005:10005" # one port just for convenience in case you need to ssh from outside for some reason 58 | networks: 59 | - "netpalm-network" 60 | 61 | networks: 62 | 63 | netpalm-network: 64 | name: "netpalm-network" 65 | -------------------------------------------------------------------------------- /docker-compose.dev.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | services: 4 | 5 | controller: 6 | build: 7 | context: . 8 | dockerfile: ./dockerfiles/netpalm_controller_dockerfile 9 | 10 | command: gunicorn -c gunicorn.conf.py netpalm.netpalm_controller:app 11 | environment: 12 | - NET_TEXTFSM=/usr/local/lib/python3.8/site-packages/ntc_templates/templates/ 13 | - NETPALM_CONFIG=/code/config/config.json 14 | - NETPALM_LOG_CONFIG_FILENAME=/code/config/log-config.yml 15 | # volumes: 16 | # - .:/code 17 | ports: 18 | - "9000:9000" 19 | networks: 20 | - "netpalm-network" 21 | depends_on: 22 | - redis 23 | - cisgo 24 | 25 | second-ctrlr: # using --scale doesn't work on controllers because they all want to own port 9000 26 | image: netpalm_controller 27 | command: gunicorn -c gunicorn.conf.py netpalm.netpalm_controller:app 28 | environment: 29 | - NET_TEXTFSM=/usr/local/lib/python3.8/site-packages/ntc_templates/templates/ 30 | - NETPALM_CONFIG=/code/config/config.json 31 | - NETPALM_LOG_CONFIG_FILENAME=/code/config/log-config.yml 32 | # volumes: 33 | # - .:/code 34 | ports: 35 | - "9001:9000" 36 | networks: 37 | - "netpalm-network" 38 | depends_on: 39 | - redis 40 | 41 | worker-pinned: 42 | image: netpalm_controller 43 | command: python3 worker.py pinned 44 | environment: 45 | - NET_TEXTFSM=/usr/local/lib/python3.8/site-packages/ntc_templates/templates/ 46 | - NETPALM_CONFIG=/code/config/config.json 47 | - NETPALM_LOG_CONFIG_FILENAME=/code/config/log-config.yml 48 | depends_on: 49 | - redis 50 | networks: 51 | - "netpalm-network" 52 | 53 | worker-fifo: 54 | image: netpalm_controller 55 | command: python3 worker.py fifo 56 | environment: 57 | - NET_TEXTFSM=/usr/local/lib/python3.8/site-packages/ntc_templates/templates/ 58 | - NETPALM_CONFIG=/code/config/config.json 59 | - NETPALM_LOG_CONFIG_FILENAME=/code/config/log-config.yml 60 | depends_on: 61 | - redis 62 | networks: 63 | - "netpalm-network" 64 | 65 | redis: 66 | build: 67 | context: . 68 | dockerfile: ./dockerfiles/netpalm_redis_dockerfile 69 | networks: 70 | - "netpalm-network" 71 | restart: always 72 | 73 | cisgo: 74 | image: apcela/cisshgo:v0.1.0 75 | # dockerfile: Dockerfile 76 | ports: 77 | - "10005:10005" # one port just for convenience in case you need to ssh from outside for some reason 78 | networks: 79 | - "netpalm-network" 80 | 81 | networks: 82 | netpalm-network: 83 | name: "netpalm-network" 84 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | services: 4 | 5 | netpalm-controller: 6 | build: 7 | context: . 8 | dockerfile: ./dockerfiles/netpalm_controller_dockerfile 9 | environment: 10 | - NET_TEXTFSM=/usr/local/lib/python3.8/site-packages/ntc_templates/templates/ 11 | - NETPALM_CONFIG=/code/config/config.json 12 | - NETPALM_LOG_CONFIG_FILENAME=/code/config/log-config.yml 13 | ports: 14 | - "9000:9000" 15 | networks: 16 | - "netpalm-network" 17 | depends_on: 18 | - redis 19 | restart: always 20 | 21 | netpalm-worker-pinned: 22 | build: 23 | context: . 24 | dockerfile: ./dockerfiles/netpalm_pinned_worker_dockerfile 25 | environment: 26 | - NET_TEXTFSM=/usr/local/lib/python3.8/site-packages/ntc_templates/templates/ 27 | - NETPALM_CONFIG=/code/config/config.json 28 | - NETPALM_LOG_CONFIG_FILENAME=/code/config/log-config.yml 29 | depends_on: 30 | - redis 31 | networks: 32 | - "netpalm-network" 33 | restart: always 34 | 35 | netpalm-worker-fifo: 36 | build: 37 | context: . 38 | dockerfile: ./dockerfiles/netpalm_fifo_worker_dockerfile 39 | environment: 40 | - NET_TEXTFSM=/usr/local/lib/python3.8/site-packages/ntc_templates/templates/ 41 | - NETPALM_CONFIG=/code/config/config.json 42 | - NETPALM_LOG_CONFIG_FILENAME=/code/config/log-config.yml 43 | depends_on: 44 | - redis 45 | networks: 46 | - "netpalm-network" 47 | restart: always 48 | 49 | redis: 50 | build: 51 | context: . 52 | dockerfile: ./dockerfiles/netpalm_redis_dockerfile 53 | networks: 54 | - "netpalm-network" 55 | restart: always 56 | 57 | networks: 58 | 59 | netpalm-network: 60 | name: "netpalm-network" 61 | -------------------------------------------------------------------------------- /dockerfiles/dev_dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim 2 | WORKDIR /usr/local/lib/python3.8/site-packages 3 | RUN apt-get update \ 4 | && apt-get install -y git \ 5 | && git clone https://github.com/networktocode/ntc-templates.git \ 6 | && mv ntc-templates ntc_templates \ 7 | && pip3 install --upgrade pip 8 | 9 | ADD netpalm/requirements.txt /code/ 10 | RUN pip3 install -r /code/requirements.txt 11 | -------------------------------------------------------------------------------- /dockerfiles/netpalm_controller_dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim 2 | WORKDIR /usr/local/lib/python3.8/site-packages 3 | RUN apt-get update \ 4 | && apt-get install -y git \ 5 | && git clone https://github.com/networktocode/ntc-templates.git \ 6 | && mv ntc-templates ntc_templates \ 7 | && pip3 install --upgrade pip 8 | 9 | ADD netpalm/requirements.txt /code/ 10 | RUN pip3 install -r /code/requirements.txt 11 | 12 | ADD netpalm/controller_addtl_requirements.txt /code/ 13 | RUN pip3 install -r /code/controller_addtl_requirements.txt 14 | 15 | ADD . /code 16 | WORKDIR /code 17 | 18 | CMD gunicorn -p controller.pid -c gunicorn.conf.py netpalm.netpalm_controller:app -------------------------------------------------------------------------------- /dockerfiles/netpalm_fifo_worker_dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim 2 | WORKDIR /usr/local/lib/python3.8/site-packages 3 | RUN apt-get update \ 4 | && apt-get install -y git \ 5 | && git clone https://github.com/networktocode/ntc-templates.git \ 6 | && mv ntc-templates ntc_templates \ 7 | && pip3 install --upgrade pip 8 | 9 | ADD netpalm/requirements.txt /code/ 10 | RUN pip3 install -r /code/requirements.txt 11 | 12 | ADD . /code 13 | WORKDIR /code 14 | STOPSIGNAL SIGINT 15 | CMD ["python3", "worker.py", "fifo"] 16 | -------------------------------------------------------------------------------- /dockerfiles/netpalm_pinned_worker_dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim 2 | WORKDIR /usr/local/lib/python3.8/site-packages 3 | RUN apt-get update \ 4 | && apt-get install -y git \ 5 | && git clone https://github.com/networktocode/ntc-templates.git \ 6 | && mv ntc-templates ntc_templates \ 7 | && pip3 install --upgrade pip 8 | 9 | ADD netpalm/requirements.txt /code/ 10 | RUN pip3 install -r /code/requirements.txt 11 | 12 | ADD . /code 13 | WORKDIR /code 14 | STOPSIGNAL SIGINT 15 | CMD ["python3", "worker.py", "pinned"] 16 | -------------------------------------------------------------------------------- /dockerfiles/netpalm_redis_dockerfile: -------------------------------------------------------------------------------- 1 | FROM redis:6.0.7-alpine 2 | WORKDIR /etc/redis_certs 3 | ADD ./netpalm/backend/core/security/cert /etc/redis_certs 4 | COPY ./config/redis.conf /usr/local/etc/redis/redis.conf 5 | CMD [ "redis-server", "/usr/local/etc/redis/redis.conf" ] -------------------------------------------------------------------------------- /gen_service_package.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | if __name__ == "__main__": 5 | parser = argparse.ArgumentParser(description='netpalm service package generate') 6 | required_files = parser.add_argument_group('required arguments') 7 | required_files.add_argument('-n', '--name', help='service package name', required=True) 8 | required_files.add_argument('-o', '--output', help='python | base64', default="python", required=True) 9 | args = parser.parse_args() 10 | 11 | package_name = args.name.replace(" ", "_") 12 | 13 | if os.path.isdir(package_name) or os.path.isfile(package_name): 14 | print("Please use a different package name, this is in use already") 15 | exit() 16 | 17 | os.mkdir(package_name) 18 | 19 | 20 | 21 | example_service = """ 22 | """ 23 | 24 | with open(f'{package_name}.py', 'w') as fp: 25 | pass 26 | -------------------------------------------------------------------------------- /gunicorn.conf.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | log = logging.getLogger(__name__) 5 | DEFAULTS_FILENAME = "/code/config/defaults.json" 6 | CONFIG_FILENAME = "/code/config/config.json" 7 | 8 | 9 | def load_config_files(defaults_filename: str = DEFAULTS_FILENAME, config_filename: str = CONFIG_FILENAME) -> dict: 10 | data = {} 11 | 12 | for fname in (defaults_filename, config_filename): 13 | try: 14 | with open(fname) as infil: 15 | data.update(json.load(infil)) 16 | except FileNotFoundError: 17 | log.warning(f"Couldn't find {fname}") 18 | 19 | if not data: 20 | raise RuntimeError(f"Could not find either {defaults_filename} or {config_filename}") 21 | 22 | return data 23 | 24 | 25 | data = load_config_files() 26 | 27 | bind = data["listen_ip"] + ":" + str(data["listen_port"]) 28 | workers = data["gunicorn_workers"] 29 | timeout = 3 * 60 30 | keepalive = 24 * 60 * 60 31 | worker_class = "uvicorn.workers.UvicornWorker" 32 | threads = 45 33 | -------------------------------------------------------------------------------- /netpalm.postman_collection.json: -------------------------------------------------------------------------------- 1 | { 2 | "please_visit":"https://documenter.getpostman.com/view/2391814/T1DqgwcU?version=latest#3e64ddc7-3c13-413c-8c3b-217d82722d49" 3 | } -------------------------------------------------------------------------------- /netpalm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/calls/dryrun/dryrun.py: -------------------------------------------------------------------------------- 1 | from netpalm.backend.core.utilities.rediz_meta import ( 2 | render_netpalm_payload, 3 | write_mandatory_meta, 4 | ) 5 | from netpalm.backend.core.utilities.rediz_meta import write_meta_error 6 | from netpalm.backend.core.utilities.jinja2.j2 import render_j2template 7 | from netpalm.backend.core.utilities.webhook.webhook import exec_webhook_func 8 | 9 | from netpalm.backend.core.driver import driver_map 10 | 11 | 12 | def dryrun(**kwargs): 13 | lib = kwargs.get("library", False) 14 | config = kwargs.get("config", False) 15 | j2conf = kwargs.get("j2config", False) 16 | webhook = kwargs.get("webhook", False) 17 | enable_mode = kwargs.get("enable_mode", False) 18 | result = False 19 | 20 | try: 21 | write_mandatory_meta() 22 | 23 | if j2conf: 24 | j2confargs = j2conf.get("args") 25 | res = render_j2template( 26 | j2conf["template"], template_type="config", kwargs=j2confargs 27 | ) 28 | config = res["data"]["task_result"]["template_render_result"] 29 | 30 | if ( 31 | j2conf and config and lib == "ncclient" 32 | ): # move this into the driver in future 33 | if not kwargs.get("args", False): 34 | kwargs["args"] = {} 35 | kwargs["args"]["config"] = config 36 | 37 | result = {} 38 | 39 | if not driver_map.get(lib): 40 | raise NotImplementedError(f"unknown 'driver' {lib}") 41 | 42 | driver_obj = driver_map[lib](**kwargs) 43 | sesh = driver_obj.connect() 44 | 45 | if config and not enable_mode: 46 | result = driver_obj.dryrun(session=sesh, command=config, dry_run=True) 47 | if config and enable_mode: 48 | result = driver_obj.dryrun( 49 | session=sesh, command=config, dry_run=True, enable_mode=enable_mode 50 | ) 51 | 52 | driver_obj.logout(sesh) 53 | 54 | if webhook: 55 | current_jobdata = render_netpalm_payload(job_result=result) 56 | exec_webhook_func(jobdata=current_jobdata, webhook_payload=webhook) 57 | 58 | except Exception as e: 59 | write_meta_error(e) 60 | 61 | return result 62 | -------------------------------------------------------------------------------- /netpalm/backend/core/calls/getconfig/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/core/calls/getconfig/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/calls/getconfig/exec_command.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from netpalm.backend.core.utilities.rediz_meta import ( 4 | render_netpalm_payload, 5 | write_mandatory_meta, 6 | ) 7 | from netpalm.backend.core.utilities.rediz_meta import write_meta_error 8 | from netpalm.backend.core.utilities.webhook.webhook import exec_webhook_func 9 | from netpalm.exceptions import NetpalmCheckError 10 | 11 | from netpalm.backend.core.driver import driver_map 12 | 13 | log = logging.getLogger(__name__) 14 | 15 | 16 | def exec_command(**kwargs): 17 | """main function for executing getconfig commands to southbound drivers""" 18 | lib = kwargs.get("library", False) 19 | command = kwargs.get("command", False) 20 | webhook = kwargs.get("webhook", False) 21 | post_checks = kwargs.get("post_checks", False) 22 | 23 | result = False 24 | 25 | if type(command) == str: 26 | commandlst = [command] 27 | else: 28 | commandlst = command 29 | 30 | log.debug(f"driver_map: {driver_map}") 31 | 32 | if not driver_map.get(lib): 33 | raise NotImplementedError(f"unknown 'driver' {lib}") 34 | 35 | try: 36 | write_mandatory_meta() 37 | if not post_checks: 38 | result = {} 39 | 40 | driver_obj = driver_map[lib](**kwargs) 41 | sesh = driver_obj.connect() 42 | if commandlst: 43 | result = driver_obj.sendcommand(sesh, commandlst) 44 | else: 45 | result = driver_obj.sendcommand(sesh) 46 | driver_obj.logout(sesh) 47 | 48 | else: 49 | result = {} 50 | driver_obj = driver_map[lib](**kwargs) 51 | sesh = driver_obj.connect() 52 | if commandlst: 53 | result = driver_obj.sendcommand(sesh, commandlst) 54 | if post_checks: 55 | for postcheck in post_checks: 56 | command = postcheck["get_config_args"]["command"] 57 | post_check_result = driver_obj.sendcommand(sesh, [command]) 58 | for matchstr in postcheck["match_str"]: 59 | if postcheck["match_type"] == "include" and matchstr not in str( 60 | post_check_result 61 | ): 62 | raise NetpalmCheckError( 63 | f"PostCheck Failed: {matchstr} not found in {post_check_result}" 64 | ) 65 | if postcheck["match_type"] == "exclude" and matchstr in str( 66 | post_check_result 67 | ): 68 | raise NetpalmCheckError( 69 | f"PostCheck Failed: {matchstr} found in {post_check_result}" 70 | ) 71 | driver_obj.logout(sesh) 72 | 73 | if webhook: 74 | current_jobdata = render_netpalm_payload(job_result=result) 75 | exec_webhook_func(jobdata=current_jobdata, webhook_payload=webhook) 76 | except Exception as e: 77 | write_meta_error(e) 78 | 79 | return result 80 | -------------------------------------------------------------------------------- /netpalm/backend/core/calls/getconfig/ncclient_get.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from netpalm.backend.core.utilities.rediz_meta import write_meta_error, write_mandatory_meta 4 | from netpalm.backend.plugins.drivers.ncclient.ncclient_drvr import ncclien 5 | 6 | log = logging.getLogger(__name__) 7 | 8 | 9 | def ncclient_get(**kwargs): 10 | """main function for executing getconfig commands to southbound drivers""" 11 | lib = kwargs.get("library", False) 12 | 13 | result = False 14 | 15 | try: 16 | write_mandatory_meta() 17 | result = {} 18 | if lib == "ncclient": 19 | ncc = ncclien(**kwargs) 20 | sesh = ncc.connect() 21 | result = ncc.getmethod(sesh) 22 | ncc.logout(sesh) 23 | else: 24 | raise NotImplementedError(f"unknown 'library' parameter {lib}") 25 | except Exception as e: 26 | write_meta_error(e) 27 | 28 | return result 29 | -------------------------------------------------------------------------------- /netpalm/backend/core/calls/scriptrunner/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/core/calls/scriptrunner/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/calls/scriptrunner/script.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import inspect 3 | 4 | import logging 5 | 6 | from netpalm.backend.core.confload.confload import config 7 | from netpalm.backend.core.utilities.rediz_meta import ( 8 | render_netpalm_payload, 9 | write_mandatory_meta, 10 | ) 11 | from netpalm.backend.core.utilities.rediz_meta import write_meta_error 12 | from netpalm.backend.core.utilities.webhook.webhook import exec_webhook_func 13 | 14 | from netpalm.backend.core.models.models import Script, ScriptCustom 15 | 16 | log = logging.getLogger(__name__) 17 | 18 | 19 | def script_model_finder(script_name: str): 20 | log.debug(f"script_model_finder: locating model for {script_name}") 21 | model = Script 22 | model_defined = False 23 | model_mode = None 24 | # first check whether there is the legacy _model.py file against the script name 25 | try: 26 | model_name = f"{script_name}_model" 27 | template_model_path_raw = config.custom_scripts 28 | template_model_path = template_model_path_raw.replace("/", ".") + model_name 29 | module = importlib.import_module(template_model_path) 30 | model = getattr(module, model_name) 31 | model_defined = True 32 | model_mode = "legacy" 33 | except Exception as e: 34 | log.debug( 35 | f"script_model_finder: no legacy model found for {script_name} import error {e} attempting with newer model in file" 36 | ) 37 | model = Script 38 | pass 39 | 40 | # if this does not exist, check within the file to see if a model exists 41 | # clean this up at some point -_-' 42 | 43 | try: 44 | model_name = f"{script_name}" 45 | template_model_path_raw = config.custom_scripts 46 | template_model_path = template_model_path_raw.replace("/", ".") + model_name 47 | module = importlib.import_module(template_model_path) 48 | runscrp = getattr(module, "run") 49 | for item in inspect.getfullargspec(runscrp): 50 | if type(item) is dict: 51 | for key, value in item.items(): 52 | if issubclass(value, ScriptCustom): 53 | model = value 54 | model_defined = True 55 | model_mode = "new" 56 | except Exception as e: 57 | pass 58 | log.debug(f"script_model_finder: returning {model}") 59 | return model, model_defined, model_mode 60 | 61 | 62 | def script_kiddy(**kwargs): 63 | webhook = kwargs.get("webhook", False) 64 | result = False 65 | 66 | log.debug(f'script_kiddy: locating model for script {kwargs["script"]}') 67 | model = script_model_finder(kwargs["script"]) 68 | model_to_validate = model[0] 69 | model_is_defined = model[1] 70 | model_mode = model[2] 71 | log.debug( 72 | f"script_kiddy: model located is {model} and a user model was found is {model_is_defined}" 73 | ) 74 | 75 | try: 76 | write_mandatory_meta() 77 | 78 | # execute the script 79 | scrp_path = config.custom_scripts 80 | kwarg = kwargs 81 | arg = kwarg.get("args", False) 82 | script_name = kwarg.get("script", False) 83 | script_path_full_name = scrp_path.replace("/", ".") + script_name 84 | log.debug( 85 | f"script_kiddy: attempting to import script {script_path_full_name} for run" 86 | ) 87 | 88 | module = importlib.import_module(script_path_full_name) 89 | runscrp = getattr(module, "run") 90 | except Exception as e: 91 | log.error(f"script_kiddy: could not import {script_path_full_name} with {e}") 92 | write_meta_error(e) 93 | 94 | try: 95 | log.debug(f"script_kiddy: attempting to run script {script_path_full_name}") 96 | if not model_is_defined or model_mode is "legacy": 97 | result = runscrp(kwargs=arg) 98 | else: 99 | data_to_send = model_to_validate(**kwarg) 100 | result = runscrp(data_to_send) 101 | # if webhook used do that too 102 | if webhook: 103 | current_jobdata = render_netpalm_payload(job_result=result) 104 | exec_webhook_func(jobdata=current_jobdata, webhook_payload=webhook) 105 | except Exception as e: 106 | log.error( 107 | f"script_kiddy: could not run script {script_path_full_name} with {e}" 108 | ) 109 | write_meta_error(e) 110 | 111 | return result 112 | -------------------------------------------------------------------------------- /netpalm/backend/core/calls/service/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/core/calls/service/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/calls/service/netpalmservice.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from pydantic import BaseModel 4 | 5 | from netpalm.backend.core.confload.confload import config 6 | 7 | log = logging.getLogger(__name__) 8 | 9 | 10 | class NetpalmService: 11 | def __init__(self, model, service_id=None): 12 | log.info(f"netpalm service: invoking") 13 | self.model = model 14 | self.service_id = service_id 15 | 16 | def create(self): 17 | log.info(f"netpalm service: create method not implemented on your service") 18 | pass 19 | 20 | def update(self): 21 | log.info(f"netpalm service: update method not implemented on your service") 22 | pass 23 | 24 | def delete(self): 25 | log.info(f"netpalm service: delete method not implemented on your service") 26 | pass 27 | 28 | def re_deploy(self): 29 | log.info(f"netpalm service: re_deploy method not implemented on your service") 30 | pass 31 | 32 | def validate(self): 33 | log.info(f"netpalm service: validate method not implemented on your service") 34 | pass 35 | 36 | def health_check(self): 37 | log.info( 38 | f"netpalm service: health_check method not implemented on your service" 39 | ) 40 | pass 41 | -------------------------------------------------------------------------------- /netpalm/backend/core/calls/setconfig/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/core/calls/setconfig/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/calls/setconfig/exec_config.py: -------------------------------------------------------------------------------- 1 | from netpalm.backend.core.utilities.rediz_meta import ( 2 | render_netpalm_payload, 3 | write_mandatory_meta, 4 | write_meta_error, 5 | ) 6 | from netpalm.backend.core.utilities.jinja2.j2 import render_j2template 7 | from netpalm.backend.core.utilities.webhook.webhook import exec_webhook_func 8 | from netpalm.exceptions import NetpalmCheckError 9 | 10 | from netpalm.backend.core.driver import driver_map 11 | 12 | 13 | def exec_config(**kwargs): 14 | """main function for executing setconfig commands to southbound drivers""" 15 | lib = kwargs.get("library", False) 16 | config = kwargs.get("config", False) 17 | j2conf = kwargs.get("j2config", False) 18 | webhook = kwargs.get("webhook", False) 19 | pre_checks = kwargs.get("pre_checks", False) 20 | post_checks = kwargs.get("post_checks", False) 21 | enable_mode = kwargs.get("enable_mode", False) 22 | 23 | result = False 24 | pre_check_ok = True 25 | 26 | try: 27 | write_mandatory_meta() 28 | 29 | if j2conf: 30 | j2confargs = j2conf.get("args") 31 | res = render_j2template( 32 | j2conf["template"], template_type="config", kwargs=j2confargs 33 | ) 34 | config = res["data"]["task_result"]["template_render_result"] 35 | 36 | if ( 37 | j2conf and config and lib == "ncclient" 38 | ): # move this into the driver in future 39 | if not kwargs.get("args", False): 40 | kwargs["args"] = {} 41 | kwargs["args"]["config"] = config 42 | 43 | if not driver_map.get(lib): 44 | raise NotImplementedError(f"unknown 'driver' {lib}") 45 | 46 | if not pre_checks and not post_checks: 47 | driver_obj = driver_map[lib](**kwargs) 48 | sesh = driver_obj.connect() 49 | if enable_mode: 50 | result = driver_obj.config(sesh, config, enable_mode) 51 | else: 52 | result = driver_obj.config(sesh, config) 53 | driver_obj.logout(sesh) 54 | 55 | else: 56 | driver_obj = driver_map[lib](**kwargs) 57 | sesh = driver_obj.connect() 58 | if pre_checks: 59 | for precheck in pre_checks: 60 | command = precheck["get_config_args"]["command"] 61 | pre_check_result = driver_obj.sendcommand(sesh, [command]) 62 | for matchstr in precheck["match_str"]: 63 | if precheck["match_type"] == "include" and matchstr not in str( 64 | pre_check_result 65 | ): 66 | raise NetpalmCheckError( 67 | f"PreCheck Failed: {matchstr} not found in {pre_check_result}" 68 | ) 69 | if precheck["match_type"] == "exclude" and matchstr in str( 70 | pre_check_result 71 | ): 72 | raise NetpalmCheckError( 73 | f"PreCheck Failed: {matchstr} found in {pre_check_result}" 74 | ) 75 | 76 | if pre_check_ok: 77 | result = driver_obj.config(sesh, config, enable_mode) 78 | if post_checks: 79 | for postcheck in post_checks: 80 | command = postcheck["get_config_args"]["command"] 81 | post_check_result = driver_obj.sendcommand(sesh, [command]) 82 | for matchstr in postcheck["match_str"]: 83 | if postcheck[ 84 | "match_type" 85 | ] == "include" and matchstr not in str(post_check_result): 86 | raise NetpalmCheckError( 87 | f"PostCheck Failed: {matchstr} not found in {post_check_result}" 88 | ) 89 | if postcheck["match_type"] == "exclude" and matchstr in str( 90 | post_check_result 91 | ): 92 | raise NetpalmCheckError( 93 | f"PostCheck Failed: {matchstr} found in {post_check_result}" 94 | ) 95 | driver_obj.logout(sesh) 96 | 97 | if webhook: 98 | current_jobdata = render_netpalm_payload(job_result=result) 99 | exec_webhook_func(jobdata=current_jobdata, webhook_payload=webhook) 100 | 101 | except Exception as e: 102 | write_meta_error(e) 103 | 104 | return result 105 | -------------------------------------------------------------------------------- /netpalm/backend/core/confload/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/core/confload/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/driver/__init__.py: -------------------------------------------------------------------------------- 1 | from netpalm.backend.core.driver.driver_auto_loader import driver_auto_loader 2 | 3 | driver_map = driver_auto_loader() -------------------------------------------------------------------------------- /netpalm/backend/core/driver/driver_auto_loader.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | import importlib 5 | import os 6 | 7 | from netpalm.backend.core.driver.netpalm_driver import NetpalmDriver 8 | 9 | from netpalm.backend.core.confload.confload import config 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | def driver_auto_loader(): 15 | driver_map = {} 16 | driver_dir = config.drivers # config.drivers 17 | driver_dir_module_path = driver_dir.replace("/", ".") 18 | drivers = os.listdir(driver_dir) 19 | for driver in drivers: 20 | driver_files = os.listdir(f"{driver_dir}{driver}") 21 | for driver_file in driver_files: 22 | if driver_file.endswith(".py") and not driver_file.startswith("__"): 23 | driver_module = driver_file.replace(".py", "") 24 | driver_module = importlib.import_module( 25 | f"{driver_dir_module_path}{driver}.{driver_module}" 26 | ) 27 | for driver_class in driver_module.__dict__.values(): 28 | if type(driver_class) == type and issubclass( 29 | driver_class, NetpalmDriver 30 | ): 31 | if driver_class != NetpalmDriver: 32 | try: 33 | driver_map[driver_class.driver_name] = driver_class 34 | log.debug(f"loaded driver: {driver_class.driver_name}") 35 | except Exception as e: 36 | log.error(f"unable to load driver with error: {e}") 37 | return driver_map 38 | -------------------------------------------------------------------------------- /netpalm/backend/core/driver/netpalm_driver.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | log = logging.getLogger(__name__) 5 | 6 | 7 | class NetpalmDriver: 8 | """ NetPalmDriver is the base class for all NetPalm drivers. """ 9 | 10 | def __init__(self, **kwargs): 11 | log.info(f"netpalm service: invoking") 12 | self.driver_name = None 13 | 14 | def connect(self): 15 | """connect to the device""" 16 | raise NotImplementedError 17 | 18 | def sendcommand(self, session=False, command=False): 19 | """send a command to the device""" 20 | raise NotImplementedError 21 | 22 | def config(self, sesh, config): 23 | """send a config to the device""" 24 | raise NotImplementedError 25 | 26 | def logout(self, session=False): 27 | """logout of the device""" 28 | raise NotImplementedError -------------------------------------------------------------------------------- /netpalm/backend/core/manager/__init__.py: -------------------------------------------------------------------------------- 1 | from netpalm.backend.core.manager.netpalm_manager import NetpalmManager 2 | 3 | ntplm = NetpalmManager() -------------------------------------------------------------------------------- /netpalm/backend/core/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/core/models/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/models/napalm.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Optional, Any, List 3 | 4 | from pydantic import BaseModel 5 | 6 | from netpalm.backend.core.models.models import GenericPrePostCheck 7 | from netpalm.backend.core.models.models import J2Config, CacheConfig 8 | from netpalm.backend.core.models.models import QueueStrategy 9 | from netpalm.backend.core.models.models import Webhook 10 | 11 | 12 | class NapalmDeviceType(str, Enum): 13 | cisco_ios = "cisco_ios" 14 | cisco_xr = "cisco_xr" 15 | nxos = "nxos" 16 | cisco_nxos_ssh = "cisco_nxos_ssh" 17 | arista_eos = "arista_eos" 18 | juniper = "juniper" 19 | 20 | 21 | class NapalmConnectionOptionalArgs(BaseModel): 22 | fortios_vdom: Optional[str] = None 23 | port: Optional[int] = None 24 | config_lock: Optional[bool] = None 25 | dest_file_system: Optional[str] = None 26 | auto_rollback_on_error: Optional[bool] = None 27 | global_delay_factor: Optional[int] = None 28 | nxos_protocol: Optional[str] = None 29 | 30 | 31 | class NapalmConnectionArgs(BaseModel): 32 | device_type: NapalmDeviceType 33 | optional_args: Optional[NapalmConnectionOptionalArgs] = None 34 | host: str 35 | username: str 36 | password: str 37 | 38 | 39 | class NapalmGetConfig(BaseModel): 40 | connection_args: NapalmConnectionArgs 41 | command: Any 42 | webhook: Optional[Webhook] = None 43 | queue_strategy: Optional[QueueStrategy] = None 44 | post_checks: Optional[List[GenericPrePostCheck]] = None 45 | cache: Optional[CacheConfig] = {} 46 | 47 | class Config: 48 | schema_extra = { 49 | "example": { 50 | "library": "napalm", 51 | "connection_args": { 52 | "device_type": "cisco_ios", "host": "10.0.2.23", "username": "admin", "password": "admin" 53 | }, 54 | "command": "get_facts", 55 | "queue_strategy": "fifo", 56 | "cache": { 57 | "enabled": True, 58 | "ttl": 300, 59 | "poison": False 60 | } 61 | } 62 | } 63 | 64 | 65 | class NapalmSetConfig(BaseModel): 66 | connection_args: NapalmConnectionArgs 67 | config: Optional[Any] = None 68 | j2config: Optional[J2Config] = None 69 | webhook: Optional[Webhook] = None 70 | queue_strategy: Optional[QueueStrategy] = None 71 | pre_checks: Optional[List[GenericPrePostCheck]] = None 72 | post_checks: Optional[List[GenericPrePostCheck]] = None 73 | 74 | class Config: 75 | schema_extra = { 76 | "example": { 77 | "library": "napalm", 78 | "connection_args": { 79 | "device_type": "cisco_ios", "host": "10.0.2.33", "username": "admin", "password": "admin" 80 | }, 81 | "config": "hostnam cat", 82 | "queue_strategy": "fifo" 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /netpalm/backend/core/models/netmiko.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Any, List 2 | 3 | from pydantic import BaseModel 4 | 5 | from netpalm.backend.core.models.models import GenericPrePostCheck 6 | from netpalm.backend.core.models.models import J2Config, CacheConfig 7 | from netpalm.backend.core.models.models import QueueStrategy 8 | from netpalm.backend.core.models.models import Webhook 9 | 10 | 11 | class NetmikoSendConfigArgs(BaseModel): 12 | command_string: Optional[str] = None 13 | expect_string: Optional[str] = None 14 | delay_factor: Optional[int] = None 15 | commit_label: Optional[str] = None 16 | max_loops: Optional[int] = None 17 | auto_find_prompt: Optional[bool] = None 18 | strip_prompt: Optional[bool] = None 19 | strip_command: Optional[bool] = None 20 | normalize: Optional[bool] = None 21 | use_textfsm: Optional[bool] = None 22 | textfsm_template: Optional[str] = None 23 | use_ttp: Optional[bool] = None 24 | ttp_template: Optional[str] = None 25 | use_genie: Optional[bool] = None 26 | cmd_verify: Optional[bool] = None 27 | 28 | 29 | class NetmikoConnectionArgs(BaseModel): 30 | ip: Optional[str] = None 31 | host: Optional[str] = None 32 | username: str 33 | password: str 34 | secret: Optional[str] = None 35 | port: Optional[int] = 22 36 | device_type: str 37 | verbose: Optional[bool] = None 38 | global_delay_factor: Optional[int] = 1 39 | global_cmd_verify: Optional[bool] = None 40 | use_keys: Optional[bool] = None 41 | key_file: Optional[str] = None 42 | pkey: Optional[str] = None 43 | passphrase: Optional[str] = None 44 | allow_agent: Optional[bool] = False 45 | ssh_strict: Optional[bool] = None 46 | system_host_keys: Optional[bool] = False 47 | alt_host_keys: Optional[bool] = False 48 | alt_key_file: Optional[str] = "" 49 | ssh_config_file: Optional[str] = None 50 | timeout: Optional[int] = 100 51 | session_timeout: Optional[int] = None 52 | auth_timeout: Optional[float] = None 53 | blocking_timeout: Optional[int] = 20 54 | banner_timeout: Optional[int] = 15 55 | keepalive: Optional[int] = 0 56 | default_enter: Optional[str] = None 57 | response_return: Optional[str] = None 58 | serial_settings: Optional[str] = None 59 | fast_cli: Optional[bool] = False 60 | session_log: Optional[str] = None 61 | session_log_record_writes = False 62 | session_log_file_mode: Optional[str] = "write" 63 | allow_auto_change: Optional[bool] = False 64 | encoding: Optional[str] = "ascii" 65 | sock: Optional[bool] = None 66 | auto_connect: Optional[bool] = True 67 | 68 | 69 | class NetmikoGetConfig(BaseModel): 70 | connection_args: NetmikoConnectionArgs 71 | command: Any 72 | args: Optional[NetmikoSendConfigArgs] = None 73 | webhook: Optional[Webhook] = None 74 | queue_strategy: Optional[QueueStrategy] = None 75 | post_checks: Optional[List[GenericPrePostCheck]] = None 76 | cache: Optional[CacheConfig] = {} 77 | ttl: Optional[int] = None 78 | enable_mode: Optional[bool] = False 79 | 80 | class Config: 81 | schema_extra = { 82 | "example": { 83 | "library": "netmiko", 84 | "connection_args": { 85 | "device_type": "cisco_ios", 86 | "host": "10.0.2.33", 87 | "username": "admin", 88 | "password": "admin" 89 | }, 90 | "command": "show ip int brief", 91 | "args": { 92 | "use_textfsm": True 93 | }, 94 | "queue_strategy": "fifo", 95 | "cache": { 96 | "enabled": True, 97 | "ttl": 300, 98 | "poison": False 99 | } 100 | } 101 | } 102 | 103 | 104 | class NetmikoSetConfig(BaseModel): 105 | connection_args: dict 106 | config: Optional[Any] = None 107 | args: Optional[NetmikoSendConfigArgs] = {} 108 | j2config: Optional[J2Config] = None 109 | webhook: Optional[Webhook] = None 110 | queue_strategy: Optional[QueueStrategy] = None 111 | pre_checks: Optional[List[GenericPrePostCheck]] = None 112 | post_checks: Optional[List[GenericPrePostCheck]] = None 113 | enable_mode: Optional[bool] = False 114 | ttl: Optional[int] = None 115 | 116 | class Config: 117 | schema_extra = { 118 | "example": { 119 | "library": "netmiko", 120 | "connection_args": { 121 | "device_type": "cisco_ios", 122 | "host": "10.0.2.33", 123 | "username": "admin", 124 | "password": "admin" 125 | }, 126 | "config": ["hostname cat"], 127 | "queue_strategy": "pinned" 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /netpalm/backend/core/models/puresnmp.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Any 2 | from enum import Enum 3 | 4 | from pydantic import BaseModel 5 | 6 | from netpalm.backend.core.models.models import CacheConfig 7 | from netpalm.backend.core.models.models import QueueStrategy 8 | from netpalm.backend.core.models.models import Webhook 9 | 10 | 11 | class PureSNMPConnectionArgs(BaseModel): 12 | host: str 13 | community: str 14 | port: Optional[int] = None 15 | timeout: Optional[int] = None 16 | 17 | 18 | class SNMPtypes(str, Enum): 19 | table = "table" 20 | get = "get" 21 | walk = "walk" 22 | # bulkget = "bulkget" 23 | # bulkwalk = "bulkwalk" 24 | 25 | 26 | class PureSNMPArgs(BaseModel): 27 | type: SNMPtypes 28 | 29 | 30 | class PureSNMPGetConfig(BaseModel): 31 | connection_args: PureSNMPConnectionArgs 32 | command: Any 33 | args: PureSNMPArgs 34 | webhook: Optional[Webhook] = None 35 | queue_strategy: Optional[QueueStrategy] = None 36 | cache: Optional[CacheConfig] = {} 37 | ttl: Optional[int] = None 38 | 39 | class Config: 40 | schema_extra = { 41 | "example": { 42 | "library": "puresnmp", 43 | "connection_args": { 44 | "host": "10.0.2.33", 45 | "community": "test", 46 | "port": 161, 47 | "timeout": 2 48 | }, 49 | "command": [".1.3.6.1.4.1.9.2.1.58.0","1.3.6.1.2.1.1.2.0", "1.3.6.1.2.1.1.3.0"], 50 | "queue_strategy": "fifo" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /netpalm/backend/core/models/restconf.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Optional 3 | 4 | from pydantic import BaseModel 5 | 6 | from netpalm.backend.core.models.models import CacheConfig 7 | from netpalm.backend.core.models.models import QueueStrategy 8 | from netpalm.backend.core.models.models import Webhook 9 | 10 | 11 | class SupportedOptions(str, Enum): 12 | get = "get" 13 | post = "post" 14 | patch = "patch" 15 | put = "put" 16 | delete = "delete" 17 | 18 | 19 | class RestconfConnectionArgs(BaseModel): 20 | host: str 21 | username: str 22 | password: str 23 | port: int 24 | verify: bool 25 | transport: str 26 | headers: dict 27 | 28 | 29 | class RestconfPayload(BaseModel): 30 | uri: str 31 | action: SupportedOptions 32 | payload: Optional[dict] = None 33 | 34 | 35 | class Restconf(BaseModel): 36 | connection_args: RestconfConnectionArgs 37 | args: RestconfPayload 38 | webhook: Optional[Webhook] = None 39 | queue_strategy: Optional[QueueStrategy] = None 40 | cache: Optional[CacheConfig] = {} 41 | ttl: Optional[int] = None 42 | 43 | class Config: 44 | schema_extra = { 45 | "example": { 46 | "library": "restconf", 47 | "connection_args": { 48 | "host": "ios-xe-mgmt-latest.cisco.com", "port": 9443, "username": "developer", 49 | "password": "C1sco12345", "verify": False, "timeout": 10, "transport": "https", "headers": { 50 | "Content-Type": "application/yang-data+json", "Accept": "application/yang-data+json" 51 | } 52 | }, 53 | "args": { 54 | "uri": "/restconf/data/Cisco-IOS-XE-native:native/interface/", 55 | "action": "post", 56 | "payload": { 57 | "Cisco-IOS-XE-native:BDI": { 58 | "name": "4001", 59 | "description": "netpalm" 60 | } 61 | } 62 | }, 63 | "queue_strategy": "fifo", 64 | "cache": { 65 | "enabled": True, 66 | "ttl": 300, 67 | "poison": False 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /netpalm/backend/core/models/service.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Optional, List, Any 3 | 4 | from pydantic import BaseModel 5 | 6 | from netpalm.backend.core.models.models import QueueStrategy 7 | 8 | 9 | # now redundant 10 | class ServiceLifecycle(str, Enum): 11 | create = "create" 12 | retrieve = "retrieve" 13 | delete = "delete" 14 | validate = "validate" 15 | script = "script" 16 | 17 | 18 | class ServiceInstanceState(str, Enum): 19 | deployed = "deployed" 20 | errored = "errored" 21 | deploying = "deploying" 22 | 23 | 24 | class ServiceMeta(BaseModel): 25 | service_model: str 26 | created_at: str 27 | updated_at: Optional[str] = None 28 | service_id: str 29 | service_state: Optional[ServiceInstanceState] = None 30 | 31 | 32 | class ServiceInstanceData(BaseModel): 33 | service_meta: ServiceMeta 34 | service_data: Any 35 | 36 | 37 | # now redundant 38 | class ServiceModel(BaseModel): 39 | operation: ServiceLifecycle 40 | args: dict 41 | queue_strategy: Optional[QueueStrategy] = None 42 | ttl: Optional[int] = None 43 | 44 | class Config: 45 | schema_extra = { 46 | "example": { 47 | "operation": "retrieve", 48 | "args": { 49 | "your_payload_goes": "here" 50 | }, 51 | "queue_strategy": "fifo" 52 | } 53 | } 54 | 55 | # now redundant 56 | class ServiceModelMethods(BaseModel): 57 | operation: ServiceLifecycle 58 | path: Optional[str] = None 59 | payload: dict 60 | 61 | # now redundant 62 | class ServiceModelSupportedMethods(BaseModel): 63 | supported_methods: List[ServiceModelMethods] = None 64 | 65 | # now redundant 66 | class ServiceModelTemplate(BaseModel): 67 | __root__: List[ServiceModelSupportedMethods] 68 | 69 | 70 | class ServiceInventorySchema(BaseModel): 71 | service_meta: dict 72 | 73 | 74 | class ServiceInventoryResponse(BaseModel): 75 | __root__: List[ServiceInventorySchema] -------------------------------------------------------------------------------- /netpalm/backend/core/models/task.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Optional, Any, List, Union, Dict 3 | 4 | from pydantic import BaseModel 5 | 6 | 7 | class TaskResponseEnum(str, Enum): 8 | success = "success" 9 | error = "error" 10 | 11 | 12 | class TaskStatusEnum(str, Enum): 13 | queued = "queued" 14 | finished = "finished" 15 | failed = "failed" 16 | started = "started" 17 | deferred = "deferred" 18 | scheduled = "scheduled" 19 | 20 | 21 | class TaskMetaData(BaseModel): 22 | enqueued_at: Optional[str] 23 | started_at: Optional[str] 24 | ended_at: Optional[str] 25 | enqueued_elapsed_seconds: Optional[str] 26 | total_elapsed_seconds: Optional[str] 27 | assigned_worker: Optional[str] 28 | 29 | class TaskError(BaseModel): 30 | exception_class: str 31 | exception_args: List[str] 32 | 33 | 34 | TaskErrorList = List[Union[str, TaskError]] 35 | 36 | class ServiceTaskHostError(BaseModel): 37 | task_id: str 38 | task_errors: TaskErrorList 39 | 40 | 41 | ServiceTaskErrors = List[Dict[str, ServiceTaskHostError]] 42 | 43 | 44 | class TaskResponse(BaseModel): 45 | task_id: str 46 | created_on: str 47 | task_queue: str 48 | task_meta: Optional[TaskMetaData] = None 49 | task_status: TaskStatusEnum 50 | task_result: Any 51 | task_errors: Union[TaskErrorList, ServiceTaskErrors] # Needed to get service tasks to validate when they're polled from /task/:taskid 52 | 53 | 54 | class Response(BaseModel): 55 | status: TaskResponseEnum 56 | data: TaskResponse 57 | 58 | class Config: 59 | schema_extra = { 60 | "example": { 61 | "status": "success", 62 | "data": { 63 | "task_id": "b380cf2b-ba78-4aab-b157-9b87ebbe6bb3", 64 | "created_on": "2020-08-02 11:16:43.693850", 65 | "task_queue": "10.0.2.33", 66 | "task_meta": { 67 | "enqueued_at": "2020-08-02 11:16:43.693939", 68 | "started_at": "2020-08-02 11:17:32.503873", 69 | "ended_at": "2020-08-02 11:17:42.440347", 70 | "enqueued_elapsed_seconds": "35", 71 | "started_elapsed_seconds": None, 72 | "total_elapsed_seconds": "58" 73 | }, 74 | "task_status": "finished", 75 | "task_result": { 76 | "show run | i hostname": [ 77 | "hostname cat" 78 | ] 79 | }, 80 | "task_errors": [] 81 | } 82 | } 83 | } 84 | 85 | 86 | class ServiceTaskResponse(BaseModel): 87 | service_id: str 88 | task_id: str 89 | created_on: str 90 | task_queue: str 91 | task_meta: Optional[TaskMetaData] = None 92 | task_status: TaskStatusEnum 93 | task_result: Any 94 | task_errors: list 95 | 96 | 97 | class ServiceResponse(BaseModel): 98 | status: TaskResponseEnum 99 | data: ServiceTaskResponse 100 | 101 | class Config: 102 | schema_extra = { 103 | "example": { 104 | "status": "success", 105 | "data": { 106 | "task_id": "b380cf2b-ba78-4aab-b157-9b87ebbe6bb3", 107 | "created_on": "2020-08-02 11:16:43.693850", 108 | "task_queue": "10.0.2.33", 109 | "task_meta": { 110 | "enqueued_at": "2020-08-02 11:16:43.693939", 111 | "started_at": "2020-08-02 11:17:32.503873", 112 | "ended_at": "2020-08-02 11:17:42.440347", 113 | "enqueued_elapsed_seconds": "35", 114 | "started_elapsed_seconds": None, 115 | "total_elapsed_seconds": "58" 116 | }, 117 | "task_status": "finished", 118 | "task_result": { 119 | "show run | i hostname": [ 120 | "hostname cat" 121 | ] 122 | }, 123 | "task_errors": [] 124 | } 125 | } 126 | } 127 | 128 | 129 | class ResponseBasic(BaseModel): 130 | status: TaskResponseEnum 131 | data: dict 132 | 133 | 134 | class WorkerResponse(BaseModel): 135 | hostname: Optional[Any] = None 136 | pid: str 137 | name: Optional[Any] = None 138 | last_heartbeat: Optional[Any] = None 139 | birth_date: Optional[Any] = None 140 | successful_job_count: Optional[Any] = None 141 | failed_job_count: Optional[Any] = None 142 | total_working_time: Optional[Any] = None -------------------------------------------------------------------------------- /netpalm/backend/core/models/transaction_log.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Union, Literal 3 | 4 | from pydantic import BaseModel 5 | 6 | 7 | class TransactionLogEntryType(str, Enum): 8 | tfsm_pull = "TFSM_PULL" 9 | tfsm_delete = "TFSM_DELETE" 10 | tfsm_push = "TFSM_PUSH" 11 | unvrsl_tmp_push = "UNVRSL_PUSH" 12 | unvrsl_tmp_delete = "UNVRSL_DELETE" 13 | echo = "ECHO" 14 | init = "INITIALIZE" 15 | 16 | 17 | class EchoModel(BaseModel): 18 | msg: str 19 | 20 | 21 | class TFSMPullTemplateModel(BaseModel): 22 | key: str 23 | driver: str 24 | command: str 25 | 26 | 27 | class TFSMPushTemplateModel(BaseModel): 28 | driver: str 29 | command: str 30 | template_text: str 31 | 32 | 33 | class TFSMDeleteTemplateModel(BaseModel): 34 | fsm_template: str 35 | 36 | 37 | class UnivsersalTemplatePushModel(BaseModel): 38 | """general template ingest0r for handling base64 ingestion and writing""" 39 | route_type: str 40 | base64_payload: str 41 | name: str 42 | 43 | 44 | class UnivsersalTemplateRemoveModel(BaseModel): 45 | """general template remover """ 46 | route_type: str 47 | name: str = None 48 | 49 | 50 | class InitEntryModel(BaseModel): 51 | init: Literal[True] # only here to stop model from greedily matching literally any input 52 | 53 | 54 | extn_update_types = { 55 | TransactionLogEntryType.tfsm_pull: TFSMPullTemplateModel, 56 | TransactionLogEntryType.tfsm_delete: TFSMDeleteTemplateModel, 57 | TransactionLogEntryType.tfsm_push: TFSMPushTemplateModel, 58 | TransactionLogEntryType.unvrsl_tmp_push: UnivsersalTemplatePushModel, 59 | TransactionLogEntryType.unvrsl_tmp_delete: UnivsersalTemplateRemoveModel, 60 | TransactionLogEntryType.init: InitEntryModel, 61 | TransactionLogEntryType.echo: EchoModel 62 | } 63 | 64 | 65 | class TransactionLogEntryModel(BaseModel): 66 | seq: int 67 | type: TransactionLogEntryType 68 | data: Union[ 69 | TFSMPullTemplateModel, 70 | TFSMDeleteTemplateModel, 71 | TFSMPushTemplateModel, 72 | EchoModel, 73 | InitEntryModel, 74 | UnivsersalTemplatePushModel, 75 | UnivsersalTemplateRemoveModel 76 | ] 77 | -------------------------------------------------------------------------------- /netpalm/backend/core/redis/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/core/redis/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/routes/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/core/routes/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/routes/routes.py: -------------------------------------------------------------------------------- 1 | # load plugins 2 | from netpalm.backend.core.calls.dryrun.dryrun import dryrun 3 | from netpalm.backend.core.calls.getconfig.exec_command import exec_command 4 | from netpalm.backend.core.calls.getconfig.ncclient_get import ncclient_get 5 | from netpalm.backend.core.calls.scriptrunner.script import script_kiddy 6 | from netpalm.backend.core.calls.service.procedures import ( 7 | create, 8 | update, 9 | delete, 10 | re_deploy, 11 | validate, 12 | health_check, 13 | ) 14 | from netpalm.backend.core.calls.setconfig.exec_config import exec_config 15 | from netpalm.backend.core.utilities.jinja2.j2 import j2gettemplate 16 | from netpalm.backend.core.utilities.jinja2.j2 import render_j2template 17 | from netpalm.backend.core.utilities.ls.ls import list_files 18 | from netpalm.backend.core.utilities.textfsm.template import ( 19 | listtemplates, 20 | pushtemplate, 21 | addtemplate, 22 | removetemplate, 23 | gettemplate, 24 | ) 25 | 26 | routes = { 27 | "getconfig": exec_command, 28 | "setconfig": exec_config, 29 | "listtemplates": listtemplates, 30 | "gettemplate": gettemplate, # replace with universal template mgr get_template in future 31 | "addtemplate": addtemplate, 32 | "pushtemplate": pushtemplate, 33 | "removetemplate": removetemplate, 34 | "ls": list_files, 35 | "script": script_kiddy, 36 | "j2gettemplate": j2gettemplate, 37 | "render_j2template": render_j2template, 38 | "dryrun": dryrun, 39 | "ncclient_get": ncclient_get, 40 | "service_create": create, 41 | "service_update": update, 42 | "service_delete": delete, 43 | "service_re_deploy": re_deploy, 44 | "service_validate": validate, 45 | "service_health_check": health_check, 46 | } 47 | -------------------------------------------------------------------------------- /netpalm/backend/core/schedule/__init__.py: -------------------------------------------------------------------------------- 1 | from netpalm.backend.core.schedule.schedule import Schedulr 2 | 3 | sched = Schedulr() 4 | schedule_r = sched.init_scheduler() 5 | -------------------------------------------------------------------------------- /netpalm/backend/core/security/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/core/security/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/security/cert/tls/ca.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFPTCCAyWgAwIBAgIJAJmgbenJU3cuMA0GCSqGSIb3DQEBCwUAMDUxEzARBgNV 3 | BAoMClJlZGlzIFRlc3QxHjAcBgNVBAMMFUNlcnRpZmljYXRlIEF1dGhvcml0eTAe 4 | Fw0yMzEyMDQxNTA5NDhaFw0zMzEyMDExNTA5NDhaMDUxEzARBgNVBAoMClJlZGlz 5 | IFRlc3QxHjAcBgNVBAMMFUNlcnRpZmljYXRlIEF1dGhvcml0eTCCAiIwDQYJKoZI 6 | hvcNAQEBBQADggIPADCCAgoCggIBANzDgsD+Glytgo1+nkb6Kge1McyHiHN7SfPl 7 | rlAOfxtopF/ENfpt5G0GoGf+YY+4f8iYB+fMgWYE8wDVatxOznkJQeVebu+6/rlr 8 | zNplyBwMCIsKUu49+hlnOf3X+sLAhmMcggbRUnTuCXUbEg7BRes8H7TzbmTyvAgP 9 | Idi7XsVSnnyeRvjQiWYbv4N8WJmcWLEvCzsiwGOFmkZuz1fbqrehDJTJ/Djgf6yT 10 | ZVYH8rd0MZai2KJMI7//sJUEY+cuFNP88hizn61Cw6DNR1eqOufktEHC0Q/yXK6g 11 | YSbtAgNjZ1kCG2nP1grpFTnmHeU1GvR3RrLv3oE4FNMs6TVuGwUUnV8hhIFnjuPP 12 | 6Lx9X0tYDIsYKVf1BmFSB75CydzWiBesgaGYmwt+OUbeUGxzjUQ/rjyiUl9djfl4 13 | gATGknxKiXO9h+XkS5Pja3PLe1xtFHxd8XwlYhoIAEHsL9I7EvREtPUFwFkRIFbt 14 | 6OFf15475ZlCHKdG1Br65mV4wty5beUrdLVoOOZEhOHLvBefvbj8VBXz57nGJuES 15 | cYvZ1CHUAwo+Z0Q3ilZMKDfE84v659ZtWbc9BYuUVRJhcUKSqzOPEL4rmVPePLx0 16 | ++7HVeJY2EFHxGJBJi2yQBrhdvxFglXidLTdgIvjkNm9h4S/gjwayM4P20Mo/wW9 17 | FZgVhKL7AgMBAAGjUDBOMB0GA1UdDgQWBBQ2N9LA6zvwTdTxiCtfz0d/qE/XdzAf 18 | BgNVHSMEGDAWgBQ2N9LA6zvwTdTxiCtfz0d/qE/XdzAMBgNVHRMEBTADAQH/MA0G 19 | CSqGSIb3DQEBCwUAA4ICAQAVMrQG0Zl2MEuRjHfNJOWT4xTA+5SCRZaRruAIr6Ge 20 | XaUn+bVwQjG0FDsb+XmDz0oc39yp2ZGG5IgFqNonYARnsUusiJLVAPcm830nOYrm 21 | 7zlqyY3Ax7rn1fUgI8pbGdrN/WyuuGFBjIo9iRaqW6cHP08pBenXg7VTXVAPdf3w 22 | IZpB+S/hzYLoJSwLafD2f/HGCtw2avA30UozuXTVri1e0fOVUcq3yC6/qBKkiesY 23 | tjrJdjsmlCM+h+abaKR1aVoEUCPAq03O0w1RTCxntCnU7i1fLmnsGfy8s3FcusN5 24 | pEqHaN5swnXVkC++YVm2iObzhag53mslCbhelxk7PdnbrOZmCdeCnWhpaPf97CtR 25 | 4aobqlsni6iIaKcac3DTUqTRU6NcYHOxdviaK+VZ9d2HZ5+SpqoPckYK0waA/XN9 26 | zfyFmpEWXa63kWqVJZdEH5lLWfVvKGucyDcf+tIsUzbCaArHh9HJEuUf4QiTF16i 27 | kEUMCHiCNMHlPIvWbo3+K8UDuZ+g3VcAgdV8C57J/f6nWgmIeow01KilYSrAFBm4 28 | E4qvLYAVzo2z2T03iyLPWucisdkDRk1YaP68oMx99ijtfjrdcDN5K1XCpV5uefu4 29 | 9r/fwNTjDJmL61N+qtnlwJJ2P6CaZY9tQ9BHrdqAezQpuAv53AS0FN/BtFrZXm6l 30 | AQ== 31 | -----END CERTIFICATE----- 32 | -------------------------------------------------------------------------------- /netpalm/backend/core/security/cert/tls/ca.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIJKgIBAAKCAgEA3MOCwP4aXK2CjX6eRvoqB7UxzIeIc3tJ8+WuUA5/G2ikX8Q1 3 | +m3kbQagZ/5hj7h/yJgH58yBZgTzANVq3E7OeQlB5V5u77r+uWvM2mXIHAwIiwpS 4 | 7j36GWc5/df6wsCGYxyCBtFSdO4JdRsSDsFF6zwftPNuZPK8CA8h2LtexVKefJ5G 5 | +NCJZhu/g3xYmZxYsS8LOyLAY4WaRm7PV9uqt6EMlMn8OOB/rJNlVgfyt3QxlqLY 6 | okwjv/+wlQRj5y4U0/zyGLOfrULDoM1HV6o65+S0QcLRD/JcrqBhJu0CA2NnWQIb 7 | ac/WCukVOeYd5TUa9HdGsu/egTgU0yzpNW4bBRSdXyGEgWeO48/ovH1fS1gMixgp 8 | V/UGYVIHvkLJ3NaIF6yBoZibC345Rt5QbHONRD+uPKJSX12N+XiABMaSfEqJc72H 9 | 5eRLk+Nrc8t7XG0UfF3xfCViGggAQewv0jsS9ES09QXAWREgVu3o4V/XnjvlmUIc 10 | p0bUGvrmZXjC3Llt5St0tWg45kSE4cu8F5+9uPxUFfPnucYm4RJxi9nUIdQDCj5n 11 | RDeKVkwoN8Tzi/rn1m1Ztz0Fi5RVEmFxQpKrM48QviuZU948vHT77sdV4ljYQUfE 12 | YkEmLbJAGuF2/EWCVeJ0tN2Ai+OQ2b2HhL+CPBrIzg/bQyj/Bb0VmBWEovsCAwEA 13 | AQKCAgADjAykxzBrPspzNY6be1eDijDDpYXWvStiBNW5KZJ3QUYq1786R9u8uLeJ 14 | FDhDE+bRtUejPNDzb67r24H9CjMdFL3/TnYf2fRPzGCdPbdqY/MbMM5GT66eC9+N 15 | qZO431kJbKLWMzTM9iomvImcvRQ/hFbKFIM4kgHda0jS76fFKd+sBusDAm7Cuj1o 16 | gm32AeIW7reSeqt2reXdrdgdBEWI78iA94Ha9Bh5EGE3cac/tVk6n2E1sl18r0IK 17 | Euj8BOyGv3CwXup44TK9ea7kxLJvmWggCl5LkExW7fLBqls27LjzYCpKl/FLeZVC 18 | ueTPEvxc4zG6nEoJ73m+LuGMXzmF4rwk2m3oEQO+mGBG7O4/XD5LOBnmcZ/VXGxR 19 | aGbsjC7zIY/+kOo4R2hUQNR1/CN8oxzaWPmqrpzi+KwQzdEcfilBEQTnbpJuvM51 20 | u9NQ6mjJrpgkT80ytnCbnz9wu8FNyheqajUJdAt2Drri6Wm1trwRSF7x9lxCFhz7 21 | 3/637cvxm+HTCju3uc9ouBewQmBRnTgm27jle/v/YfWZiqDugCk+ujXP5J9pz/Vc 22 | KX/JLv9K+bmfeGYqEdX0l+rOMlq1xV/v2KVqEUkhaVr3HnLLSFTr3SBFb6/+85dJ 23 | lJHrxvF3MCMD0RsMTdzA/cDMqrknhx9lxv6w42oNQbDA5r7kCQKCAQEA76tySH1/ 24 | bGl3GEu9U23KeSelGm3Q+dxiueC6fJ70JeK+6DOzvb49o68+N/D+RffPfGL1Oq+2 25 | ryTBBBGzKyk+nPSWGXYd+FLxrZeR++GHmRk8EJidUOBr0fOPXZipvhh0naqxqUGX 26 | nNbE4doZCuocedHNbZqr2+La82/OYHqN4WSqvMvS7FMDzHVO2/XZvC7eBJ9WcJK0 27 | MiwEIp+l6LCIYiQc1iHGgTlP/zABnNa+pkvZumI7+HcQOmOQn4Rg+Vn5fuLwsP2c 28 | y1T8bWn2/EeopBj9mguEjqTnehJ/NN0eGxk6kGXD5JJ9x1WaAcrJOpaqcY9qym2r 29 | 563P6ynh0p3p1wKCAQEA685JlAqIr5VaNhBnKqFyppNXrEKREH2hLQRKWHEucBl6 30 | raoJ1c8kfkA1lmWNoyMA5PfHUzXTpQjOE+fCrx++tLF/fS2hJphI7dFR0UplThjj 31 | lRvfFkRKYQDFEZ09l3gOgfnZ4V6BAkmLrfxklPV/tIQccAu/S0ipsaBoqkAPgj9Y 32 | InWbPL89RABfRjlkh0blSmwJsZVloy3T87bKahhienERCNbZQsMuz3r6YnkFDnn1 33 | KH0lr52vIoHQEkykDU5HafWW4SOvnDeQv0YAM526hr+z/HVi7y27A9mxGl36KCnd 34 | bEPzngmrq6e2vIR4iRi6MaE437E0hmBdBjAgYZWTfQKCAQEAl5gdAEcJCxymbOKz 35 | TQr776go4U+mx/QjKilAK8Uq7q++Swcw4IptWJRmyWGQo9b8EVRSgp1T8vJPd/Bu 36 | IJXE+egm7NHuYPytyzw85VIgv7Dbci/mCZr59+GTxALlEs1LLdMi7skDSfv6qlSW 37 | VHYCLveKdZJItBuInEQTCKlcBr3y1NojfsF/fJqSl8SoxUPrwIa8WRYiDj4TG4dZ 38 | nJXUnibzamoQsfn7ZJEL5BVcn7kIbcDm5+D/jb8eYheMcwv7KMwIrBquS1plFBvb 39 | UnVM8QaxmfdBECy0EV2Twm8+NKu/hEgqIPCbyDKX6BxJLdrQ0mWBiarW+gLRqZBj 40 | OyIRHwKCAQEAwcX4IsZxQlBqHGloTdj0HTyRXEYrbTC/pP2Ulo8TiRz2+gT0JhYd 41 | t/3Ny3RSaw/VqVKQIqnl7QWKS0M1cbGVhmkiCdzYQ536uCs9n84R55Z+VkbcETE3 42 | KfV2/B2JwpHu4hJOYGvHefiz5tq0UNBZDB9QOyb8IMJqaEQYHmecFv000MRyN/4z 43 | O1FO+acqHBw/SeP0J1FMf0ammOzJ81tseY+2XtwnFzzLl8ZIQWmPpTDFLGuC74iV 44 | pbalJJSEb0a3WrRtMz+fDLGAqh29Bq2D3T1li31vOnsK5oQajD9T1XNpuJEyfdkW 45 | kG22tb3qS0bdovEIipHVRuOjLfWYdnhzBQKCAQEAzqQgiY4TGU1IC/sporRLc9dY 46 | ZCSgK9YJWOjiE3qRhLtOBCWXmCJx40BRH1eYRybJ4ACmc+HaVoLATkvac/Mu8xBI 47 | c2f1p8NPqatnLQPBF8vOs+GF5pXzyS/qFfUyWWT6bW+EZlvW1B23BYKqhM2VdC3Q 48 | fUOp+inetsGs/iZGqYCNdugDIMn8LqqEIrAd+TcOUEHZDMt4ZdZBuXDaeUj7yNX6 49 | qPhtgBKRLSjI84xYijv4Xawt8PD7n4O9YjMe5beMZa4yrhjfQ2m4U9qZXWmL7pKt 50 | efm0OspJX+qMtHcyAr1JIDyqD5XSLYelLeJKOL2VjbWCc/exJLLc7EYY2EpxCg== 51 | -----END RSA PRIVATE KEY----- 52 | -------------------------------------------------------------------------------- /netpalm/backend/core/security/cert/tls/ca.txt: -------------------------------------------------------------------------------- 1 | 316E8D484E56C063DF5C9F3F94C6FB0AF34AB172 2 | -------------------------------------------------------------------------------- /netpalm/backend/core/security/cert/tls/redis.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID4jCCAcoCFDFujUhOVsBj31yfP5TG+wrzSrFyMA0GCSqGSIb3DQEBCwUAMDUx 3 | EzARBgNVBAoMClJlZGlzIFRlc3QxHjAcBgNVBAMMFUNlcnRpZmljYXRlIEF1dGhv 4 | cml0eTAeFw0yMzEyMDQxNTA5NDhaFw0yNDEyMDMxNTA5NDhaMCYxEzARBgNVBAoM 5 | ClJlZGlzIFRlc3QxDzANBgNVBAMMBlNlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQAD 6 | ggEPADCCAQoCggEBAOnIceKAWUiYmfWVNQJIM7ffiI+Pg3A4pjQ39SG1+QlHkvSw 7 | 6Y3je9HbmgcXhG6PEXi71pPDebFPr+LgHdA9W76CY5Q7iiY6qM7/F79FAhq+HOIA 8 | 9Ne5b176tFUYFe4+kMF1/WG6jtoPLdBp5SrwEFuuPHOWyL4x8s+mqaOLX2E2OQoK 9 | 4f5KyLC4de1i+miSyrTw0eHli8CwLV2ogSTJB2f2khI6iX/hgXPyFitWj92oqBBI 10 | Z8o/Cc+m+lwh24Syu/fAtJP2aZEaX1S07juStNkSEaLI/8uh+HjoMEv0S9vZuhJC 11 | MvJu6SaKpLcrf4WnqXkCJRT3U9TdnWbf1r/iLLcCAwEAATANBgkqhkiG9w0BAQsF 12 | AAOCAgEAb5PusKGBfdYeB6LEr1p4/yRX9gK7nVgHqQkNmDzh7TAjL8ZAkV5TCBi0 13 | 1HWZNL32DCKZcLoaWnQaUouAha426mJTDDZmH1LQ6N1htiRHHzsCRR01WwpZm+dD 14 | JrNPBLh8iWDk06jV+XMafhU+vPmHXCkvQ8ISEhS10YxJ7Z7kjw07Dp+/CwQE79xx 15 | 6HPrs9AGBsEIsgmXjG0ONlgQW1e7tIfsBK0oPb/Yl1I2D/e4ehpASNwIQvvHf0e+ 16 | q6z7oSPos2e5DWBiedKrM+4BPnxzLd4h1jzs1y9TAmNF4B/Rbf2lGiU1uqOb4mwS 17 | sPzaw9mUtG+ItQdNp3GiGFIrr8Jg9pIX7ooCm0yVGVDHCzBN56Qeoj/tN/AWzNk7 18 | eBqzaiVA6lMgE1mPCunhwnJ4EicaFQsU6fLmHtX43F2qsszv72VrCY+ZXZ3Cocen 19 | bakjHqEg8gV6Ki+ZEgvABVoeMVOdnBQdjhiq3GBnidw2pLOOEpWbOlMzG0oFhHxG 20 | sditGsNKHGxBrRd2Z8tcsbbuh3A/2reTUq/Ry5hEwGhrRmpWr52DZE2kGvxaPAnt 21 | cGb8jpnwkzeF4V4V2k1CLdIZNC2kHvLTg6yXMOcnvizpzA4cu0qGj3Ka5X2MrTQW 22 | xEbrrB/XuplBWc/BJAjxV9+rjWEKEt+P+DKGe4uaGVOHiADlcCo= 23 | -----END CERTIFICATE----- 24 | -------------------------------------------------------------------------------- /netpalm/backend/core/security/cert/tls/redis.dh: -------------------------------------------------------------------------------- 1 | -----BEGIN DH PARAMETERS----- 2 | MIIBCAKCAQEA3VGxJe0EnE5qjYaJOKH7Cf0sVP0ve5ZajERRQWErm+DnnKnMUXHm 3 | g8rp2OAXwhPIuAZ012jiP3sxBbi53F+8h4EDpNLGVMxaF0t/yeSSPIj33KYuZKRi 4 | xiec1rfnLF8B7W44UBdAop0Xxxv4TbMVH2azZVOfA0NAarGjT+O5REAnjmHHzOTl 5 | gQh4EEyeZKB5nuKWZ5iXpYQyLRuWgpsWQOhi4XkEjd9XR3ogRMlM8YWFxsuKMVwh 6 | tAUnq2mN7EAuxZopItMaKKEFDoarWSkNjjCYs72iJ8j+XjmAmBVLwCt0kkynPY7v 7 | DHKnt6Bbb0GKM/BDCSCwMPOC0x7N7l3GUwIBAg== 8 | -----END DH PARAMETERS----- 9 | -------------------------------------------------------------------------------- /netpalm/backend/core/security/cert/tls/redis.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpQIBAAKCAQEA6chx4oBZSJiZ9ZU1Akgzt9+Ij4+DcDimNDf1IbX5CUeS9LDp 3 | jeN70duaBxeEbo8ReLvWk8N5sU+v4uAd0D1bvoJjlDuKJjqozv8Xv0UCGr4c4gD0 4 | 17lvXvq0VRgV7j6QwXX9YbqO2g8t0GnlKvAQW648c5bIvjHyz6apo4tfYTY5Cgrh 5 | /krIsLh17WL6aJLKtPDR4eWLwLAtXaiBJMkHZ/aSEjqJf+GBc/IWK1aP3aioEEhn 6 | yj8Jz6b6XCHbhLK798C0k/ZpkRpfVLTuO5K02RIRosj/y6H4eOgwS/RL29m6EkIy 7 | 8m7pJoqktyt/haepeQIlFPdT1N2dZt/Wv+IstwIDAQABAoIBAQCE5x/fHQok3CQC 8 | We2Fu5gzOs++lV6qiHnsC7e+j13hKaBgnMZZlG5KsmYRDAyMcacEFPz6idD/vQaH 9 | mtM4E/m5BxUnKuXxr71R1/EfGlME8vkmflay4MdwnmOwUH3/L/BJGxON4Gnfi2uH 10 | JPxT+Z/DDv1s1NaQUYLxC5U61TvG3d9M8hmUZwU+1y35lFgdzwGoRDlk8mf54mUn 11 | lR2ncAoNqUgRPvfyRk3WCX4g4rbPel+MhFtqkyhBx0cgxU992xQfKngjnGbGWHGf 12 | iSLca4EfV5yn7OszxdTgZngw/RAgN5wqVg7p8qxRMuSsPpNB2YnOWR1Em/cJkYpn 13 | HoLYCvD5AoGBAPYEcVp4WYkYcVq4I/zfqAvO4jUuscFYG5GtWIy21tUn9q/R757f 14 | vCvrYGhcJF0CPeW/ihE8nU1320HrEZt3DJyem3gijXLyR1nXlDP8aS6Oo1BqKTd/ 15 | jBSLwpF8pcq+ZFF4XrgJX820YpGjXI8tR9lnAip64v9JJw3Km81xj2SjAoGBAPNE 16 | 6j3eEXMBjGIGLt2jQN91mNsfpROXckQDMNclxC0x4kHP4vX+55wEDIqlmVZb9kdG 17 | ErzGxMeqyEkX+be/kxNevAr9kJuEWt866WzqHW1z05brHroP7ZDTFsnK9lphV0hc 18 | oyL1hrSSQd6u/9g0K95yFJFRBfcLuC7KcNY2/0TdAoGAGbAPEe49ExaNR3aws0e7 19 | KTh54Cm2/8QuZMCf3Vl+BxW8KVGXXpRn9oIGN/9fAKMqSWH9jZ0xo5QfPbvIw93j 20 | YDsjaqDY+iaYEqZy4Ba7+NOF8/CRyyyzOLX7I+vK9+d3XBZgdYDtEvKB1EklcgG9 21 | XdLmo/7e0t8Vh9ge+o2f9lcCgYEAvDerzKZOdd7S0Nzb0dwHy1zfIVrAjAH3BKw7 22 | U5sCx7z6w9SmU5IffNSdJ4CsvOZ2C/spcCmSArIeOH92VDbDAo+fihlZkxm6fVDF 23 | XeFdV1vwb6oGogrE0fKghQylwpPhxjj2d4vj5z3p/PMFPDBRaNB3HV/E77l/jaRP 24 | 3U27/3kCgYEA8IvOTXJc9+YKxZ6/Ep2AkKJXALVklh9RsZ0V/thQ2o3s3i3mtyHQ 25 | qK7CsrVU8p2IiQyrj7nUJQIFXfZ0msLPnCqeTlFXVK78rr2EB5IF6qmimM8orcj9 26 | h9SfLx8wve1QvqWF/L72l3T881BvgFAX0C0OzflfhWg2V92oxxu5bRM= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /netpalm/backend/core/security/get_api_key.py: -------------------------------------------------------------------------------- 1 | from fastapi import Security, HTTPException 2 | from fastapi.security.api_key import APIKeyQuery, APIKeyCookie, APIKeyHeader 3 | from starlette.status import HTTP_403_FORBIDDEN 4 | 5 | from netpalm.backend.core.confload.confload import config 6 | 7 | api_key_query = APIKeyQuery(name=config.api_key_name, auto_error=False) 8 | api_key_header = APIKeyHeader(name=config.api_key_name, auto_error=False) 9 | api_key_cookie = APIKeyCookie(name=config.api_key_name, auto_error=False) 10 | 11 | 12 | async def get_api_key( 13 | api_key_query: str = Security(api_key_query), 14 | api_key_header: str = Security(api_key_header), 15 | api_key_cookie: str = Security(api_key_cookie), 16 | ): 17 | """checks for an API key""" 18 | if api_key_query == config.api_key: 19 | return api_key_query 20 | elif api_key_header == config.api_key: 21 | return api_key_header 22 | elif api_key_cookie == config.api_key: 23 | return api_key_cookie 24 | else: 25 | raise HTTPException( 26 | status_code=HTTP_403_FORBIDDEN, detail="Could not validate credentials" 27 | ) 28 | -------------------------------------------------------------------------------- /netpalm/backend/core/security/whitelist.py: -------------------------------------------------------------------------------- 1 | import ipaddress 2 | from fnmatch import fnmatch 3 | 4 | from typing import List 5 | 6 | 7 | class WhiteListRule: 8 | """ 9 | if `definition` is a valid IPv4 or IPv6 Address or network in CIDR format, evaluate candidate 10 | hosts as addresses and return if they are in the equivalent network. 11 | 12 | else do unix filesystem-like wildcard matching ('*.foo.com' matches 'a.foo.com' but not 'foo.com' itself) 13 | """ 14 | 15 | def __init__(self, definition: str): 16 | try: 17 | self.definition = ipaddress.ip_interface(definition).network 18 | self.type = "ip" 19 | except ValueError: 20 | self.definition = definition 21 | self.type = "str" 22 | 23 | def match(self, host: str) -> bool: 24 | if self.type == "ip": 25 | try: 26 | return ipaddress.ip_address(host) in self.definition 27 | except ValueError: 28 | return False 29 | 30 | return fnmatch(host, self.definition) 31 | 32 | 33 | class DeviceWhitelist: 34 | """ 35 | evaluate rules in order, return True if any match. If rule list is empty, return True for anything 36 | """ 37 | 38 | def __init__(self, definition: List[str]): 39 | self.definition = definition 40 | if self.definition is None: 41 | definition = [] 42 | 43 | self.rules = [WhiteListRule(rule_definition) for rule_definition in definition] 44 | 45 | def match(self, hostname): 46 | if not self.rules: 47 | return True 48 | 49 | return any(rule.match(hostname) for rule in self.rules) 50 | -------------------------------------------------------------------------------- /netpalm/backend/core/utilities/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/core/utilities/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/utilities/extensibles_reload.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import os, signal 4 | 5 | log = logging.getLogger(__name__) 6 | 7 | def reload_extensibles_func(): 8 | try: 9 | with open("controller.pid", encoding = 'utf-8') as f: 10 | pid = f.readline() 11 | log.info(f"reload_extensibles: reloading extensibles for {pid}") 12 | os.kill(int(pid), signal.SIGHUP) 13 | return True 14 | except FileNotFoundError: 15 | return False 16 | except Exception as e: 17 | log.error(f"reload_extensibles: reloading extensibles for {e}") 18 | return False -------------------------------------------------------------------------------- /netpalm/backend/core/utilities/jinja2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/core/utilities/jinja2/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/utilities/jinja2/j2.py: -------------------------------------------------------------------------------- 1 | from jinja2 import Environment, FileSystemLoader 2 | from jinja2schema import infer, to_json_schema 3 | 4 | from netpalm.backend.core.confload.confload import config 5 | 6 | 7 | class j2: 8 | 9 | def __init__(self, j2_type=False, **kwargs): 10 | self.kwarg = kwargs.get('kwargs', False) 11 | if j2_type == "config": 12 | self.jinja_template_dir = config.jinja2_config_templates 13 | if j2_type == "webhook": 14 | self.jinja_template_dir = config.webhook_jinja2_templates 15 | self.file_loader = FileSystemLoader(self.jinja_template_dir) 16 | self.env = Environment(loader=self.file_loader, lstrip_blocks=True, trim_blocks=True) 17 | 18 | def opentemplate(self, template): 19 | try: 20 | with open(template) as f: 21 | res = f.read() 22 | return str(res) 23 | except Exception as e: 24 | return e 25 | 26 | def gettemplate(self, template): 27 | try: 28 | templat = self.jinja_template_dir + template + '.j2' 29 | res = self.opentemplate(templat) 30 | try: 31 | schema = infer(res) 32 | js_schema = to_json_schema(schema) 33 | except Exception: 34 | js_schema = "error reading schema" 35 | resultdata = { 36 | 'status': 'success', 37 | 'data': { 38 | "task_result": { 39 | "template_schema": js_schema, 40 | "template_data": res 41 | } 42 | } 43 | } 44 | return resultdata 45 | except Exception as e: 46 | resultdata = { 47 | 'status': 'error', 48 | 'data': str(e) 49 | } 50 | return resultdata 51 | 52 | def render_j2template(self, template, **kwargs): 53 | try: 54 | kwargs = kwargs.get("kwargs", False) 55 | templat = template + '.j2' 56 | tmp_template = self.env.get_template(templat) 57 | output = tmp_template.render(kwargs) 58 | resultdata = { 59 | 'status': 'success', 60 | 'data': { 61 | "task_result": { 62 | "template": template, 63 | "template_render_result": str(output), 64 | } 65 | } 66 | } 67 | return resultdata 68 | except Exception as e: 69 | return e 70 | 71 | 72 | def j2gettemplate(tmplate, template_type=False): 73 | t = j2(j2_type=template_type) 74 | res = t.gettemplate(tmplate) 75 | return res 76 | 77 | 78 | def render_j2template(templat, template_type=False, **kwargs): 79 | t = j2(j2_type=template_type) 80 | res = t.render_j2template(template=templat, kwargs=kwargs["kwargs"]) 81 | return res 82 | -------------------------------------------------------------------------------- /netpalm/backend/core/utilities/ls/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/core/utilities/ls/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/utilities/ls/ls.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from netpalm.backend.core.confload.confload import config 5 | from netpalm.backend.core.models.task import ResponseBasic 6 | 7 | log = logging.getLogger(__name__) 8 | 9 | 10 | class ls: 11 | 12 | def __init__(self, folder=False): 13 | if folder == "config": 14 | self.folder_dir = config.jinja2_config_templates 15 | self.strip = ".j2" 16 | elif folder == "service": 17 | self.folder_dir = config.python_service_templates 18 | self.strip = ".py" 19 | elif folder == "webhook": 20 | self.folder_dir = config.webhook_jinja2_templates 21 | self.strip = ".j2" 22 | elif folder == "webhook_script": 23 | self.folder_dir = config.custom_webhooks 24 | self.strip = ".py" 25 | elif folder == "ttp_templates": 26 | self.folder_dir = config.ttp_templates 27 | self.strip = ".ttp" 28 | elif folder == "script": 29 | self.folder_dir = config.custom_scripts 30 | self.strip = ".py" 31 | 32 | def path_hierarchy(self, path, strip=False): 33 | try: 34 | files = [] 35 | fileresult = [] 36 | for r, d, f in os.walk(path): 37 | for file in f: 38 | file.strip(path) 39 | files.append(os.path.join(r, file)) 40 | if len(files) > 0: 41 | for f in files: 42 | if "__init__" not in f: 43 | if "__pycache__" not in f: 44 | if "_model.py" not in f: 45 | if self.strip: 46 | if self.strip in f: 47 | ftmpfile = f.replace(self.strip, '') 48 | fileresult.append(ftmpfile.replace(path, '')) 49 | resultdata = ResponseBasic(status="success", data={"task_result": {"templates": fileresult}}).dict() 50 | return resultdata 51 | except Exception as e: 52 | return str(e) 53 | 54 | def getfiles(self): 55 | try: 56 | res = self.path_hierarchy(path=self.folder_dir, strip=False) 57 | return res 58 | except Exception as e: 59 | return str(e) 60 | 61 | def bond_models(self): 62 | try: 63 | res = self.getfiles() 64 | return res 65 | except Exception as e: 66 | return str(e) 67 | 68 | 69 | def list_files(fldr=False): 70 | res = ls(folder=fldr) 71 | response = res.getfiles() 72 | return response 73 | -------------------------------------------------------------------------------- /netpalm/backend/core/utilities/rediz_kill_worker.py: -------------------------------------------------------------------------------- 1 | import os 2 | import signal 3 | import socket 4 | import logging 5 | 6 | log = logging.getLogger(__name__) 7 | 8 | 9 | def kill_worker_pid(**kwargs): 10 | """kills a rq worker process by its pid""" 11 | try: 12 | hostname = kwargs.get("hostname") 13 | pid = kwargs.get("pid") 14 | output = socket.gethostname() 15 | if f"{output}" == hostname: 16 | log.info({f"kill_worker_pid: killing worker PID {pid} on {output}"}) 17 | os.kill(int(pid), signal.SIGINT) 18 | except Exception as e: 19 | log.error({f"kill_worker_pid: {e}"}) 20 | return e 21 | -------------------------------------------------------------------------------- /netpalm/backend/core/utilities/rediz_meta.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | from logging import getLogger 3 | 4 | from rq import get_current_job 5 | 6 | from netpalm.backend.core.confload.confload import config 7 | from netpalm.backend.core.models.task import Response 8 | from netpalm.exceptions import NetpalmMetaProcessedException 9 | 10 | log = getLogger(__name__) 11 | 12 | 13 | def exception_full_name(exception: BaseException): 14 | name = exception.__class__.__name__ 15 | if (module := inspect.getmodule(exception)) is None: 16 | return name 17 | 18 | name = f'{module.__name__}.{name}' 19 | return name 20 | 21 | 22 | def yield_exception_chain(exc: BaseException): 23 | yield exc 24 | if exc.__context__ is None: 25 | return 26 | yield from yield_exception_chain(exc.__context__) 27 | 28 | 29 | def write_meta_error(exception: Exception): 30 | """custom exception handler for within an rpc job""" 31 | if isinstance(exception, NetpalmMetaProcessedException): 32 | raise exception from None # Don't process the same exception twice 33 | 34 | log.exception('`write_meta_error` processing error') 35 | 36 | job = get_current_job() 37 | job.meta["result"] = "failed" 38 | 39 | exception_chain = yield_exception_chain(exception) 40 | 41 | for exception in reversed(list(exception_chain)): 42 | task_error = { 43 | 'exception_class': exception_full_name(exception), 44 | 'exception_args': [arg for arg in exception.args if arg is not None] 45 | } 46 | job.meta["errors"].append(task_error) 47 | 48 | job.save_meta() 49 | raise NetpalmMetaProcessedException from exception 50 | 51 | 52 | def write_meta_error_string(data): 53 | """custom exception handler for within an rpc job""" 54 | job = get_current_job() 55 | job.meta["result"] = "failed" 56 | job.meta["errors"].append(data) 57 | job.save_meta() 58 | raise Exception(f"failed: {data}") 59 | 60 | 61 | def write_mandatory_meta(): 62 | job = get_current_job() 63 | if job is None: # it will be None in many/all unit tests 64 | return 65 | job.meta["assigned_worker"] = config.worker_name 66 | job.save_meta() 67 | 68 | 69 | def render_netpalm_payload(job_result={}): 70 | """in band rpc job result renderer""" 71 | try: 72 | job = get_current_job() 73 | resultdata = Response(status="success", 74 | data={"task_id": job.id, 75 | "created_on": job.created_at.strftime("%Y-%m-%d %H:%M:%S.%f"), 76 | "task_queue": job.description, 77 | "task_status": "finished", 78 | "task_result": job_result, 79 | "task_errors": job.meta["errors"] 80 | }).dict() 81 | return resultdata 82 | 83 | except Exception as e: 84 | return e 85 | -------------------------------------------------------------------------------- /netpalm/backend/core/utilities/textfsm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/core/utilities/textfsm/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/utilities/universal_template_mgr/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/core/utilities/universal_template_mgr/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/utilities/universal_template_mgr/unvrsl.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import os, signal 3 | 4 | from typing import Dict 5 | 6 | from netpalm.backend.core.confload.confload import config 7 | from netpalm.backend.core.models.task import ResponseBasic 8 | 9 | from netpalm.backend.core.utilities.extensibles_reload import reload_extensibles_func 10 | 11 | class unvrsl: 12 | 13 | def __init__(self): 14 | self.routing_table = { 15 | "j2_config_templates": {"path": config.jinja2_config_templates, "extn": ".j2"}, 16 | "python_service_templates": {"path": config.python_service_templates, "extn": ".py"}, 17 | "j2_webhook_templates": {"path": config.webhook_jinja2_templates, "extn": ".j2"}, 18 | "ttp_templates": {"path": config.ttp_templates, "extn": ".ttp"}, 19 | "custom_scripts": {"path": config.custom_scripts, "extn": ".py"}, 20 | "custom_webhooks": {"path": config.custom_webhooks, "extn": ".py"} 21 | } 22 | 23 | def add_template(self, payload: Dict[str, str]): 24 | try: 25 | raw_base = base64.b64decode(payload["base64_payload"]).decode('utf-8') 26 | template_path = self.routing_table[payload["route_type"]]["path"] + payload["name"] + self.routing_table[payload["route_type"]]["extn"] 27 | with open(template_path, "w") as file: 28 | file.write(raw_base) 29 | reload_extensibles_func() 30 | resultdata = ResponseBasic(status="success", data={"task_result": {"added": payload["name"]}}).dict() 31 | return resultdata 32 | except Exception as e: 33 | error = ResponseBasic(status="error", data={"task_result": {"error": str(e)}}).dict() 34 | return error 35 | 36 | def remove_template(self, payload: Dict[str, str]): 37 | try: 38 | template_path = self.routing_table[payload["route_type"]]["path"] + payload["name"] + self.routing_table[payload["route_type"]]["extn"] 39 | os.remove(template_path) 40 | resultdata = ResponseBasic(status="success", data={"task_result": {"removed": payload["name"]}}).dict() 41 | reload_extensibles_func() 42 | return resultdata 43 | except Exception as e: 44 | error = ResponseBasic(status="error", data={"task_result": {"error": str(e)}}).dict() 45 | return error 46 | 47 | def get_template(self, payload: Dict[str, str]): 48 | try: 49 | template_path = self.routing_table[payload["route_type"]]["path"] + payload["name"] + self.routing_table[payload["route_type"]]["extn"] 50 | result = None 51 | with open(template_path, "r") as file: 52 | result = file.read() 53 | raw_base = base64.b64encode(result.encode('utf-8')) 54 | resultdata = ResponseBasic(status="success", data={"task_result": {"base64_payload": raw_base}}).dict() 55 | return resultdata 56 | except Exception as e: 57 | error = ResponseBasic(status="error", data={"task_result": {"error": str(e)}}).dict() 58 | return error 59 | -------------------------------------------------------------------------------- /netpalm/backend/core/utilities/webhook/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/core/utilities/webhook/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/core/utilities/webhook/webhook.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import json 3 | import logging 4 | 5 | from netpalm.backend.core.confload.confload import config 6 | from netpalm.backend.core.utilities.jinja2.j2 import render_j2template 7 | 8 | log = logging.getLogger(__name__) 9 | 10 | 11 | class webhook_runner: 12 | def __init__(self, whook_payload: dict): 13 | self.webhook_dir_path = config.custom_webhooks 14 | self.webhook_raw_name = whook_payload.get("name") 15 | self.webhook_args = whook_payload.get("args", False) 16 | if not self.webhook_raw_name: 17 | self.webhook_name = config.default_webhook_name 18 | self.webhook_name = ( 19 | self.webhook_dir_path.replace("/", ".") + self.webhook_raw_name 20 | ) 21 | self.webhook_j2_name = whook_payload.get("j2template") 22 | 23 | def webhook_exec(self, job_data: dict): 24 | try: 25 | log.info(f"webhook_exec: importing {self.webhook_name}") 26 | module = importlib.import_module(self.webhook_name) 27 | run_whook = getattr(module, "run_webhook") 28 | job_data["webhook_args"] = self.webhook_args 29 | whook_data = job_data 30 | log.info(f"webhook_exec: webhook data loaded {whook_data}") 31 | if self.webhook_j2_name: 32 | log.info( 33 | f"webhook_exec: rendering webhook j2 template {self.webhook_j2_name}" 34 | ) 35 | res = render_j2template( 36 | self.webhook_j2_name, template_type="webhook", kwargs=job_data 37 | ) 38 | whook_data = json.loads( 39 | res["data"]["task_result"]["template_render_result"] 40 | ) 41 | res = run_whook(payload=whook_data) 42 | return res 43 | except Exception as e: 44 | log.error(f"webhook_exec: webhook run failure {e}") 45 | return e 46 | 47 | 48 | def exec_webhook_func(jobdata: dict, webhook_payload: dict): 49 | """ executes a webhook """ 50 | webhook = webhook_runner(whook_payload=webhook_payload) 51 | execute = webhook.webhook_exec(job_data=jobdata) 52 | return execute 53 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/drivers/napalm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/plugins/drivers/napalm/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/plugins/drivers/napalm/napalm_drvr.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import napalm 4 | 5 | from netpalm.backend.core.utilities.rediz_meta import write_meta_error 6 | from netpalm.backend.core.driver.netpalm_driver import NetpalmDriver 7 | 8 | log = logging.getLogger(__name__) 9 | 10 | 11 | class naplm(NetpalmDriver): 12 | driver_name = "napalm" 13 | 14 | def __init__(self, **kwargs): 15 | log.debug(f"initializing napalm driver with args: {kwargs}") 16 | self.connection_args = kwargs.get("connection_args", False) 17 | # convert the netmiko naming format to the native napalm format 18 | driver_lookup = { 19 | "arista_eos": "eos", 20 | "juniper": "junos", 21 | "cisco_xr": "iosxr", 22 | "nxos": "nxos", 23 | "cisco_nxos_ssh": "nxos_ssh", 24 | "cisco_ios": "ios", 25 | } 26 | self.driver = driver_lookup[self.connection_args.get("device_type", False)] 27 | self.connection_args["hostname"] = self.connection_args.pop("host") 28 | del self.connection_args["device_type"] 29 | 30 | def connect(self): 31 | try: 32 | driver = napalm.get_network_driver(self.driver) 33 | napalmses = driver(**self.connection_args) 34 | return napalmses 35 | except Exception as e: 36 | write_meta_error(e) 37 | 38 | def sendcommand(self, session=False, command=False): 39 | log.debug(f"running send command on napalm driver: {session} {command}") 40 | try: 41 | result = {} 42 | session.open() 43 | for c in command: 44 | if hasattr(session, str(c)): 45 | response = getattr(session, str(c))() 46 | result[c] = response 47 | else: 48 | response = session.cli([c]) 49 | result[c] = response[c].split("\n") 50 | return result 51 | except Exception as e: 52 | write_meta_error(e) 53 | 54 | def config(self, session=False, command=False, dry_run=False): 55 | try: 56 | if type(command) == list: 57 | napalmconfig = "" 58 | for comm in command: 59 | napalmconfig += comm + "\n" 60 | else: 61 | napalmconfig = command 62 | session.open() 63 | session.load_merge_candidate(config=napalmconfig) 64 | diff = session.compare_config() 65 | if dry_run: 66 | response = session.discard_config() 67 | else: 68 | response = session.commit_config() 69 | result = {} 70 | result["changes"] = diff.split("\n") 71 | return result 72 | except Exception as e: 73 | write_meta_error(e) 74 | 75 | def logout(self, session): 76 | try: 77 | response = session.close() 78 | return response 79 | except Exception as e: 80 | write_meta_error(e) 81 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/drivers/ncclient/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/drivers/ncclient/ncclient_drvr.py: -------------------------------------------------------------------------------- 1 | import xmltodict 2 | import logging 3 | from ncclient import manager 4 | 5 | from netpalm.backend.core.utilities.rediz_meta import ( 6 | write_meta_error_string, 7 | write_meta_error, 8 | ) 9 | from netpalm.backend.core.driver.netpalm_driver import NetpalmDriver 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | class ncclien(NetpalmDriver): 15 | driver_name = "ncclient" 16 | 17 | def __init__(self, **kwargs): 18 | self.kwarg = kwargs.get("args", False) 19 | self.connection_args = kwargs.get("connection_args", False) 20 | 21 | def connect(self): 22 | try: 23 | conn = manager.connect(**self.connection_args) 24 | return conn 25 | except Exception as e: 26 | write_meta_error(e) 27 | 28 | @staticmethod 29 | def __get_capabilities(session=False): 30 | try: 31 | capabilities = session.server_capabilities 32 | return capabilities 33 | except Exception as e: 34 | write_meta_error(e) 35 | 36 | def getmethod(self, session=False, command=False): 37 | try: 38 | result = {} 39 | if self.kwarg: 40 | rjsflag = False 41 | if "render_json" in self.kwarg: 42 | if self.kwarg.get("render_json"): 43 | rjsflag = True 44 | del self.kwarg["render_json"] 45 | response = session.get(**self.kwarg).data_xml 46 | if rjsflag: 47 | respdict = xmltodict.parse(response) 48 | if respdict: 49 | result["get_config"] = respdict 50 | else: 51 | write_meta_error_string("failed to parse response") 52 | else: 53 | result["get_config"] = response 54 | else: 55 | write_meta_error_string("args are required") 56 | return result 57 | except Exception as e: 58 | write_meta_error(e) 59 | 60 | def sendcommand(self, session=False, command=False): 61 | try: 62 | result = {} 63 | if self.kwarg: 64 | rjsflag = False 65 | 66 | if "render_json" in self.kwarg: 67 | if self.kwarg.get("render_json"): 68 | rjsflag = True 69 | del self.kwarg["render_json"] 70 | 71 | if "capabilities" in self.kwarg: 72 | if self.kwarg.get("capabilities"): 73 | result["capabilities"] = self.__get_capabilities( 74 | session=session 75 | ) 76 | del self.kwarg["capabilities"] 77 | 78 | # check whether RPC required 79 | if self.kwarg.get("rpc", False): 80 | response = session.rpc(**self.kwarg).data_xml 81 | # else a standard get_config method call 82 | else: 83 | response = session.get_config(**self.kwarg).data_xml 84 | if rjsflag: 85 | respdict = xmltodict.parse(response) 86 | if respdict: 87 | result["get_config"] = respdict 88 | else: 89 | write_meta_error_string("failed to parse response") 90 | else: 91 | result["get_config"] = response 92 | else: 93 | write_meta_error_string("args are required") 94 | return result 95 | except Exception as e: 96 | write_meta_error(e) 97 | 98 | def config(self, session=False, dry_run=False): 99 | try: 100 | result = {} 101 | if self.kwarg: 102 | rjsflag = False 103 | if "render_json" in self.kwarg: 104 | if self.kwarg.get("render_json"): 105 | rjsflag = True 106 | del self.kwarg["render_json"] 107 | # edit_config returns an RPCReply object which doesnt have a 108 | # data_xml property. Fixes 'Unserializable return value' 109 | # message from rq.job:restore 110 | response = session.edit_config(**self.kwarg).xml 111 | if dry_run: 112 | session.discard_changes() 113 | else: 114 | session.commit() 115 | if rjsflag: 116 | respdict = xmltodict.parse(response) 117 | if respdict: 118 | result["edit_config"] = respdict 119 | else: 120 | write_meta_error_string("failed to parse response") 121 | else: 122 | result["edit_config"] = response 123 | else: 124 | write_meta_error_string("args are required") 125 | return result 126 | except Exception as e: 127 | write_meta_error(e) 128 | 129 | def logout(self, session): 130 | try: 131 | response = session.close_session() 132 | return response 133 | except Exception as e: 134 | write_meta_error(e) 135 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/drivers/netmiko/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/plugins/drivers/netmiko/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/plugins/drivers/netmiko/netmiko_drvr.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from netmiko import ConnectHandler, BaseConnection 4 | from netmiko.cisco_base_connection import CiscoBaseConnection 5 | from typing import Optional 6 | 7 | from netpalm.backend.core.confload.confload import config 8 | 9 | from netpalm.backend.core.driver.netpalm_driver import NetpalmDriver 10 | from netpalm.backend.core.utilities.rediz_meta import write_meta_error 11 | 12 | log = logging.getLogger(__name__) 13 | 14 | 15 | class netmko(NetpalmDriver): 16 | driver_name = "netmiko" 17 | 18 | def __init__(self, **kwargs): 19 | self.kwarg = kwargs.get("args", False) 20 | self.connection_args = kwargs.get("connection_args", False) 21 | # support IOSXR commit labels 22 | self.commit_label = None 23 | if self.kwarg: 24 | if commit_label := self.kwarg.get("commit_label", None): 25 | self.commit_label = commit_label 26 | del self.kwarg["commit_label"] 27 | self.enable_mode = kwargs.get("enable_mode", False) 28 | 29 | def connect(self): 30 | try: 31 | netmikoses = ConnectHandler(**self.connection_args) 32 | return netmikoses 33 | except Exception as e: 34 | write_meta_error(e) 35 | 36 | def sendcommand(self, session=False, command=False): 37 | try: 38 | if self.enable_mode: 39 | session.enable() 40 | result = {} 41 | for commands in command: 42 | if self.kwarg: 43 | # normalise the ttp template name for ease of use 44 | if "ttp_template" in self.kwarg.keys(): 45 | if self.kwarg["ttp_template"]: 46 | template_name = ( 47 | config.ttp_templates 48 | + self.kwarg["ttp_template"] 49 | + ".ttp" 50 | ) 51 | self.kwarg["ttp_template"] = template_name 52 | response = session.send_command(commands, **self.kwarg) 53 | if response: 54 | result[commands] = response 55 | else: 56 | response = session.send_command(commands) 57 | if response: 58 | result[commands] = response.split("\n") 59 | if self.enable_mode: 60 | session.exit_enable_mode() 61 | return result 62 | except Exception as e: 63 | write_meta_error(e) 64 | 65 | def config(self, session=False, command="", enter_enable=False, dry_run=False): 66 | try: 67 | if type(command) == list: 68 | comm = command 69 | else: 70 | comm = command.splitlines() 71 | 72 | if enter_enable: 73 | session.enable() 74 | 75 | if self.kwarg: 76 | response = session.send_config_set(comm, **self.kwarg) 77 | else: 78 | response = session.send_config_set(comm) 79 | 80 | if not dry_run: 81 | response += self.__try_commit_or_save(session) 82 | 83 | result = {} 84 | result["changes"] = response.split("\n") 85 | return result 86 | 87 | except Exception as e: 88 | write_meta_error(e) 89 | 90 | def __try_commit_or_save(self, session: BaseConnection) -> Optional[str]: 91 | """Attempt to commit, failing that attempt to save. If neither method exists, then the driver doesn't 92 | support it, so not our problem and we can presume user is aware I think.""" 93 | 94 | result = None 95 | try: 96 | if self.commit_label: 97 | result = session.commit(label=self.commit_label) 98 | else: 99 | result = session.commit() 100 | 101 | except ( 102 | NotImplementedError, 103 | AttributeError, 104 | ): # Netmiko uses AttributeError sometimes 105 | # commit not implemented try save_config 106 | try: 107 | session.set_base_prompt() # this is needed if there's any chance you've changed the hostname 108 | result = session.save_config() 109 | except NotImplementedError: 110 | pass 111 | 112 | return result 113 | 114 | def logout(self, session): 115 | try: 116 | response = session.disconnect() 117 | return response 118 | except Exception as e: 119 | write_meta_error(e) 120 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/drivers/puresnmp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/plugins/drivers/puresnmp/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/plugins/drivers/puresnmp/puresnmp_drvr.py: -------------------------------------------------------------------------------- 1 | from puresnmp import puresnmp 2 | 3 | from netpalm.backend.core.driver.netpalm_driver import NetpalmDriver 4 | from netpalm.backend.core.utilities.rediz_meta import write_meta_error 5 | 6 | 7 | class pursnmp(NetpalmDriver): 8 | driver_name = "puresnmp" 9 | 10 | def __init__(self, **kwargs): 11 | self.connection_args = kwargs.get("connection_args", False) 12 | if "port" not in self.connection_args.keys(): 13 | self.connection_args["port"] = 161 14 | if "timeout" not in self.connection_args.keys(): 15 | self.connection_args["timeout"] = 2 16 | self.input_args = kwargs.get("args", False) 17 | if "type" not in self.input_args.keys() or not self.input_args: 18 | self.input_args = {} 19 | self.input_args["type"] = "get" 20 | 21 | def connect(self): 22 | try: 23 | return True 24 | except Exception as e: 25 | write_meta_error(e) 26 | 27 | def sendcommand(self, session=False, command=False): 28 | try: 29 | result = {} 30 | for c in command: 31 | # remove timeout weirdness for tables 32 | if self.input_args["type"] == "table": 33 | response = getattr(puresnmp, self.input_args["type"])( 34 | ip=self.connection_args["host"], 35 | community=self.connection_args["community"], 36 | oid=c, 37 | port=self.connection_args["port"], 38 | ) 39 | else: 40 | response = getattr(puresnmp, self.input_args["type"])( 41 | ip=self.connection_args["host"], 42 | community=self.connection_args["community"], 43 | oid=c, 44 | port=self.connection_args["port"], 45 | timeout=self.connection_args["timeout"], 46 | ) 47 | 48 | # remnder result data for get call 49 | if self.input_args["type"] == "get": 50 | if isinstance(response, bytes): 51 | response = response.decode(errors="ignore") 52 | result[c] = response 53 | # remnder result data for walk call 54 | elif self.input_args["type"] == "walk": 55 | result[c] = [] 56 | for row in response: 57 | oid = str(row[0]) 58 | oid_raw = row[1] 59 | if isinstance(oid_raw, bytes): 60 | oid_raw = oid_raw.decode(errors="ignore") 61 | result[c].append({oid: oid_raw}) 62 | # remnder result data for table call 63 | elif self.input_args["type"] == "table": 64 | result[c] = [] 65 | for key in response[0]: 66 | oid = str(key) 67 | oid_raw = response[0][key] 68 | if isinstance(response[0][key], bytes): 69 | oid_raw = oid_raw.decode(errors="ignore") 70 | result[c].append({oid: oid_raw}) 71 | else: 72 | result[c] = f"{response}" 73 | return result 74 | except Exception as e: 75 | write_meta_error(e) 76 | 77 | def config(self, session=False, command=False, dry_run=False): 78 | try: 79 | return True 80 | except Exception as e: 81 | write_meta_error(e) 82 | 83 | def logout(self, session): 84 | try: 85 | return True 86 | except Exception as e: 87 | write_meta_error(e) 88 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/drivers/restconf/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/plugins/drivers/restconf/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/plugins/drivers/restconf/restconf.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import requests 4 | 5 | from netpalm.backend.core.utilities.rediz_meta import write_meta_error 6 | 7 | from netpalm.backend.core.driver.netpalm_driver import NetpalmDriver 8 | 9 | 10 | class restconf(NetpalmDriver): 11 | driver_name = "restconf" 12 | 13 | def __init__(self, **kwargs): 14 | self.connection_args = kwargs.get("connection_args", False) 15 | self.host = self.connection_args.get("host", False) 16 | del self.connection_args["host"] 17 | self.kwarg = kwargs.get("args", False) 18 | self.port = self.connection_args.get("port", False) 19 | del self.connection_args["port"] 20 | self.default_headers = { 21 | "Content-Type": "application/vnd.yang.data+json", 22 | "Accept": "application/vnd.yang.data+json", 23 | } 24 | self.username = self.connection_args.get("username", False) 25 | del self.connection_args["username"] 26 | self.password = self.connection_args.get("password", False) 27 | del self.connection_args["password"] 28 | self.headers = self.connection_args.get("headers", False) 29 | self.transport = self.connection_args.get("transport", False) 30 | del self.connection_args["transport"] 31 | self.action = self.kwarg.get("action", False) 32 | self.payload = self.kwarg.get("payload", False) 33 | self.params = self.kwarg.get("params", False) 34 | 35 | def connect(self): 36 | try: 37 | if not self.headers: 38 | self.headers = self.default_headers 39 | else: 40 | del self.connection_args["headers"] 41 | return True 42 | except Exception as e: 43 | write_meta_error(e) 44 | 45 | def sendcommand(self, session=False, command=False): 46 | try: 47 | # restconf get call 48 | result = {} 49 | url = ( 50 | self.transport 51 | + "://" 52 | + self.host 53 | + ":" 54 | + str(self.port) 55 | + self.kwarg["uri"] 56 | ) 57 | response = requests.get( 58 | url, 59 | auth=(self.username, self.password), 60 | params=self.params, 61 | headers=self.headers, 62 | **self.connection_args 63 | ) 64 | try: 65 | res = json.loads(response.text) 66 | except Exception: 67 | res = "" 68 | pass 69 | result[url] = {} 70 | result[url]["status_code"] = response.status_code 71 | result[url]["result"] = res 72 | return result 73 | except Exception as e: 74 | write_meta_error(e) 75 | 76 | def config(self, session=False, command=False): 77 | try: 78 | result = {} 79 | url = ( 80 | self.transport 81 | + "://" 82 | + self.host 83 | + ":" 84 | + str(self.port) 85 | + self.kwarg["uri"] 86 | ) 87 | if hasattr(requests, str(self.action)): 88 | response = getattr(requests, str(self.action))( 89 | url, 90 | auth=(self.username, self.password), 91 | data=json.dumps(self.payload), 92 | params=self.params, 93 | headers=self.headers, 94 | **self.connection_args 95 | ) 96 | try: 97 | res = json.loads(response.text) 98 | except Exception: 99 | res = "" 100 | pass 101 | result[url] = {} 102 | result[url]["status_code"] = response.status_code 103 | result[url]["result"] = res 104 | return result 105 | else: 106 | raise Exception(self.action + " not found in requests") 107 | except Exception as e: 108 | write_meta_error(e) 109 | 110 | def logout(self, session): 111 | try: 112 | return True 113 | except Exception as e: 114 | write_meta_error(e) 115 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/custom_scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/plugins/extensibles/custom_scripts/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/custom_scripts/hello_world.py: -------------------------------------------------------------------------------- 1 | # all functions need to be wrapped in the "run" function and pass in kwargs 2 | # JSON example to send into the /script route is as below 3 | # 4 | # { 5 | # "script":"hello_world", 6 | # "args":{ 7 | # "hello":"world" 8 | # } 9 | # } 10 | # 11 | def run(**kwargs): 12 | try: 13 | # mandatory get of kwargs - payload comes through as {"kwargs": {"hello": "world"}} 14 | args = kwargs.get("kwargs") 15 | # access your vars here in a dict format - payload is now {"hello": "world"} 16 | world = args["hello"] 17 | # reutn "world" 18 | return world 19 | except KeyError as e: 20 | raise Exception(f"Required args: {e}") 21 | except Exception as e: 22 | raise Exception(e) 23 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/custom_scripts/hello_world_advanced_using_netpalm_manager.py: -------------------------------------------------------------------------------- 1 | from netpalm.backend.core.manager.netpalm_manager import NetpalmManager 2 | 3 | import logging 4 | 5 | # all functions need to be wrapped in the "run" function and pass in kwargs 6 | # JSON example to send into the /script route is as below 7 | # 8 | # { 9 | # "script": "hello_world_advanced_using_netpalm_manager", 10 | # "args": { 11 | # "host": "10.0.2.33", 12 | # "username": "admin", 13 | # "password": "admin" 14 | # }, 15 | # "queue_strategy": "fifo" 16 | # } 17 | # 18 | 19 | log = logging.getLogger(__name__) 20 | 21 | 22 | def run(**kwargs): 23 | try: 24 | # mandatory get of kwargs - payload comes through as {"kwargs": {"host": "10.0.2.33", "username": "admin", "password": "admin"}} 25 | args = kwargs.get("kwargs") 26 | # access your passed in vars here in a dict format - payload is now {"host": "10.0.2.33", "username": "admin", "password": "admin"} 27 | username = args["username"] 28 | password = args["password"] 29 | host = args["host"] 30 | 31 | log.info(f"hello_world_advanced_using_netpalm_manager: recieved {args}") 32 | 33 | # we will now use the netpalm manager to interface with netpalm 34 | # this section shows how to prepare a payload to send into netpalm 35 | netmiko_send_data = { 36 | "library": "netmiko", 37 | "connection_args": { 38 | "device_type": "cisco_ios", 39 | "host": host, 40 | "username": username, 41 | "password": password, 42 | "timeout": 5 43 | }, 44 | "command": "show run | i hostname", 45 | "queue_strategy": "pinned" 46 | } 47 | mgr = NetpalmManager() 48 | job_result = mgr.get_config_netmiko(netmiko_send_data) 49 | return_result = mgr.retrieve_task_result(job_result) 50 | 51 | log.info(f"hello_world_advanced_using_netpalm_manager: got back from task {return_result}") 52 | 53 | # we can also trigger webhooks from within the script if required using the manager as below 54 | # webhooks can also be triggered outside of the script by simply using the webhook key against the REST API 55 | webhook_meta = { 56 | "name": "default_webhook", 57 | "args": { 58 | "insert": "something useful" 59 | } 60 | } 61 | 62 | log.info(f"hello_world_advanced_using_netpalm_manager: triggering webhook with {webhook_meta}") 63 | 64 | mgr.trigger_webhook(webhook_meta_data=webhook_meta, webhook_payload=return_result) 65 | 66 | if return_result["data"]["task_result"]: 67 | if "bufoon" in return_result["data"]["task_result"]: 68 | return "MeowCat" 69 | else: 70 | return "MeowMeow" 71 | else: 72 | return "MeowMeow" 73 | 74 | except KeyError as e: 75 | raise Exception(f"Required args: {e}") 76 | except Exception as e: 77 | raise Exception(e) 78 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/custom_scripts/hello_world_embedded_pydanticmodel.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Any, List 2 | from netpalm.backend.core.models.models import ScriptCustom 3 | 4 | class MyCustomScriptModel(ScriptCustom): 5 | script: str 6 | test: Optional[str] = None 7 | 8 | def run(payload: MyCustomScriptModel): 9 | try: 10 | # mandatory get of kwargs - payload comes through as {"kwargs": {"hello": "world"}} 11 | args = payload.test 12 | # reutn "world" 13 | return args 14 | except Exception as e: 15 | raise Exception(e) 16 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/custom_scripts/hello_world_model.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from pydantic import BaseModel 4 | 5 | from netpalm.backend.core.models.models import QueueStrategy 6 | from netpalm.backend.core.models.models import Webhook 7 | 8 | # 9 | # 10 | # !!!! WARNING, this is considered a legacy way of doing this, check the example hello_world_embedded_pydanticmodel.py 11 | # for an updated method of defining models within the same file as the script !!! 12 | # 13 | # 14 | 15 | class hello_world_model_args(BaseModel): 16 | # your model goes here! 17 | hello: str 18 | 19 | class hello_world_model(BaseModel): 20 | # this class MUST match the filename & the filename must be formatted $servicetemplatename_model.py 21 | script: str 22 | args: hello_world_model_args 23 | queue_strategy: Optional[QueueStrategy] = None 24 | webhook: Optional[Webhook] = None 25 | 26 | class Config: 27 | # add an example payload under the "example" dict 28 | schema_extra = { 29 | "example": { 30 | "script": "hello_world", 31 | "args": { 32 | "hello": "world" 33 | }, 34 | "queue_strategy": "fifo" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/custom_scripts/hello_world_test.py: -------------------------------------------------------------------------------- 1 | # all functions need to be wrapped in the "run" function and pass in kwargs 2 | # JSON example to send into the /script route is as below 3 | # 4 | # { 5 | # "script":"hello_world", 6 | # "args":{ 7 | # "hello":"world" 8 | # } 9 | # } 10 | # 11 | def run(**kwargs): 12 | try: 13 | # mandatory get of kwargs - payload comes through as {"kwargs": {"hello": "world"}} 14 | args = kwargs.get("kwargs") 15 | # access your vars here in a dict format - payload is now {"hello": "world"} 16 | world = args["hello"] 17 | # reutn "world" 18 | return world 19 | except KeyError as e: 20 | raise Exception(f"Required args: {e}") 21 | except Exception as e: 22 | raise Exception(e) 23 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/custom_webhooks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/plugins/extensibles/custom_webhooks/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/custom_webhooks/default_webhook.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import requests 4 | 5 | from netpalm.backend.core.confload.confload import config 6 | 7 | """ 8 | default webhook and boilerplate webhook template 9 | 10 | IMPORTANT NOTES: 11 | - webhook function name must be "run_webhook" 12 | input params: 13 | - "netpalm_task_result" (dict) 14 | - **kwargs (dict) 15 | 16 | """ 17 | def run_webhook(payload=False): 18 | try: 19 | if payload: 20 | # convert to json 21 | pl = json.dumps(payload) 22 | #prepare requests data 23 | url_val = config.default_webhook_url 24 | headers_val = config.default_webhook_headers 25 | verify_val = config.default_webhook_ssl_verify 26 | timeout_val = config.default_webhook_timeout 27 | pl = pl 28 | #execute request 29 | response = requests.request("POST", url=url_val, headers=headers_val, verify=verify_val, timeout=timeout_val, data=pl) 30 | if str(response.status_code)[:1] != "2": 31 | return False 32 | else: 33 | return True 34 | else: 35 | return False 36 | except Exception as e: 37 | return e -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/custom_webhooks/elastic.py: -------------------------------------------------------------------------------- 1 | import json 2 | import requests 3 | import uuid 4 | 5 | import logging 6 | from netpalm.backend.core.confload.confload import config 7 | 8 | from datetime import datetime 9 | import re 10 | 11 | """ 12 | netpalm webhook for posting a document directly to an elasticsearch index 13 | IMPORTANT NOTES: 14 | webook requires a payload as per below 15 | "webhook": { 16 | "name": "elastic", 17 | "args": { 18 | "username": "elastic", 19 | "password": "changeme", 20 | "index": "test", 21 | "elastic_instance": "http://127.0.0.1:9200" 22 | } 23 | } 24 | """ 25 | 26 | log = logging.getLogger(__name__) 27 | 28 | 29 | def run_webhook(payload=False): 30 | try: 31 | if payload: 32 | log.info(f"run webhook: running elastic webhook") 33 | # set variables for POST 34 | password = payload["webhook_args"]["password"] 35 | username = payload["webhook_args"]["username"] 36 | index = payload["webhook_args"]["index"] 37 | elastic_instance = payload["webhook_args"]["elastic_instance"] 38 | del payload["webhook_args"] 39 | 40 | headers_val = config.default_webhook_headers 41 | verify_val = config.default_webhook_ssl_verify 42 | timeout_val = config.default_webhook_timeout 43 | 44 | # prepare time for elastic format 45 | now = datetime.now() 46 | time = now.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] 47 | timez = f"{time}Z" 48 | 49 | # append elastic attrs 50 | my_id = uuid.uuid4().hex 51 | 52 | payload["@version"] = 1 53 | payload["@timestamp"] = timez 54 | 55 | pl = json.dumps(payload) 56 | 57 | def cleanup_crappy_string_vals(payload): 58 | payload = re.sub(r'\"(\d+)\"', r'\1', f"{payload}") 59 | payload = re.sub(r'\"(\d+\.\d+)\"', r'\1', f"{payload}") 60 | return payload 61 | 62 | def cleanup_crappy_router_null_outputs(pattern, value, payload): 63 | # hack some weird datatyping shit in tfsm 64 | empty_string_fields = re.findall(r'\"(\w*)\":\s\"\"', payload) 65 | if len(empty_string_fields) >= 1: 66 | empty_string_fields = list(dict.fromkeys(empty_string_fields)) 67 | # do some magic 68 | for empty_string_key in empty_string_fields: 69 | int_regex = "\""+re.escape(empty_string_key)+"\": "+pattern 70 | is_int = re.findall(int_regex, payload) 71 | if len(is_int) >= 1: 72 | payload = re.sub("\""+re.escape(empty_string_key)+"\": \"\"", "\""+re.escape(empty_string_key)+f"\": {value}", f"{payload}") 73 | return payload 74 | 75 | pl = cleanup_crappy_string_vals(payload=pl) 76 | pl = cleanup_crappy_router_null_outputs(pattern="\d+", value=0, payload=pl) 77 | pl = cleanup_crappy_router_null_outputs(pattern="\d+\.\d+", value=0.00, payload=pl) 78 | 79 | # execute request 80 | response = requests.request("POST", url=f"{elastic_instance}/{index}/{index}/{my_id}", 81 | headers=headers_val, 82 | verify=verify_val, 83 | timeout=timeout_val, 84 | data=pl, 85 | auth=(username, password) 86 | ) 87 | if str(response.status_code)[:1] != "2": 88 | return False 89 | else: 90 | return True 91 | else: 92 | return False 93 | except Exception as e: 94 | log.error(f"elastic webhook: {e}") 95 | return e 96 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/custom_webhooks/servicenow_request_item_patch_netpalm_webhook.py: -------------------------------------------------------------------------------- 1 | import json 2 | import requests 3 | 4 | import logging 5 | from netpalm.backend.core.confload.confload import config 6 | 7 | """ 8 | netpalm webhook for updating a request item state and worknotes 9 | IMPORTANT NOTES: 10 | webook requires a payload as per below 11 | "webhook": { 12 | "name": "servicenow_request_item_patch_netpalm_webhook", 13 | "args": { 14 | "username": "admin", 15 | "password": "", 16 | "sys_id": "5d558ebf07679010430dff4c7c1ed036", 17 | "servicenow_instance": "dev98005.service-now.com" 18 | } 19 | } 20 | """ 21 | 22 | log = logging.getLogger(__name__) 23 | 24 | 25 | def run_webhook(payload=False): 26 | try: 27 | if payload: 28 | log.info(f"run webhook: running servicenow webhook") 29 | # set variables for POST 30 | password = payload["webhook_args"]["password"] 31 | username = payload["webhook_args"]["username"] 32 | sys_id = payload["webhook_args"]["sys_id"] 33 | servicenow_instance = payload["webhook_args"]["servicenow_instance"] 34 | # generate clean payload for snow 35 | pl = {} 36 | if len(payload["data"]["task_errors"]) > 0: 37 | pl["state"] = "4" 38 | else: 39 | pl["state"] = "3" 40 | pl["work_notes"] = json.dumps(payload["data"]["task_result"], indent=3) 41 | # convert to json 42 | # Prepare requests data 43 | headers_val = config.default_webhook_headers 44 | verify_val = config.default_webhook_ssl_verify 45 | timeout_val = config.default_webhook_timeout 46 | # execute request 47 | response = requests.request("PATCH", url=f"https://{servicenow_instance}/api/now/table/sc_req_item/{sys_id}", 48 | headers=headers_val, 49 | verify=verify_val, 50 | timeout=timeout_val, 51 | json=pl, 52 | auth=(username, password) 53 | ) 54 | if str(response.status_code)[:1] != "2": 55 | return False 56 | else: 57 | return True 58 | else: 59 | return False 60 | except Exception as e: 61 | log.error(f"run webhook: {e}") 62 | return e 63 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/j2_config_templates/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/plugins/extensibles/j2_config_templates/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/j2_config_templates/cisco_ios_add_ip_int.j2: -------------------------------------------------------------------------------- 1 | {% for int in interfaces -%} 2 | int {{int}} 3 | ip address {{interfaces[int]}} 4 | {% endfor %} -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/j2_config_templates/cisco_ios_add_snmp_config.j2: -------------------------------------------------------------------------------- 1 | snmp-server community {{snmp_community}} 2 | snmp-server location {{snmp_location}} 3 | snmp-server contact {{snmp_contact}} 4 | 5 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/j2_config_templates/cisco_ios_create_vlans.j2: -------------------------------------------------------------------------------- 1 | {% for vlan in vlans -%} 2 | int vlan {{vlan}} 3 | no shut 4 | {% endfor -%} -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/j2_config_templates/cisco_ios_remove_ip_int.j2: -------------------------------------------------------------------------------- 1 | {% for int in interfaces -%} 2 | int {{int}} 3 | no ip address {{interfaces[int]}} 4 | {% endfor %} -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/j2_config_templates/cisco_ios_remove_snmp_config.j2: -------------------------------------------------------------------------------- 1 | no snmp-server community {{snmp_community}} 2 | no snmp-server location {{snmp_location}} 3 | no snmp-server contact {{snmp_contact}} 4 | 5 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/j2_config_templates/cisco_ios_remove_vlans.j2: -------------------------------------------------------------------------------- 1 | {% for vlan in vlans -%} 2 | no int vlan {{vlan}} 3 | {% endfor -%} -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/j2_config_templates/cisco_ios_shut_interfaces.j2: -------------------------------------------------------------------------------- 1 | {% for interface in interfaces -%} 2 | int {{interface}} 3 | shut 4 | {% endfor -%} -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/j2_config_templates/cisco_ios_un_shut_interfaces.j2: -------------------------------------------------------------------------------- 1 | {% for interface in interfaces -%} 2 | int {{interface}} 3 | no shut 4 | {% endfor -%} -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/j2_config_templates/ncclient_test.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | <__XML__MODE__exec_configure> 4 | 5 | 6 | helloworld 7 | <__XML__MODE_if-ethernet-switch> 8 | 9 | 10 | 11 | 12 | <__XML__BLK_Cmd_switchport_trunk_allowed_allow-vlans> 13 | {{vlans|join(',')}} 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/j2_config_templates/test.j2: -------------------------------------------------------------------------------- 1 | {% for vlan in vlans -%} 2 | int vlan {{vlan}} 3 | no shut 4 | {% endfor -%} -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/j2_webhook_templates/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/plugins/extensibles/j2_webhook_templates/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/j2_webhook_templates/example.j2: -------------------------------------------------------------------------------- 1 | {# EXAMPLE OF USING A j2 to transform a webhook payload shown below 2 | { 3 | "status": "success", 4 | "data": { 5 | "task_id": "4f93000e-153d-40fa-a1dc-a2b7ba4cc00f", 6 | "created_on": "07/16/2020, 09:55:40", 7 | "task_queue": "fifo", 8 | "task_status": "finished", 9 | "task_result": { 10 | "show run | i hostname": [ 11 | "hostname with_great_power_comes_great" 12 | ] 13 | } 14 | }, 15 | "webhook_args": { 16 | "insert": "something useful" 17 | } 18 | } 19 | #} 20 | { 21 | "task_id": "{{ data["task_id"] }}", 22 | "task_status": "{{ data["task_status"] }}" 23 | } -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/services/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/backend/plugins/extensibles/services/__init__.py -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/services/example_simple.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from pydantic import BaseModel 4 | from netpalm.backend.core.calls.service.netpalmservice import NetpalmService 5 | from netpalm.backend.core.manager.netpalm_manager import NetpalmManager 6 | 7 | log = logging.getLogger(__name__) 8 | 9 | 10 | class NetpalmUserServiceModel(BaseModel): 11 | hostname: str 12 | 13 | 14 | class NetpalmUserService(NetpalmService): 15 | 16 | mgr = NetpalmManager() 17 | model = NetpalmUserServiceModel 18 | 19 | def create(self, model_data: model): 20 | log.info(f"netpalm service: made it with your {model_data.hostname}!") 21 | log.info(f"netpalm service: service created {self.service_id}!") 22 | 23 | netmiko_send_data = { 24 | "library": "netmiko", 25 | "connection_args": { 26 | "device_type": "cisco_ios", 27 | "host": model_data.hostname, 28 | "username": "admin", 29 | "password": "admin", 30 | "timeout": 5, 31 | }, 32 | "command": "show run | i hostname", 33 | "queue_strategy": "pinned", 34 | } 35 | job_result = self.mgr.get_config_netmiko(netmiko_send_data) 36 | return_result = self.mgr.retrieve_task_result(job_result) 37 | self.mgr.set_service_instance_status(self.service_id, state="errored") 38 | return return_result 39 | 40 | def update(self, model: model): 41 | log.info(f"netpalm service: update method not implemented on your service") 42 | pass 43 | 44 | def delete(self, model: model): 45 | log.info(f"netpalm service: delete method not implemented on your service") 46 | pass 47 | 48 | def re_deploy(self, model: model): 49 | log.info(f"netpalm service: re_deploy method not implemented on your service") 50 | pass 51 | 52 | def validate(self, model: model): 53 | log.info(f"netpalm service: validate method not implemented on your service") 54 | pass 55 | 56 | def health_check(self, model: model): 57 | log.info( 58 | f"netpalm service: health_check method not implemented on your service" 59 | ) 60 | pass 61 | -------------------------------------------------------------------------------- /netpalm/backend/plugins/extensibles/ttp_templates/example.ttp: -------------------------------------------------------------------------------- 1 | 2 | Cisco {{ios}} Software, 3700 Software (C3745-ADVIPSERVICESK9-M), Version 12.4(25d), RELEASE SOFTWARE (fc1) 3 | -------------------------------------------------------------------------------- /netpalm/controller_addtl_requirements.txt: -------------------------------------------------------------------------------- 1 | fastapi 2 | uvicorn 3 | uvloop 4 | httptools 5 | gunicorn 6 | pytest 7 | pytest-timeout 8 | pytest-mock 9 | aiofiles -------------------------------------------------------------------------------- /netpalm/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | class NetpalmError(Exception): 3 | """Baseclass for all netpalm errors""" 4 | pass 5 | 6 | 7 | class NetpalmDriverError(NetpalmError): 8 | """Errors related to driver plugins""" 9 | pass 10 | 11 | 12 | class NetpalmCheckError(NetpalmError): 13 | """Errors due to pre or post check validation failure""" 14 | pass 15 | 16 | 17 | class NetpalmMetaProcessedException(NetpalmError): 18 | pass 19 | -------------------------------------------------------------------------------- /netpalm/netpalm_controller.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import filelock 4 | # load fast api 5 | 6 | 7 | from fastapi import FastAPI, Depends 8 | from fastapi.openapi.docs import get_swagger_ui_html 9 | from fastapi.openapi.utils import get_openapi 10 | from fastapi.staticfiles import StaticFiles 11 | from starlette.responses import JSONResponse 12 | 13 | from netpalm.backend.core.confload.confload import config 14 | from netpalm.backend.core.security.get_api_key import get_api_key 15 | from netpalm.netpalm_worker_common import start_broadcast_listener_process 16 | from netpalm.routers import getconfig, setconfig, task, template, script, service, util, public, schedule 17 | 18 | log = logging.getLogger(__name__) 19 | 20 | config.setup_logging(max_debug=True) 21 | 22 | app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None) 23 | 24 | app.mount("/static", StaticFiles(directory="netpalm/static"), name="static") 25 | 26 | app.include_router(getconfig.router, dependencies=[Depends(get_api_key)]) 27 | app.include_router(setconfig.router, dependencies=[Depends(get_api_key)]) 28 | app.include_router(task.router, dependencies=[Depends(get_api_key)]) 29 | app.include_router(template.router, dependencies=[Depends(get_api_key)]) 30 | app.include_router(script.router, dependencies=[Depends(get_api_key)]) 31 | app.include_router(service.router, dependencies=[Depends(get_api_key)]) 32 | app.include_router(util.router, dependencies=[Depends(get_api_key)]) 33 | app.include_router(schedule.router, dependencies=[Depends(get_api_key)]) 34 | app.include_router(public.router) 35 | 36 | broadcast_worker_lock = filelock.FileLock("broadcast_worker_lock") 37 | try: 38 | broadcast_worker_lock.acquire(timeout=0.01) 39 | with broadcast_worker_lock: 40 | log.info(f"Creating broadcast listener because I got the lock!") 41 | start_broadcast_listener_process() 42 | except filelock.Timeout: 43 | log.info(f"skipping broadcast listener creation because I couldn't get the lock") 44 | 45 | 46 | # swaggerui routers 47 | @app.get("/swaggerfile", tags=["swagger file"], include_in_schema=False) 48 | async def get_open_api_endpoint(): 49 | response = JSONResponse( 50 | get_openapi(title="netpalm", version="0.4", routes=app.routes) 51 | ) 52 | return response 53 | 54 | 55 | @app.get("/", tags=["swaggerui"], include_in_schema=False) 56 | async def get_documentation(): 57 | response = get_swagger_ui_html( 58 | openapi_url="/swaggerfile", 59 | title="docs", 60 | # swagger_js_url="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.34.0/swagger-ui-bundle.min.js", 61 | swagger_js_url="/static/js/swagger-ui-bundle.min.js", 62 | swagger_css_url="/static/css/swagger-ui.css", 63 | ) 64 | return response -------------------------------------------------------------------------------- /netpalm/netpalm_fifo_worker.py: -------------------------------------------------------------------------------- 1 | from multiprocessing import Process 2 | import time 3 | import sys 4 | 5 | import logging 6 | 7 | from .backend.core.confload.confload import config 8 | from .netpalm_worker_common import start_broadcast_listener_process 9 | from .backend.core.utilities.rediz_worker_controller import RedisWorker, RedisFifoWorker 10 | 11 | config.setup_logging(max_debug=True) 12 | log = logging.getLogger(__name__) 13 | 14 | def fifo_worker(queue, counter): 15 | try: 16 | wr = RedisFifoWorker(config, queue, counter) 17 | wr.listen() 18 | except Exception as e: 19 | return e 20 | 21 | 22 | def fifo_worker_constructor(queue): 23 | try: 24 | start_broadcast_listener_process() 25 | for i in range(config.fifo_process_per_node): 26 | p = Process(target=fifo_worker, args=(queue, i,)) 27 | p.start() 28 | while True: 29 | time.sleep(99999999) 30 | finally: 31 | cleanup = RedisWorker(config) 32 | cleanup.worker_cleanup() 33 | sys.exit() 34 | 35 | 36 | def start_worker(): 37 | fifo_worker_constructor(config.redis_fifo_q) 38 | -------------------------------------------------------------------------------- /netpalm/netpalm_pinned_worker.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from multiprocessing import Process 3 | import sys 4 | import time 5 | 6 | from .backend.core.confload.confload import config 7 | from .netpalm_worker_common import start_broadcast_listener_process 8 | from .backend.core.utilities.rediz_worker_controller import RedisWorker, RedisPinnedWorker, RedisProcessWorker 9 | 10 | config.setup_logging(max_debug=True) 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | def start_processworkerprocess(): 15 | try: 16 | """process to run the processworker core function""" 17 | p = Process(target=processworker) 18 | p.start() 19 | while True: 20 | time.sleep(99999999) 21 | finally: 22 | cleanup = RedisWorker(config) 23 | cleanup.worker_cleanup() 24 | sys.exit() 25 | 26 | 27 | def we_are_controller(): 28 | import sys 29 | for part in sys.argv: 30 | if 'controller' in part: 31 | return True 32 | return False 33 | 34 | 35 | def processworker(): 36 | """ 37 | listens on the core queue for messages from the controller, 38 | single processesworker runs per controller. 39 | used to create new processes on demand as needed 40 | """ 41 | if not we_are_controller(): 42 | start_broadcast_listener_process() 43 | wr = RedisProcessWorker(config) 44 | wr.listen() 45 | 46 | 47 | def pinned_worker(queue): 48 | try: 49 | wr = RedisPinnedWorker(config, queue) 50 | wr.listen() 51 | except Exception as e: 52 | return e 53 | 54 | 55 | def pinned_worker_constructor(queue): 56 | """process constructor to run the pinned_worker""" 57 | p = Process(target=pinned_worker, args=(queue,)) 58 | p.start() 59 | 60 | 61 | if __name__ == '__main__': 62 | start_processworkerprocess() 63 | -------------------------------------------------------------------------------- /netpalm/requirements.txt: -------------------------------------------------------------------------------- 1 | fastapi 2 | ttp 3 | netmiko==3.3.2 4 | napalm 5 | ncclient==0.6.9 6 | requests 7 | redis==4.5.1 8 | rq 9 | xmltodict 10 | jinja2 11 | jinja2schema 12 | jsonschema 13 | genie 14 | pyyaml 15 | cachelib==0.3.0 16 | python-redis-lock 17 | filelock 18 | jsonpath_ng 19 | apscheduler==3.6.3 20 | puresnmp==1.9.1 21 | pydantic==1.10.13 22 | names_generator==0.1.0 23 | -------------------------------------------------------------------------------- /netpalm/routers/getconfig.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from fastapi import APIRouter 4 | 5 | # load models 6 | from netpalm.backend.core.models.models import GetConfig 7 | from netpalm.backend.core.models.napalm import NapalmGetConfig 8 | from netpalm.backend.core.models.ncclient import NcclientGet 9 | from netpalm.backend.core.models.ncclient import NcclientGetConfig 10 | from netpalm.backend.core.models.netmiko import NetmikoGetConfig 11 | from netpalm.backend.core.models.puresnmp import PureSNMPGetConfig 12 | from netpalm.backend.core.models.restconf import Restconf 13 | from netpalm.backend.core.models.task import Response 14 | 15 | from netpalm.backend.core.manager import ntplm 16 | 17 | from netpalm.routers.route_utils import error_handle_w_cache, whitelist 18 | 19 | log = logging.getLogger(__name__) 20 | router = APIRouter() 21 | 22 | 23 | # read config 24 | @router.post("/getconfig", response_model=Response, status_code=201) 25 | @router.post("/get", response_model=Response, status_code=201) 26 | @error_handle_w_cache 27 | @whitelist 28 | def get_config(getcfg: GetConfig): 29 | return ntplm._get_config(getcfg) 30 | 31 | 32 | # read config 33 | @router.post("/getconfig/netmiko", response_model=Response, status_code=201) 34 | @router.post("/get/netmiko", response_model=Response, status_code=201) 35 | @error_handle_w_cache 36 | @whitelist 37 | def get_config_netmiko(getcfg: NetmikoGetConfig): 38 | return ntplm.get_config_netmiko(getcfg) 39 | 40 | 41 | # read config 42 | @router.post("/getconfig/napalm", response_model=Response, status_code=201) 43 | @router.post("/get/napalm", response_model=Response, status_code=201) 44 | @error_handle_w_cache 45 | @whitelist 46 | def get_config_napalm(getcfg: NapalmGetConfig): 47 | return ntplm.get_config_napalm(getcfg) 48 | 49 | 50 | # read config 51 | @router.post("/getconfig/puresnmp", response_model=Response, status_code=201) 52 | @router.post("/get/puresnmp", response_model=Response, status_code=201) 53 | @error_handle_w_cache 54 | @whitelist 55 | def get_config_puresnmp(getcfg: PureSNMPGetConfig): 56 | return ntplm.get_config_puresnmp(getcfg) 57 | 58 | 59 | # read config 60 | @router.post("/getconfig/ncclient", response_model=Response, status_code=201) 61 | @router.post("/get/ncclient", response_model=Response, status_code=201) 62 | @error_handle_w_cache 63 | @whitelist 64 | def get_config_ncclient(getcfg: NcclientGetConfig): 65 | return ntplm.get_config_ncclient(getcfg) 66 | 67 | 68 | # ncclient Manager.get() rpc call 69 | # Certain device types dont have rpc methods defined in ncclient. 70 | # This is a work around for that. 71 | @router.post("/getconfig/ncclient/get", 72 | response_model=Response, 73 | status_code=201) 74 | @router.post("/get/ncclient/get", 75 | response_model=Response, 76 | status_code=201) 77 | @error_handle_w_cache 78 | @whitelist 79 | def ncclient_get(getcfg: NcclientGet, library: str = "ncclient"): 80 | return ntplm.ncclient_get(getcfg, library) 81 | 82 | 83 | # read config 84 | @router.post("/getconfig/restconf", response_model=Response, status_code=201) 85 | @router.post("/get/restconf", response_model=Response, status_code=201) 86 | @error_handle_w_cache 87 | @whitelist 88 | def get_config_restconf(getcfg: Restconf): 89 | return ntplm.get_config_restconf(getcfg) 90 | -------------------------------------------------------------------------------- /netpalm/routers/public.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, HTTPException 2 | 3 | #load config 4 | 5 | router = APIRouter() 6 | 7 | #utility route - denied 8 | @router.get("/denied") 9 | async def denied(): 10 | raise HTTPException(status_code=403, detail="forbidden") -------------------------------------------------------------------------------- /netpalm/routers/schedule.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | from fastapi import APIRouter, HTTPException 4 | from fastapi.encoders import jsonable_encoder 5 | 6 | from netpalm.backend.core.models.task import ResponseBasic 7 | from netpalm.backend.core.models.models import ScheduleInterval 8 | 9 | from netpalm.backend.core.schedule import sched 10 | 11 | router = APIRouter() 12 | 13 | 14 | @router.get("/schedule/", response_model=ResponseBasic) 15 | def get_scheduled_tasks_list(): 16 | try: 17 | r = sched.get_scheduled_jobs() 18 | resp = jsonable_encoder(r) 19 | return resp 20 | except Exception as e: 21 | raise HTTPException(status_code=500) 22 | 23 | 24 | @router.post("/schedule/{name}", status_code=201) 25 | def add_scheduled_task(name: str, schedul: ScheduleInterval): 26 | try: 27 | data = schedul.dict(exclude_none=True) 28 | pl = data["schedule_payload"] 29 | del data["schedule_payload"] 30 | 31 | r = sched.add_netpalm_job( 32 | job_name=name, 33 | input_payload=pl, 34 | trigger="interval", 35 | trigger_args=data 36 | ) 37 | resp = jsonable_encoder(r) 38 | return resp 39 | except Exception as e: 40 | raise HTTPException(status_code=500, detail=str(e).split('\n')) 41 | 42 | 43 | @router.patch("/schedule/{id}", status_code=204) 44 | def modify_scheduled_task(id: str, schedul: ScheduleInterval): 45 | try: 46 | data = schedul.dict(exclude_none=True) 47 | pl = data["schedule_payload"] 48 | del data["schedule_payload"] 49 | r = sched.modify_netpalm_job( 50 | job_id=id, 51 | input_payload=pl, 52 | trigger="interval", 53 | trigger_args=data 54 | ) 55 | resp = jsonable_encoder(r) 56 | return resp 57 | except Exception as e: 58 | raise HTTPException(status_code=500, detail=str(e).split('\n')) 59 | 60 | 61 | @router.delete("/schedule/{id}", status_code=204) 62 | def remove_scheduled_task(id: str): 63 | try: 64 | r = sched.remove_job(id) 65 | except Exception as e: 66 | raise HTTPException(status_code=500, detail=str(e).split("\n")) 67 | -------------------------------------------------------------------------------- /netpalm/routers/script.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | 3 | import inspect 4 | 5 | import logging 6 | 7 | from fastapi import APIRouter 8 | from fastapi.encoders import jsonable_encoder 9 | 10 | from pydantic import BaseModel 11 | 12 | from netpalm.backend.core.confload.confload import config 13 | 14 | # load models 15 | from netpalm.backend.core.models.models import Script, ScriptCustom 16 | from netpalm.backend.core.models.task import Response 17 | from netpalm.backend.core.models.task import ResponseBasic 18 | 19 | from netpalm.backend.core.routes.routes import routes 20 | from netpalm.routers.route_utils import HttpErrorHandler 21 | 22 | from netpalm.backend.core.manager import ntplm 23 | 24 | from netpalm.backend.core.calls.scriptrunner.script import script_model_finder 25 | 26 | from netpalm.routers.route_utils import error_handle_w_cache 27 | 28 | router = APIRouter() 29 | 30 | log = logging.getLogger(__name__) 31 | 32 | # get template list 33 | @router.get("/script", response_model=ResponseBasic) 34 | @HttpErrorHandler() 35 | async def list_scripts(): 36 | r = routes["ls"](fldr="script") 37 | resp = jsonable_encoder(r) 38 | return resp 39 | 40 | 41 | @router.post("/script", response_model=Response, status_code=201) 42 | @error_handle_w_cache 43 | def execute_script(script: Script): 44 | if isinstance(script, dict): 45 | req_data = script 46 | else: 47 | req_data = script.dict(exclude_none=True) 48 | return ntplm.execute_script(**req_data) 49 | 50 | 51 | r = routes["ls"](fldr="script") 52 | for script in r["data"]["task_result"]["templates"]: 53 | model = script_model_finder(script_name=script)[0] 54 | 55 | @router.post(f"/script/v1/{script}", response_model=Response, status_code=201) 56 | @error_handle_w_cache 57 | def execute_script(script: model): 58 | if isinstance(script, dict): 59 | req_data = script 60 | else: 61 | req_data = script.dict(exclude_none=True) 62 | return ntplm.execute_script(**req_data) 63 | 64 | 65 | # get template list 66 | @router.get("/webhook", response_model=ResponseBasic) 67 | @HttpErrorHandler() 68 | async def list_webhooks(): 69 | r = routes["ls"](fldr="webhook_script") 70 | resp = jsonable_encoder(r) 71 | return resp 72 | -------------------------------------------------------------------------------- /netpalm/routers/service.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import inspect 3 | 4 | from typing import Any 5 | 6 | import logging 7 | 8 | from pydantic import BaseModel 9 | 10 | from fastapi import APIRouter, Request, HTTPException 11 | 12 | from netpalm.backend.core.confload.confload import config 13 | 14 | # load models 15 | from netpalm.backend.core.models.service import ServiceModel, ServiceInventoryResponse 16 | from netpalm.backend.core.models.task import ServiceResponse, Response, ResponseBasic 17 | 18 | from netpalm.backend.core.manager import ntplm 19 | 20 | 21 | from netpalm.backend.core.calls.service.procedures import get_service 22 | 23 | # load routes 24 | from netpalm.backend.core.routes.routes import routes 25 | from netpalm.routers.route_utils import HttpErrorHandler 26 | 27 | router = APIRouter() 28 | 29 | log = logging.getLogger(__name__) 30 | 31 | 32 | # @router.get("/service/instances/", response_model=ServiceInventoryResponse) 33 | @router.get("/service/instances/") 34 | def list_service_instances(): 35 | res = ntplm.list_service_instances() 36 | if res["data"]["task_result"] is None: 37 | raise HTTPException( 38 | status_code=404, 39 | detail=ResponseBasic(status="success", data={"task_result": None}).dict(), 40 | ) 41 | return res 42 | 43 | 44 | @router.get("/service/instance/{service_id}") 45 | def get_service_instance(service_id: str): 46 | res = ntplm.get_service_instance(service_id) 47 | if res: 48 | return res 49 | else: 50 | raise HTTPException( 51 | status_code=404, 52 | detail=ResponseBasic( 53 | status="success", data={"task_result": f"{service_id} not found"} 54 | ).dict(), 55 | ) 56 | 57 | 58 | r = routes["ls"](fldr="service") 59 | for service_model in r["data"]["task_result"]["templates"]: 60 | try: 61 | model_name = f"{service_model}" 62 | model = get_service(model_name)["service_model"] 63 | except Exception as e: 64 | log.error( 65 | f"dynamic_service_route: no model found for {service_model} import error {e}" 66 | ) 67 | model = ServiceModel 68 | 69 | @router.post( 70 | f"/service/instance/create/{service_model}", 71 | response_model=ServiceResponse, 72 | status_code=201, 73 | ) 74 | @HttpErrorHandler() 75 | def create_service_instance(service: model, request: Request): 76 | # url hack 77 | service_model_name = f"{request.url.path}".split("/")[-1] 78 | return ntplm.create_new_service_instance(service_model_name, service) 79 | 80 | @router.patch( 81 | f"/service/instance/update/{service_model}" + "/{service_id}", 82 | response_model=Response, 83 | status_code=201, 84 | ) 85 | def update_service_instance_state( 86 | service: model, service_id: str, request: Request 87 | ): 88 | res = ntplm.update_service_instance(service_id, service) 89 | if res: 90 | return res 91 | else: 92 | raise HTTPException( 93 | status_code=404, 94 | detail=ResponseBasic( 95 | status="success", data={"task_result": f"{service_id} not found"} 96 | ).dict(), 97 | ) 98 | 99 | 100 | @router.post( 101 | "/service/instance/delete/{service_id}", response_model=Response, status_code=201 102 | ) 103 | @HttpErrorHandler() 104 | def delete_service_instance_state(service_id: str): 105 | return ntplm.delete_service_instance_state(service_id) 106 | 107 | 108 | @router.post( 109 | "/service/instance/redeploy/{service_id}", response_model=Response, status_code=201 110 | ) 111 | def redeploy_service_instance_state(service_id: str): 112 | res = ntplm.redeploy_service_instance_state(service_id) 113 | if res: 114 | return res 115 | else: 116 | raise HTTPException( 117 | status_code=404, 118 | detail=ResponseBasic( 119 | status="success", data={"task_result": f"{service_id} not found"} 120 | ).dict(), 121 | ) 122 | 123 | 124 | @router.post( 125 | "/service/instance/validate/{service_id}", response_model=Response, status_code=201 126 | ) 127 | def validate_service_instance_state(service_id: str): 128 | res = ntplm.validate_service_instance_state(service_id) 129 | if res: 130 | return res 131 | else: 132 | raise HTTPException( 133 | status_code=404, 134 | detail=ResponseBasic( 135 | status="success", data={"task_result": f"{service_id} not found"} 136 | ).dict(), 137 | ) 138 | 139 | 140 | @router.post( 141 | "/service/instance/healthcheck/{service_id}", 142 | response_model=Response, 143 | status_code=201, 144 | ) 145 | def health_check_service_instance_state(service_id: str): 146 | res = ntplm.health_check_service_instance_state(service_id) 147 | if res: 148 | return res 149 | else: 150 | raise HTTPException( 151 | status_code=404, 152 | detail=ResponseBasic( 153 | status="success", data={"task_result": f"{service_id} not found"} 154 | ).dict(), 155 | ) 156 | -------------------------------------------------------------------------------- /netpalm/routers/setconfig.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from fastapi import APIRouter 4 | 5 | # load models 6 | from netpalm.backend.core.models.models import SetConfig 7 | from netpalm.backend.core.models.napalm import NapalmSetConfig 8 | from netpalm.backend.core.models.ncclient import NcclientSetConfig 9 | from netpalm.backend.core.models.netmiko import NetmikoSetConfig 10 | from netpalm.backend.core.models.restconf import Restconf 11 | from netpalm.backend.core.models.task import Response 12 | 13 | from netpalm.backend.core.manager import ntplm 14 | 15 | from netpalm.routers.route_utils import HttpErrorHandler, poison_host_cache, whitelist 16 | 17 | log = logging.getLogger(__name__) 18 | router = APIRouter() 19 | 20 | 21 | # deploy a configuration 22 | @router.post("/setconfig", response_model=Response, status_code=201) 23 | @HttpErrorHandler() 24 | @poison_host_cache 25 | @whitelist 26 | def set_config(setcfg: SetConfig): 27 | return ntplm._set_config(setcfg) 28 | 29 | 30 | # dry run a configuration 31 | @router.post("/setconfig/dry-run", response_model=Response, status_code=201) 32 | @HttpErrorHandler() 33 | @whitelist 34 | def set_config_dry_run(setcfg: SetConfig): 35 | return ntplm.set_config_dry_run(setcfg) 36 | 37 | 38 | # deploy a configuration 39 | @router.post("/setconfig/netmiko", response_model=Response, status_code=201) 40 | @HttpErrorHandler() 41 | @poison_host_cache 42 | @whitelist 43 | def set_config_netmiko(setcfg: NetmikoSetConfig): 44 | return ntplm.set_config_netmiko(setcfg) 45 | 46 | 47 | # deploy a configuration 48 | @router.post("/setconfig/napalm", response_model=Response, status_code=201) 49 | @HttpErrorHandler() 50 | @poison_host_cache 51 | @whitelist 52 | def set_config_napalm(setcfg: NapalmSetConfig): 53 | return ntplm.set_config_napalm(setcfg) 54 | 55 | 56 | # deploy a configuration 57 | @router.post("/setconfig/ncclient", response_model=Response, status_code=201) 58 | @HttpErrorHandler() 59 | @poison_host_cache 60 | @whitelist 61 | def set_config_ncclient(setcfg: NcclientSetConfig): 62 | return ntplm.set_config_ncclient(setcfg) 63 | 64 | 65 | # deploy a configuration 66 | @router.post("/setconfig/restconf", response_model=Response, status_code=201) 67 | @HttpErrorHandler() 68 | @poison_host_cache 69 | @whitelist 70 | def set_config_restconf(setcfg: Restconf): 71 | return ntplm.set_config_restconf(setcfg) 72 | -------------------------------------------------------------------------------- /netpalm/routers/task.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from fastapi import APIRouter, HTTPException 4 | from fastapi.encoders import jsonable_encoder 5 | 6 | from netpalm.backend.core.models.task import Response, WorkerResponse 7 | from netpalm.backend.core.models.models import PinnedStore 8 | 9 | from netpalm.backend.core.manager import ntplm 10 | 11 | router = APIRouter() 12 | 13 | 14 | # get specific task 15 | @router.get("/task/{task_id}", response_model=Response) # this can *also* return ServiceResponse, but trying to typdef it doesn't seem to work 16 | def get_task(task_id: str): 17 | try: 18 | r = ntplm.fetchtask(task_id=task_id) 19 | resp = jsonable_encoder(r) 20 | if not resp: 21 | raise HTTPException(status_code=404) 22 | return resp 23 | except Exception as e: 24 | raise HTTPException(status_code=404) 25 | 26 | # get all tasks in queue 27 | @router.get("/taskqueue/") 28 | def get_task_list(): 29 | try: 30 | r = ntplm.getjoblist(q=False) 31 | resp = jsonable_encoder(r) 32 | return resp 33 | except Exception as e: 34 | raise HTTPException(status_code=500, detail=str(e).split('\n')) 35 | 36 | 37 | # task view route for specific host 38 | @router.get("/taskqueue/{host}") 39 | def get_host_task_list(host: str): 40 | try: 41 | r = ntplm.getjobliststatus(q=host) 42 | resp = jsonable_encoder(r) 43 | if not resp: 44 | raise HTTPException(status_code=404) 45 | return resp 46 | except Exception as e: 47 | raise HTTPException(status_code=500, detail=str(e).split('\n')) 48 | 49 | 50 | # get all running workers 51 | @router.get("/workers/", response_model=List[WorkerResponse]) 52 | def list_workers(): 53 | try: 54 | r = ntplm.get_workers() 55 | resp = jsonable_encoder(r) 56 | return resp 57 | except Exception as e: 58 | raise HTTPException(status_code=500, detail=str(e).split('\n')) 59 | 60 | 61 | # get all running workers 62 | @router.post("/workers/kill/{name}") 63 | def kill_worker(name: str): 64 | try: 65 | r = ntplm.kill_worker(worker_name=name) 66 | resp = jsonable_encoder(r) 67 | except Exception as e: 68 | raise HTTPException(status_code=500, detail=str(e).split('\n')) 69 | 70 | 71 | # get the container process totals 72 | @router.get("/containers/pinned/", response_model=List) 73 | def list_pinned_containers(): 74 | try: 75 | r = ntplm.fetch_pinned_store() 76 | resp = jsonable_encoder(r) 77 | return resp 78 | except Exception as e: 79 | raise HTTPException(status_code=500, detail=str(e).split('\n')) 80 | 81 | 82 | # # purge the container a container from the db 83 | # @router.delete("/containers/pinned/{hostname}") 84 | # def purge_pinned_containers_from_db(hostname: str): 85 | # try: 86 | # ntplm.purge_container_from_pinned_store(hostname) 87 | # except Exception as e: 88 | # raise HTTPException(status_code=500, detail=str(e).split('\n')) 89 | 90 | # # deregister worker 91 | # @router.post("/containers/deregister/{hostname}") 92 | # def deregister_workers_from_container(hostname: str): 93 | # try: 94 | # ntplm.deregister_worker(hostname) 95 | # except Exception as e: 96 | # raise HTTPException(status_code=500, detail=str(e).split('\n')) -------------------------------------------------------------------------------- /netpalm/routers/util.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | from typing import Optional 4 | 5 | import os, signal 6 | 7 | from fastapi import APIRouter, Query, Path 8 | from fastapi.encoders import jsonable_encoder 9 | from starlette.responses import RedirectResponse 10 | 11 | # load config 12 | from netpalm.backend.core.confload.confload import config 13 | 14 | from netpalm.backend.core.manager import ntplm 15 | 16 | from netpalm.backend.core.utilities.extensibles_reload import reload_extensibles_func 17 | 18 | from netpalm.routers.route_utils import HttpErrorHandler 19 | 20 | log = logging.getLogger(__name__) 21 | router = APIRouter() 22 | 23 | 24 | @router.get("/logout") 25 | async def route_logout_and_remove_cookie(): 26 | response = RedirectResponse(url="/") 27 | response.delete_cookie(config.api_key_name, domain=config.cookie_domain) 28 | response.delete_cookie("Authorization", domain=config.cookie_domain) 29 | return response 30 | 31 | 32 | # utility route - ping workers 33 | @router.get("/worker-ping") 34 | async def ping(): 35 | log.info(f"SENDING PING") 36 | worker_message = { 37 | "type": "ping", 38 | "kwargs": {} 39 | } 40 | rslt = ntplm.send_broadcast(json.dumps(worker_message)) 41 | # rslt = ntplm.send_broadcast("PING") # only way to see "response" is look at logs 42 | resp = jsonable_encoder(rslt) 43 | return resp 44 | 45 | 46 | # utility route - flush cache 47 | @router.delete("/cache") 48 | @HttpErrorHandler() 49 | def flush_cache(fail: Optional[bool] = Query(False, title="Fail", description="Fail on purpose")): 50 | if fail: 51 | raise RuntimeError(f"Failing on Purpose") 52 | log.info(f"Flushing Cache") 53 | rslt = { 54 | "cleared_records": int(ntplm.cache.clear()) 55 | } 56 | log.info(f"flush got this result: {rslt}") 57 | return rslt 58 | 59 | 60 | # utility route - flush cache for single device 61 | @router.delete("/cache/{cache_key}") 62 | @HttpErrorHandler() 63 | def flush_cache_device( 64 | cache_key: str = Path(..., 65 | title="The cache key to invalidate", 66 | description="must be of form host_or_ip:port:command_or_*") 67 | ): 68 | log.info(f"Flushing Cache for {cache_key}") 69 | rslt = { 70 | "cleared_records": int(ntplm.clear_cache_for_host(cache_key=cache_key)) 71 | } 72 | log.info(f"flush got this result: {rslt}") 73 | return rslt 74 | 75 | 76 | @router.get("/cache") 77 | @HttpErrorHandler() 78 | def list_cached_items(): 79 | log.info(f"Getting cache info") 80 | keys = ntplm.cache.keys() 81 | rslt = { 82 | "cache": keys, 83 | "size": len(keys) 84 | } 85 | return rslt 86 | 87 | 88 | @router.get("/cache/{cache_key}") 89 | @HttpErrorHandler() 90 | def get_cache_item( 91 | cache_key: str = Path(..., 92 | title="The cache key to retrieve", 93 | description="may include prefix, rest of the key must be complete") 94 | ): 95 | log.info(f"Getting cache info for {cache_key}") 96 | prefix = ntplm.cache.key_prefix 97 | cache_key = cache_key.replace(prefix, "") # no way to stop cache from adding this right now, so ensure no duplicate 98 | rslt = { 99 | cache_key: ntplm.cache.get(cache_key) 100 | } 101 | return rslt 102 | 103 | 104 | @router.put("/reload-extensibles") 105 | def reload_extensibles(): 106 | result = reload_extensibles_func() 107 | return result -------------------------------------------------------------------------------- /netpalm/static/images/netpalm_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/netpalm/static/images/netpalm_white.png -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | markers = 3 | getconfig: integration tests: gets config (deselect with '-m "not getconfig"') 4 | setconfig: integration tests: sets config (deselect with '-m "not setconfig"') 5 | script: integration tests: exec script (deselect with '-m "not script"') 6 | service: integration tests: exec service (deselect with '-m "not service"') 7 | fulllab: integration tests: tests that require tbotnz's full lab setup 8 | cisgo: integration tests: tests depending on the cisgo container 9 | cisgoalternate: tests which have a cisgo alternate, and might be deleted later 10 | misc_worker_router: tests for worker routes 11 | test_worker: not sure 12 | test_kill_worker: tests for worker kill 13 | test_worker_route: tests for worker route 14 | test_pinned_container: tests for pinned container 15 | whitelist: tests for device whitelist 16 | 17 | 18 | filterwarnings = 19 | ignore::DeprecationWarning 20 | 21 | norecursedirs = 22 | .git 23 | static 24 | 25 | testpaths = 26 | tests -------------------------------------------------------------------------------- /redis_gen_new_certs.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | #!/bin/bash 4 | mkdir -p netpalm/backend/core/security/cert/tls 5 | openssl genrsa -out netpalm/backend/core/security/cert/tls/ca.key 4096 6 | openssl req \ 7 | -x509 -new -nodes -sha256 \ 8 | -key netpalm/backend/core/security/cert/tls/ca.key \ 9 | -days 3650 \ 10 | -subj '/O=Redis Test/CN=Certificate Authority' \ 11 | -out netpalm/backend/core/security/cert/tls/ca.crt 12 | openssl genrsa -out netpalm/backend/core/security/cert/tls/redis.key 2048 13 | openssl req \ 14 | -new -sha256 \ 15 | -key netpalm/backend/core/security/cert/tls/redis.key \ 16 | -subj '/O=Redis Test/CN=Server' | \ 17 | openssl x509 \ 18 | -req -sha256 \ 19 | -CA netpalm/backend/core/security/cert/tls/ca.crt \ 20 | -CAkey netpalm/backend/core/security/cert/tls/ca.key \ 21 | -CAserial netpalm/backend/core/security/cert/tls/ca.txt \ 22 | -CAcreateserial \ 23 | -days 365 \ 24 | -out netpalm/backend/core/security/cert/tls/redis.crt 25 | openssl dhparam -out netpalm/backend/core/security/cert/tls/redis.dh 2048 -------------------------------------------------------------------------------- /static/images/arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/arch.png -------------------------------------------------------------------------------- /static/images/netpalm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/netpalm.png -------------------------------------------------------------------------------- /static/images/netpalm_concept.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/netpalm_concept.png -------------------------------------------------------------------------------- /static/images/netpalm_eg_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/netpalm_eg_1.png -------------------------------------------------------------------------------- /static/images/netpalm_eg_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/netpalm_eg_2.png -------------------------------------------------------------------------------- /static/images/netpalm_eg_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/netpalm_eg_3.png -------------------------------------------------------------------------------- /static/images/netpalm_eg_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/netpalm_eg_4.png -------------------------------------------------------------------------------- /static/images/netpalm_ingest.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/netpalm_ingest.gif -------------------------------------------------------------------------------- /static/images/netpalm_postman.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/netpalm_postman.png -------------------------------------------------------------------------------- /static/images/netpalm_service_closed_loop_demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/netpalm_service_closed_loop_demo.gif -------------------------------------------------------------------------------- /static/images/netpalm_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/netpalm_white.png -------------------------------------------------------------------------------- /static/images/np-basic-new1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/np-basic-new1.png -------------------------------------------------------------------------------- /static/images/np-basic-new2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/np-basic-new2.png -------------------------------------------------------------------------------- /static/images/np-basic-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/np-basic-overview.png -------------------------------------------------------------------------------- /static/images/np-basic-q.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/np-basic-q.png -------------------------------------------------------------------------------- /static/images/np-component.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/np-component.png -------------------------------------------------------------------------------- /static/images/np-scale-out.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/np-scale-out.png -------------------------------------------------------------------------------- /static/images/np_new.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/np_new.png -------------------------------------------------------------------------------- /static/images/np_service.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/np_service.gif -------------------------------------------------------------------------------- /static/images/oapi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/static/images/oapi.png -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/tests/__init__.py -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/tests/integration/__init__.py -------------------------------------------------------------------------------- /tests/integration/helper.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import time 4 | from json import JSONDecodeError 5 | 6 | import requests 7 | from typing import Dict, Tuple, List 8 | 9 | log = logging.getLogger(__name__) 10 | CONFIG_FILENAME = "config/config.json" 11 | DEFAULTS_FILENAME = "config/defaults.json" 12 | 13 | 14 | def load_config_files(defaults_filename: str = DEFAULTS_FILENAME, config_filename: str = CONFIG_FILENAME) -> dict: 15 | data = {} 16 | 17 | for fname in (defaults_filename, config_filename): 18 | try: 19 | with open(fname) as infil: 20 | data.update(json.load(infil)) 21 | except FileNotFoundError: 22 | log.warning(f"Couldn't find {fname}") 23 | 24 | if not data: 25 | raise RuntimeError(f"Could not find either {defaults_filename} or {config_filename}") 26 | 27 | return data 28 | 29 | 30 | class NetpalmTestHelper: 31 | 32 | def __init__(self): 33 | data = load_config_files() 34 | self.apikey = data["api_key"] 35 | self.ip = '127.0.0.1' 36 | self.port = data["listen_port"] 37 | self.base_url = f"http://{self.ip}:{self.port}" 38 | self.headers = {'Content-type': 'application/json', 'Accept': 'text/plain', 'x-api-key': self.apikey} 39 | # test devices go here 40 | self.test_device_ios_cli = "10.0.2.33" 41 | self.test_device_netconf = "10.0.2.39" 42 | self.test_device_restconf = "ios-xe-mgmt-latest.cisco.com" 43 | self.test_device_cisgo = "cisgo" 44 | self.http_timeout = 5 45 | self.task_timeout = 15 46 | self.task_poll_interval = 0.2 47 | 48 | def get(self, endpoint: str): 49 | try: 50 | r = requests.get(f"http://{self.ip}:{self.port}/{endpoint}", 51 | headers=self.headers, timeout=self.http_timeout) 52 | return r.json() 53 | except Exception as e: 54 | log.exception(f"error while getting {endpoint}") 55 | raise 56 | 57 | def post(self, endpoint: str, data): 58 | try: 59 | r = requests.post(f"http://{self.ip}:{self.port}/{endpoint}", 60 | headers=self.headers, json=data, timeout=self.http_timeout) 61 | return r.json() 62 | except Exception as e: 63 | log.exception(f"error while posting to {endpoint}") 64 | raise 65 | 66 | def check_task(self, taskid): 67 | return self.get(f"task/{taskid}") 68 | 69 | def poll_task(self, taskid, timeout=None) -> Tuple[Dict, List]: 70 | if timeout is None: 71 | timeout = self.task_timeout 72 | 73 | start_time = time.time() 74 | while True: 75 | task_res = self.check_task(taskid) 76 | if task_res["data"]["task_status"] == "finished" or task_res["data"]["task_status"] == "failed": 77 | result, errors = task_res["data"]["task_result"], task_res["data"]["task_errors"] 78 | break 79 | 80 | if time.time() + self.task_poll_interval > start_time + timeout: 81 | raise TimeoutError("Netmiko task timed out") 82 | 83 | time.sleep(self.task_poll_interval) 84 | 85 | log.error(f'got {task_res}') 86 | return result, errors 87 | 88 | def poll_task_errors(self, taskid, timeout=None) -> List: 89 | result, errors = self.poll_task(taskid, timeout) 90 | return errors 91 | 92 | def post_and_check(self, endpoint, payload) -> Dict: 93 | url = f"{self.base_url}{endpoint}" 94 | r = requests.post(url, json=payload, headers=self.headers, timeout=self.http_timeout) 95 | r.raise_for_status() 96 | try: 97 | task_id = r.json()["data"]["task_id"] 98 | 99 | except JSONDecodeError: 100 | log.error(f"Can't JSON decode response:\n{r.text}") 101 | raise 102 | 103 | result, errors = self.poll_task(task_id) 104 | return result 105 | 106 | def post_and_check_errors(self, endpoint, payload) -> List: 107 | url = f"{self.base_url}{endpoint}" 108 | r = requests.post(url, json=payload, headers=self.headers, timeout=self.http_timeout) 109 | task_id = r.json()["data"]["task_id"] 110 | errors = self.poll_task_errors(task_id) 111 | return errors 112 | 113 | def check_many(self, payload) -> List[Dict]: 114 | results = [] 115 | for task in payload: 116 | res, err = self.poll_task(task["data"]["data"]["task_id"]) 117 | results.append(res) 118 | return results 119 | -------------------------------------------------------------------------------- /tests/integration/test_misc.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import requests 3 | import json 4 | from tests.integration.helper import NetpalmTestHelper 5 | 6 | helper = NetpalmTestHelper() 7 | 8 | @pytest.mark.misc_worker_router 9 | def test_worker_route(): 10 | url = f"{helper.base_url}/workers/" 11 | r = requests.get(url, json={}, headers=helper.headers, timeout=helper.http_timeout) 12 | res = r.json() 13 | assert len(res) >= 2 14 | 15 | -------------------------------------------------------------------------------- /tests/integration/test_script.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import requests 3 | import random 4 | from tests.integration.helper import NetpalmTestHelper 5 | 6 | helper = NetpalmTestHelper() 7 | 8 | 9 | @pytest.mark.script 10 | def test_exec_script(): 11 | pl = { 12 | "script":"hello_world", 13 | "args":{ 14 | "hello":"world" 15 | } 16 | } 17 | res = helper.post_and_check("/script", pl) 18 | assert res == "world" 19 | 20 | 21 | @pytest.mark.script 22 | def test_exec_script_failure(): 23 | pl = { 24 | "script":"hello_world", 25 | "args":{ 26 | "bad":"args" 27 | } 28 | } 29 | res = helper.post_and_check("/script", pl) 30 | res2 = helper.post_and_check_errors("/script", pl) 31 | assert res is None 32 | assert res2 == [ 33 | # "Required args: 'hello'" 34 | 35 | { 36 | "exception_args": ["hello"], 37 | "exception_class": "KeyError" 38 | }, 39 | { 40 | "exception_args": ["Required args: 'hello'"], 41 | "exception_class": "Exception" 42 | } 43 | ] 44 | -------------------------------------------------------------------------------- /tests/integration/test_service.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import requests 3 | import random 4 | import logging 5 | 6 | from tests.integration.helper import NetpalmTestHelper 7 | 8 | log = logging.getLogger(__name__) 9 | 10 | helper = NetpalmTestHelper() 11 | 12 | @pytest.mark.service 13 | def test_prepare_vlan_service_environment(): 14 | pass 15 | # pl = { 16 | # "operation": "create", 17 | # "args": { 18 | # "hosts": ["10.0.2.25", "10.0.2.23"], 19 | # "username": "admin", 20 | # "password": "admin" 21 | # }, 22 | # "queue_strategy": "fifo" 23 | # } 24 | # reslist = helper.post_and_check('/service/vlan_service',pl) 25 | # res = helper.check_many(reslist) 26 | # if res: 27 | # assert True 28 | 29 | # @pytest.mark.service 30 | # def test_create_vlan_service_instance(): 31 | # pl = { 32 | # "operation": "create", 33 | # "args": { 34 | # "hosts": ["10.0.2.25", "10.0.2.23"], 35 | # "username": "admin", 36 | # "password": "admin" 37 | # }, 38 | # "queue_strategy": "fifo" 39 | # } 40 | # reslist = helper.post_and_check('/service/vlan_service',pl) 41 | # res = helper.check_many(reslist) 42 | # assert "(config)#int vlan 99" in res[0]["changes"][2] 43 | # assert "(config)#int vlan 99" in res[1]["changes"][2] 44 | 45 | 46 | # @pytest.mark.fulllab 47 | # @pytest.mark.service 48 | # def test_retrieve_vlan_service(): 49 | # pl = { 50 | # "operation": "retrieve", 51 | # "args": { 52 | # "hosts": ["10.0.2.25","10.0.2.23"], 53 | # "username": "admin", 54 | # "password": "admin" 55 | # }, 56 | # "queue_strategy": "fifo" 57 | # } 58 | # reslist = helper.post_and_check('/service/vlan_service',pl) 59 | # res = helper.check_many(reslist) 60 | # assert res[0]["show int vlan 99"][0]["interface"] == "Vlan99" 61 | # assert res[1]["show int vlan 99"][0]["interface"] == "Vlan99" 62 | 63 | 64 | # @pytest.mark.fulllab 65 | # @pytest.mark.service 66 | # def test_delete_vlan_service_legacy(): 67 | # pl = { 68 | # "operation": "delete", 69 | # "args": { 70 | # "hosts": ["10.0.2.25","10.0.2.23"], 71 | # "username": "admin", 72 | # "password": "admin" 73 | # }, 74 | # "queue_strategy": "fifo" 75 | # } 76 | # reslist = helper.post_and_check('/service/vlan_service',pl) 77 | # res = helper.check_many(reslist) 78 | # assert "(config)#no int vlan 99" in res[0]["changes"][2] 79 | # assert "(config)#no int vlan 99" in res[1]["changes"][2] 80 | 81 | 82 | # @pytest.mark.service 83 | # def test_create_vlan_service_instance(): 84 | # pl = { 85 | # "operation": "create", 86 | # "args": { 87 | # "hosts": ["10.0.2.25", "10.0.2.23"], 88 | # "username": "admin", 89 | # "password": "admin" 90 | # }, 91 | # "queue_strategy": "fifo" 92 | # } 93 | # reslist = helper.post('service/vlan_service', pl) 94 | # assert reslist["data"]["service_id"] 95 | 96 | # @pytest.mark.service 97 | # def test_retrieve_vlan_service_instance(): 98 | # pl = { 99 | # "operation": "create", 100 | # "args": { 101 | # "hosts": ["10.0.2.25", "10.0.2.23"], 102 | # "username": "admin", 103 | # "password": "admin" 104 | # }, 105 | # "queue_strategy": "fifo" 106 | # } 107 | # q = helper.post('service/vlan_service', pl) 108 | # # finish off at some point 109 | 110 | # assert True 111 | 112 | -------------------------------------------------------------------------------- /tests/integration/test_setconfig_cisgo.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import random 3 | from typing import List, Union 4 | 5 | import pytest 6 | 7 | from .helper import NetpalmTestHelper 8 | from .test_getconfig_cisgo import CisgoHelper 9 | 10 | log = logging.getLogger(__name__) 11 | helper = NetpalmTestHelper() 12 | 13 | CISGO_DEFAULT_HOSTNAME = "cisshgo1000v" 14 | CISGO_NEW_HOSTNAME = CISGO_DEFAULT_HOSTNAME.upper() + str( 15 | random.randint(100, 900)) 16 | 17 | 18 | @pytest.fixture(scope="function") 19 | def cisgo_helper(): 20 | return CisgoHelper() 21 | 22 | 23 | def hostname_from_config(config_lines: Union[List[str], str]) -> str: 24 | if isinstance(config_lines, str): 25 | config_lines = config_lines.splitlines() 26 | 27 | for line in config_lines: 28 | if not line: 29 | continue 30 | command, *args = line.split() 31 | if command == "hostname": 32 | hostname = ' '.join( 33 | args 34 | ) # this will false-match if there's weird whitespace in hostname like \t, etc 35 | break 36 | 37 | else: 38 | raise ValueError("No hostname found!") 39 | 40 | return hostname 41 | 42 | 43 | def get_hostname(connection_args): 44 | pl = { 45 | "library": "netmiko", 46 | "connection_args": connection_args, 47 | "command": "show running-config" 48 | } 49 | res = helper.post_and_check('/getconfig', pl) 50 | return hostname_from_config(res["show running-config"]) 51 | 52 | 53 | @pytest.mark.setconfig 54 | @pytest.mark.cisgo 55 | def test_setconfig_netmiko(cisgo_helper: CisgoHelper): 56 | pl = { 57 | "library": "netmiko", 58 | "connection_args": cisgo_helper.netmiko_connection_args, 59 | "config": ["hostname " + CISGO_NEW_HOSTNAME], 60 | "enable_mode": True 61 | } 62 | res = helper.post_and_check('/setconfig', pl) 63 | matchstr = CISGO_NEW_HOSTNAME + "#" 64 | assert matchstr in res["changes"] 65 | 66 | 67 | @pytest.mark.setconfig 68 | @pytest.mark.cisgo 69 | def test_setconfig_netmiko_multiple(cisgo_helper: CisgoHelper): 70 | pl = { 71 | "library": "netmiko", 72 | "connection_args": cisgo_helper.netmiko_connection_args, 73 | "config": ["hostname yeti", "hostname bufoon"], 74 | "enable_mode": True 75 | } 76 | res = helper.post_and_check('/setconfig', pl) 77 | assert len(res["changes"]) > 4 78 | 79 | 80 | @pytest.mark.setconfig 81 | @pytest.mark.cisgo 82 | def test_setconfig_netmiko_j2(cisgo_helper): 83 | pl = { 84 | "library": "netmiko", 85 | "connection_args": cisgo_helper.netmiko_connection_args, 86 | "enable_mode": True, 87 | "j2config": { 88 | "template": "test", 89 | "args": { 90 | "vlans": ["1", "2", "3"] 91 | } 92 | } 93 | } 94 | res = helper.post_and_check('/setconfig', pl) 95 | assert len(res["changes"]) > 6 96 | -------------------------------------------------------------------------------- /tests/integration/test_worker.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from tests.integration.helper import NetpalmTestHelper 4 | 5 | helper = NetpalmTestHelper() 6 | 7 | 8 | @pytest.mark.test_worker_route 9 | def test_worker(): 10 | res = helper.get("workers/") 11 | assert len(res) > 0 12 | 13 | 14 | @pytest.mark.test_kill_worker 15 | def test_kill_worker(): 16 | resz = helper.get("workers/") 17 | rt = "workers/kill/" + resz[0]["name"] 18 | rest = helper.post(rt, data={}) 19 | assert rest is None 20 | 21 | 22 | @pytest.mark.test_pinned_container 23 | def test_worker_pinned_container(): 24 | res = helper.get("containers/pinned/") 25 | assert len(res) > 0 26 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbotnz/netpalm/e1b47ad72a2a10e616646da30dc942d80894c86d/tests/unit/__init__.py -------------------------------------------------------------------------------- /tests/unit/test_confload.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | import pytest 4 | 5 | CONFIG_FILENAME = "config/config.json" 6 | ACTUAL_CONFIG_PATH = Path(CONFIG_FILENAME).absolute() 7 | 8 | if not ACTUAL_CONFIG_PATH.exists(): 9 | ACTUAL_CONFIG_PATH = ACTUAL_CONFIG_PATH.parent.parent / CONFIG_FILENAME # try ../config.json 10 | if not ACTUAL_CONFIG_PATH.exists(): 11 | raise FileNotFoundError(f'Can\'t run confload tests without finding config.json, ' 12 | f'tried looking in {ACTUAL_CONFIG_PATH}') 13 | 14 | 15 | os.environ["NETPALM_CONFIG"] = str(ACTUAL_CONFIG_PATH) 16 | from netpalm.backend.core.confload import confload 17 | 18 | 19 | def test_netpalm_config_honors_envvar(): 20 | with pytest.raises(KeyError): 21 | config = confload.Config("DOES NOT EXIST.json") 22 | _ = config.data["__comment__"] 23 | 24 | with pytest.raises(KeyError): 25 | os.environ["NETPALM_CONFIG"] = "DOES NOT EXIST.JSON" 26 | config = confload.initialize_config() 27 | _ = config.data["__comment__"] 28 | 29 | # with pytest.raises(FileNotFoundError): # this depends on the fact that you're running pytest from the tests directory 30 | # del os.environ["NETPALM_CONFIG"] # but we're not doing that anymore and it's okay really 31 | # config = confload.initialize_config() 32 | 33 | config = confload.Config(ACTUAL_CONFIG_PATH) 34 | _ = config.data["__comment__"] 35 | os.environ["NETPALM_CONFIG"] = str(ACTUAL_CONFIG_PATH) 36 | config = confload.initialize_config() 37 | _ = config.data["__comment__"] 38 | 39 | 40 | def test_netpalm_config_value_precedence(monkeypatch): 41 | file_config = confload.Config(ACTUAL_CONFIG_PATH) 42 | monkeypatch.setenv("NETPALM_REDIS_SERVER", "123.COM") 43 | envvar_config = confload.Config(ACTUAL_CONFIG_PATH) 44 | assert file_config.redis_key == envvar_config.redis_key 45 | assert envvar_config.redis_server == '123.COM' 46 | 47 | 48 | def test_tfsm_search(monkeypatch): 49 | monkeypatch.setenv("NETPALM_TXTFSM_INDEX_FILE", "backend/plugins/extensibles/DOESNOTEXIT/index") 50 | config = confload.initialize_config(search_tfsm=False) 51 | config.setup_logging(max_debug=True) 52 | index_file_path = Path(config.txtfsm_index_file).absolute() 53 | assert not index_file_path.exists() 54 | 55 | config = confload.initialize_config() # search_tfsm must default to True 56 | index_file_path = Path(config.txtfsm_index_file).absolute() 57 | assert index_file_path.exists() 58 | -------------------------------------------------------------------------------- /tests/unit/test_device_whitelist.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import List, Tuple 3 | 4 | import pytest 5 | 6 | from netpalm.backend.core.security.whitelist import DeviceWhitelist, WhiteListRule 7 | 8 | pytestmark = pytest.mark.whitelist 9 | 10 | log = logging.getLogger(__name__) 11 | 12 | 13 | @pytest.mark.parametrize( 14 | ("rule_definition", "hostname", "expected"), 15 | [ 16 | ("*.com", "foo.com", True), 17 | ("*.com", "bar", False), 18 | ("foo.com", "b.foo.com", False), 19 | ("*.com", "10.0.0.1", False), 20 | ("10.0.0.*", "10.0.0.1", True), 21 | ("10.0.0.0/8", "10.0.0.1", True), 22 | ("10.0.0.0", "10.0.0.1", False), 23 | ("10.0.0.1", "10.0.0.1", True), 24 | ("2600::1", "2600:0:0:0:0::1", True) 25 | ] 26 | ) 27 | def test_whitelist_rule(rule_definition: str, hostname: str, expected: bool): 28 | rule = WhiteListRule(rule_definition) 29 | assert rule.match(hostname) == expected 30 | 31 | 32 | @pytest.mark.parametrize(("whitelist_definition", "tests"), [ 33 | ([], [ 34 | ("foo.com", True), 35 | ("bar", True), 36 | ("10.0.0.1", True), 37 | ("172.24.1.1", True) 38 | ]), 39 | (None, [ 40 | ("foo.com", True), 41 | ("bar", True), 42 | ("10.0.0.1", True), 43 | ("172.24.1.1", True) 44 | ]), 45 | ([ 46 | "*.com" 47 | ], [ 48 | ("foo.com", True), 49 | ("a.foo.com", True), 50 | ("bar", False), 51 | ("10.0.0.1", False), 52 | ("172.24.1.1", False) 53 | ]), 54 | ([ 55 | "*.foo.com", 56 | "bar" 57 | ], [ 58 | ("foo.com", False), 59 | ("a.foo.com", True), 60 | ("bar", True), 61 | ("10.0.0.1", False), 62 | ("172.24.1.1", False) 63 | ]), 64 | ([ 65 | "10.0.0.0/24" 66 | ], [ 67 | ("foo.com", False), 68 | ("a.foo.com", False), 69 | ("bar", False), 70 | ("10.0.0.1", True), 71 | ("172.24.1.1", False) 72 | ]), 73 | ]) 74 | def test_device_whitelist(whitelist_definition: List[str], tests: List[Tuple[str, bool]]): 75 | dwl = DeviceWhitelist(whitelist_definition) 76 | assert dwl.definition == whitelist_definition 77 | for hostname, expected in tests: 78 | print(f"testing {hostname} expecting {expected}") 79 | assert dwl.match(hostname) == expected 80 | -------------------------------------------------------------------------------- /tests/unit/test_tfsm_templates.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | import pytest 4 | 5 | from netpalm.backend.core.confload import confload 6 | from netpalm.backend.core.utilities.textfsm.template import FSMTemplate 7 | 8 | 9 | def test_template_object(): 10 | config = confload.initialize_config() 11 | template_obj = FSMTemplate() 12 | result = template_obj.get_template_list() 13 | assert "Errno" not in result.get("data", "") 14 | assert result["status"] != "error" 15 | 16 | 17 | # pull mapping of drivers to list of template mappings 18 | def test_get_template_list(): 19 | template_obj = FSMTemplate() 20 | result = template_obj.get_template_list() 21 | assert "task_result" in result["data"] 22 | tr = result["data"]["task_result"] 23 | assert len(tr) > 0 24 | for driver, template_mappings in tr.items(): 25 | assert isinstance(driver, str) 26 | assert isinstance(template_mappings, list) 27 | for template_mapping in template_mappings: 28 | assert "command" in template_mapping 29 | assert "template" in template_mapping 30 | 31 | 32 | def get_driver_template_list( 33 | driver: str, template_obj: FSMTemplate 34 | ) -> typing.List[typing.Dict]: 35 | result = template_obj.get_template_list() 36 | template_driver_mapping = result["data"]["task_result"] 37 | return template_driver_mapping.get(driver, []) 38 | 39 | 40 | def get_matching_templates(target_template: typing.Dict, template_obj: FSMTemplate): 41 | command = target_template["command"] 42 | template_name = target_template["template_name"] 43 | driver = target_template["driver"] 44 | template_list = get_driver_template_list(driver, template_obj) 45 | templates = [] 46 | for ( 47 | template 48 | ) in ( 49 | template_list 50 | ): # was originally a list comprehension, expanded for easier debugging. 51 | command_matches = template["command"].strip() == command 52 | template_matches = template["template"].strip() == template_name 53 | if command_matches and template_matches: 54 | templates.append(template) 55 | return templates 56 | 57 | 58 | def test_add_template(): 59 | test_template = { 60 | "key": "573300637760474_59123133312286777", 61 | "driver": "dell_force10", 62 | "command": "show mac-address-table", 63 | "template_name": "dell_force10_show_mac-address-table.template", 64 | } 65 | driver = test_template["driver"] 66 | 67 | template_obj = FSMTemplate(**test_template) 68 | 69 | matching_templates = get_matching_templates(test_template, template_obj) 70 | base = len(matching_templates) 71 | 72 | template_obj.add_template(strict=False) 73 | 74 | matching_templates = get_matching_templates(test_template, template_obj) 75 | assert len(matching_templates) == base + 1 76 | 77 | 78 | def test_invalid_template_raises_error(): 79 | test_template = { 80 | "key": "573DOES NOT EXIST60474_59123133312286777", 81 | "driver": "dell_force10", 82 | "command": "show mac-address-table", 83 | "template_name": "dell_force10_show_mac-address-table.template", 84 | } 85 | template_obj = FSMTemplate(**test_template) 86 | from requests.exceptions import HTTPError 87 | 88 | with pytest.raises(HTTPError): 89 | template_obj.add_template() 90 | 91 | 92 | # commenting out due to storage issues on host 93 | 94 | # def test_add_template_new_section(): 95 | # from random import randint 96 | # rint = randint(0, 1000) 97 | # test_template = { 98 | # "key": "573300637760474_59123133312286777", 99 | # "driver": f"cisgo_ios{rint}", 100 | # "command": "show mac-address-table", 101 | # "template_name": f"cisgo_ios{rint}_show_mac-address-table.template" 102 | # } 103 | 104 | # driver = test_template["driver"] 105 | 106 | # template_obj = FSMTemplate(**test_template) 107 | # existing_driver_templates = get_driver_template_list(driver, template_obj) 108 | # assert len(existing_driver_templates) == 0 109 | 110 | # template_obj.add_template() 111 | # new_driver_templates = get_driver_template_list(driver, template_obj) 112 | # assert len(new_driver_templates) == 1 113 | 114 | 115 | def test_del_template(): 116 | test_template = { 117 | "driver": "dell_force10", 118 | "command": "show mac-address-table", 119 | "template_name": "dell_force10_show_mac-address-table.template", 120 | } 121 | test_template["template"] = test_template["template_name"] 122 | template_obj = FSMTemplate(**test_template) 123 | 124 | matching_templates = get_matching_templates(test_template, template_obj) 125 | 126 | base = len(matching_templates) 127 | assert base > 0 128 | 129 | template_obj.remove_template() 130 | 131 | matching_templates = get_matching_templates(test_template, template_obj) 132 | assert len(matching_templates) == 0 133 | -------------------------------------------------------------------------------- /tests/unit/test_update_log.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | import pytest 4 | import redis_lock 5 | 6 | from netpalm.backend.core.confload.confload import config 7 | 8 | from netpalm.backend.core.confload import confload 9 | from netpalm.backend.core.manager import ntplm 10 | from netpalm.backend.core.redis.rediz import ExtnUpdateLog, TransactionLogEntryType, TransactionLogEntryModel 11 | 12 | 13 | @pytest.fixture(scope="function") 14 | def clean_log(): 15 | config = confload.initialize_config() 16 | extn_log = ExtnUpdateLog(ntplm.base_connection, config.redis_update_log, create=False) 17 | extn_log.clear() 18 | 19 | 20 | def test_extensible_update_lock_behavior(): 21 | lock = ntplm.extn_update_log.lock 22 | assert not lock.locked() 23 | 24 | with lock: # should work 25 | assert lock.locked() 26 | 27 | with pytest.raises(redis_lock.AlreadyAcquired): 28 | with lock: 29 | pass 30 | 31 | with pytest.raises(redis_lock.AlreadyAcquired): 32 | lock.acquire() 33 | 34 | new_lock = redis_lock.Lock(ntplm.base_connection, config.redis_update_log) 35 | assert not new_lock.acquire(blocking=False) # proving 'acquire' fails with a new instance 36 | 37 | 38 | def test_extensible_update_log_creation(clean_log): 39 | extn_update_log = ntplm.extn_update_log 40 | assert not extn_update_log.exists 41 | extn_update_log.create() 42 | assert extn_update_log.exists 43 | 44 | new_log_obj = ExtnUpdateLog(ntplm.base_connection, ntplm.extn_update_log.log_name) 45 | assert new_log_obj.exists 46 | 47 | assert new_log_obj.get(-1).type is TransactionLogEntryType.init 48 | 49 | 50 | def test_extensible_update_log_add_fetch(clean_log): 51 | extn_update_log = ntplm.extn_update_log 52 | ntplm.extn_update_log.create(strict=True) 53 | 54 | item_1_dict = { 55 | "type": TransactionLogEntryType.tfsm_pull, 56 | "data": { 57 | "key": "123_432", 58 | "driver": "dell_force10", 59 | "command": "show version" 60 | } 61 | } 62 | item_2_dict = { 63 | "type": TransactionLogEntryType.tfsm_pull, 64 | "data": { 65 | "key": "999_876", 66 | "driver": "cisco_ios", 67 | "command": "show version" 68 | } 69 | } 70 | init_dict = { 71 | "type": TransactionLogEntryType.init, 72 | "data": { 73 | "init": True 74 | } 75 | } 76 | item_dicts = [item_1_dict, item_2_dict] 77 | items = [TransactionLogEntryModel(seq=index, **item_dict) 78 | for index, item_dict in enumerate(item_dicts, start=1)] 79 | 80 | pprint(items) 81 | 82 | with pytest.raises(ValueError): # init records are only valid at very start 83 | extn_update_log.add(init_dict) 84 | 85 | assert (start_len := len(extn_update_log)) == 1 # should only have the init record now 86 | 87 | for item in items: 88 | extn_update_log.add(item) 89 | 90 | new_len = start_len + len(items) 91 | 92 | assert len(extn_update_log) == new_len 93 | 94 | with pytest.raises(IndexError): 95 | extn_update_log.get(new_len + 10) 96 | 97 | with pytest.raises(IndexError): 98 | _ = extn_update_log[new_len + 10] 99 | 100 | log_items = extn_update_log[1:] 101 | assert all(item == log_item for item, log_item in zip(items, log_items)) 102 | 103 | for item in extn_update_log: 104 | print(item) # proves the we can iterate over the log like a list 105 | 106 | 107 | def test_update_log_processor(clean_log): 108 | from netpalm.netpalm_worker_common import UpdateLogProcessor, update_log_processor 109 | up = update_log_processor 110 | additional_up = UpdateLogProcessor(ntplm) 111 | 112 | echo_dict = { 113 | "type": TransactionLogEntryType.echo, 114 | "data": { 115 | "msg": "echo? ? ECHO!!" 116 | } 117 | } 118 | assert up._get_lock() 119 | assert not additional_up._get_lock() 120 | up._release_lock() 121 | 122 | extn_update_log = ntplm.extn_update_log 123 | extn_update_log.create(strict=True) 124 | 125 | assert up.last_seq_number is -1 126 | 127 | assert up.process_log() == 1 128 | 129 | assert up.last_seq_number == 0 130 | 131 | for item in [echo_dict] * 3: 132 | extn_update_log.add(item) 133 | 134 | assert len(extn_update_log) == 4 135 | 136 | assert up.process_log() == 3 137 | assert up.last_seq_number == 3 138 | -------------------------------------------------------------------------------- /worker.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from netpalm import netpalm_fifo_worker, netpalm_pinned_worker 4 | 5 | 6 | def main(args): 7 | worker_type = args[-1] 8 | if len(args) != 2 or worker_type not in ("pinned", "fifo"): 9 | print(f"Worker must be specified as either 'pinned' or 'fifo'. e.g. `python3 worker.py pinned`") 10 | sys.exit(1) 11 | 12 | if worker_type == "pinned": 13 | netpalm_pinned_worker.start_processworkerprocess() 14 | 15 | else: 16 | netpalm_fifo_worker.start_worker() 17 | 18 | 19 | if __name__ == "__main__": 20 | main(sys.argv) 21 | --------------------------------------------------------------------------------