├── .gitattributes ├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── README.md ├── bin └── xym ├── requirements.txt ├── setup.cfg ├── setup.py ├── test ├── resources │ ├── test-file-no-file-after-code-begins │ ├── test-file-with-symbol │ └── test-file.txt ├── test-file.txt └── test.py ├── versioneer.py └── xym ├── __init__.py ├── _version.py ├── xym.py └── yangParser.py /.gitattributes: -------------------------------------------------------------------------------- 1 | xym/_version.py export-subst 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *~ 3 | *.pyc 4 | *.egg-info 5 | build/ 6 | dist/ 7 | v/ 8 | .idea/ 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.5" 4 | - "3.6" 5 | - "3.7" 6 | - "3.8" 7 | - "3.9" 8 | script: 9 | - pip install -r requirements.txt 10 | - python setup.py install 11 | - cd test && python -m unittest test 12 | 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Cisco Systems, Inc 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of xym nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include test/test*.txt 3 | include requirements.txt 4 | include versioneer.py 5 | include xym/_version.py 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/xym-tool/xym.svg)](https://travis-ci.org/xym-tool/xym) 2 | 3 | # xym.py 4 | 5 | xym is a simple utility for extracting [YANG](https://tools.ietf.org/rfc/rfc6020.txt) modules from files. 6 | 7 | xym may be installed via PyPi, or the latest version may be picked up from here and manually installed (along with its dependencies). It can often be sensible to install tools into a virtualenv, which is recommended. For example: 8 | 9 | ``` 10 | $ git clone https://github.com/xym-tool/xym.git 11 | Cloning into 'xym'... 12 | remote: Counting objects: 32, done. 13 | remote: Compressing objects: 100% (20/20), done. 14 | remote: Total 32 (delta 8), reused 29 (delta 5), pack-reused 0 15 | Unpacking objects: 100% (32/32), done. 16 | Checking connectivity... done. 17 | $ cd xym 18 | $ virtualenv v 19 | New python executable in v/bin/python2.7 20 | Not overwriting existing python script v/bin/python (you must use v/bin/python2.7) 21 | Installing setuptools, pip, wheel...done. 22 | $ . v/bin/activate 23 | $ python setup.py install 24 | running install 25 | ... 26 | ... 27 | Finished processing dependencies for xym==0.2 28 | $ 29 | ``` 30 | 31 | Help with it's options may be displayed thus: 32 | 33 | ``` 34 | $ xym --help 35 | usage: xym [-h] [--srcdir SRCDIR] [--dstdir DSTDIR] [--strict STRICT] 36 | [--strict-examples] [--write-dict] [--debug DEBUG] 37 | [--force-revision FORCE_REVISION] [--version] 38 | source 39 | 40 | Extracts one or more yang models from an IETF RFC/draft text file 41 | 42 | positional arguments: 43 | source The URL or file name of the RFC/draft text from which to 44 | get the model 45 | 46 | optional arguments: 47 | -h, --help show this help message and exit 48 | --srcdir SRCDIR Optional: directory where to find the source text; 49 | default is './' 50 | --dstdir DSTDIR Optional: directory where to put the extracted yang 51 | module(s); default is './' 52 | --strict Optional flag that determines syntax enforcement; If set 53 | to 'True', the / tags are 54 | required; default is 'False' 55 | --strict-examples Only output valid examples when in strict mode 56 | --force-revision Optional: if True it will check if file contains 57 | correct revision in file name. If it doesnt it will 58 | automatically add the correct revision to the filename 59 | --debug DEBUG Optional: debug level - determines the amount of debug 60 | info printed to console; default is 0 (no debug info 61 | printed) 62 | --version show program's version number and exit 63 | ``` 64 | 65 | The following behavior is implemented with respect to the "strict" and "strict-exmaples" options (none of the other options influence this behavior): 66 | 67 | * No options -- all yang modules found in the source file will be extracted and yang files created. 68 | * ```--strict``` -- only yang modules bracketed by \ and \ will be extracted 69 | * ```--strict --strict-examples``` -- only yang module **outside** of \ and \ **and** with a name starting with "example-" will be extracted. 70 | 71 | Please note: 72 | 73 | * Some errors will be generated to aid in debugging the content of modules. For example: 74 | 75 | ``` 76 | ERROR: 'test-file.txt', Line 21 - Yang module 'ex-error' with no and not starting with 'example-' 77 | ERROR: 'test-file.txt', Line 47 - Yang module 'example-error' with and starting with 'example-' 78 | ``` 79 | 80 | * If any yang modules that will be extracted already exist, the tool will exit without creating any yang modules 81 | 82 | * If there are syntactic errors such as a yang module statement nested in a yang module, the tool will exit without creating any yang modules 83 | 84 | ## Testing 85 | 86 | xym has a simple set of tests exercising a subset of functionality. Woth xym installed, these may be invoked while in the test subdirectory thus: 87 | 88 | ``` 89 | $ cd test 90 | $ python -m unittest test 91 | ``` 92 | 93 | Expected output is: 94 | 95 | ``` 96 | $ python -m unittest xym 97 | ERROR: 'test-file.txt', Line 21 - Yang module 'ex-error' with no and not starting with 'example-' 98 | ERROR: 'test-file.txt', Line 47 - Yang module 'example-error' with and starting with 'example-' 99 | .ERROR: 'test-file.txt', Line 21 - Yang module 'ex-error' with no and not starting with 'example-' 100 | ERROR: 'test-file.txt', Line 47 - Yang module 'example-error' with and starting with 'example-' 101 | .ERROR: 'test-file.txt', Line 21 - Yang module 'ex-error' with no and not starting with 'example-' 102 | ERROR: 'test-file.txt', Line 47 - Yang module 'example-error' with and starting with 'example-' 103 | . 104 | ---------------------------------------------------------------------- 105 | Ran 3 tests in 0.004s 106 | 107 | OK 108 | $ 109 | ``` 110 | -------------------------------------------------------------------------------- /bin/xym: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import argparse 3 | from xym import xym 4 | from xym import __version__ 5 | import re 6 | 7 | 8 | def run(): 9 | parser = argparse.ArgumentParser(description='Extracts one or more YANG' 10 | 'models from an IETF RFC or draft text file') 11 | parser.add_argument( 12 | "source", 13 | help="The URL or file name of the RFC/draft text from which to get the model") 14 | parser.add_argument( 15 | "--rfcxml", action='store_true', default=False, 16 | help="Parse a file in RFCXMLv3 format") 17 | parser.add_argument( 18 | "--srcdir", default='.', 19 | help="Optional: directory where to find the source text; default is './'") 20 | parser.add_argument( 21 | "--dstdir", default='.', 22 | help="Optional: directory where to put the extracted YANG module(s); default is './'") 23 | parser.add_argument( 24 | "--strict", type=bool, default=False, 25 | help='Optional flag that determines syntax enforcement; ' 26 | "'If set to 'True', the / " 27 | "tags are required; default is 'False'") 28 | parser.add_argument( 29 | "--strict-name", action='store_true', default=False, 30 | help="Optional flag that determines name enforcement; " 31 | "If set to 'True', name will be resolved from module " 32 | "itself and not from name given in the document;" 33 | " default is 'False'") 34 | parser.add_argument( 35 | "--strict-examples", action='store_true', default=False, 36 | help="Only output valid examples when in strict mode") 37 | parser.add_argument( 38 | "--write-dict", action='store_true', 39 | default=False, help="Optional: write email and module mapping") 40 | parser.add_argument( 41 | "--debug", type=int, default=0, 42 | help="Optional: debug level - determines the amount of debug " 43 | "info printed to console; default is 0 (no debug info printed)") 44 | parser.add_argument( 45 | "--force-revision-pyang", action='store_true', default=False, 46 | help="Optional: if True it will check if file contains correct revision in file name." 47 | "If it doesnt it will automatically add the correct revision to the filename using pyang") 48 | parser.add_argument( 49 | "--force-revision-regexp", action='store_true', default=False, 50 | help="Optional: if True it will check if file contains correct revision in file name." 51 | "If it doesnt it will automatically add the correct revision to the filename using regular" 52 | " expression") 53 | parser.add_argument("--extract-code-snippets", action="store_true", default=False, 54 | help="Optional: if True all the code snippets from the RFC/draft will be extracted. " 55 | "If the source argument is a URL and this argument is set to True, " 56 | "please be sure that the code-snippets-dir argument is provided, " 57 | "otherwise this value would be overwritten to False.") 58 | parser.add_argument("--code-snippets-dir", type=str, default='', 59 | help="Optional: Directory where to store code snippets extracted from the RFC/draft." 60 | "If this argument isn't provided and the source argument isn't a URL, " 61 | "then it will be set to the dstdir + 'code-snippets' + source(without file extension). " 62 | "If this argument isn't provided and the source argument is a URL, " 63 | "then code snippets wouldn't be extracted") 64 | parser.add_argument("--add-line-refs", action='store_true', default=False, 65 | help="Optional: if present, comments are added to each " 66 | "line in the extracted YANG module that contain " 67 | "the reference to the line number in the " 68 | "original RFC/Draft text file from which the " 69 | "line was extracted.") 70 | parser.add_argument( 71 | "--version", action='version', 72 | version='%(prog)s {version}'.format(version=__version__)) 73 | group = parser.add_mutually_exclusive_group() 74 | group.add_argument( 75 | "--parse-only-modules", nargs='+', 76 | help="Optional: it will parse only modules added in the list in arguments." 77 | ) 78 | group.add_argument( 79 | "--skip-modules", nargs='+', 80 | help="Optional: it will skip modules added in the list in arguments." 81 | ) 82 | 83 | args = parser.parse_args() 84 | 85 | extracted_models = xym.xym(args.source, 86 | args.srcdir, 87 | args.dstdir, 88 | args.strict, 89 | args.strict_name, 90 | args.strict_examples, 91 | args.debug, 92 | force_revision_pyang=args.force_revision_pyang, 93 | force_revision_regexp=args.force_revision_regexp, 94 | skip_modules=args.skip_modules, 95 | parse_only_modules=args.parse_only_modules, 96 | rfcxml=args.rfcxml, 97 | extract_code_snippets=args.extract_code_snippets, 98 | code_snippets_dir=args.code_snippets_dir, 99 | add_line_refs=args.add_line_refs, 100 | ) 101 | if len(extracted_models) > 0: 102 | if args.strict: 103 | print("Created the following models that conform to the strict guidelines:") 104 | else: 105 | print("Created the following models:") 106 | for em in extracted_models: 107 | print('%s' % em) 108 | if args.write_dict: 109 | url = re.compile( 110 | r'^(?:http|ftp)s?://' # http:// or https:// 111 | r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain 112 | r'localhost|' # localhost... 113 | r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip 114 | r'(?::\d+)?' # optional port 115 | r'(?:/?|[/?]\S+)$', re.IGNORECASE) 116 | fqfn = args.dstdir + '/yang.dict' 117 | is_url = url.match(args.source) 118 | if is_url: 119 | draft_file = args.source.rsplit('/', 1)[1] 120 | else: 121 | draft_file = args.source 122 | draft_name = draft_file.split('.', 1)[0] 123 | if draft_name.startswith("draft"): 124 | draft_email = draft_name.rsplit('-', 1)[0] + "@ietf.org" 125 | else: 126 | draft_email = draft_name + "@ietf.org" 127 | with open(fqfn, "a") as of: 128 | for em in extracted_models: 129 | of.write('%s : %s\n' % (em.split('@', 1)[0], draft_email)) 130 | of.close() 131 | else: 132 | print('No models created.') 133 | 134 | 135 | if __name__ == '__main__': 136 | run() 137 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests>=2.6 2 | pyang>=2.5.0 3 | lxml -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description_file = README.md 3 | 4 | [bdist_wheel] 5 | universal=1 6 | 7 | 8 | # See the docstring in versioneer.py for instructions. Note that you must 9 | # re-run 'versioneer.py setup' after changing this section, and commit the 10 | # resulting files. 11 | 12 | [versioneer] 13 | VCS = git 14 | style = pep440 15 | versionfile_source = xym/_version.py 16 | tag_prefix = v 17 | 18 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import versioneer 3 | from setuptools import setup 4 | 5 | def read(fname): 6 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 7 | 8 | #parse requirements 9 | req_lines = [line.strip() for line in open("requirements.txt").readlines()] 10 | install_reqs = list(filter(None, req_lines)) 11 | 12 | setup( 13 | version=versioneer.get_version(), 14 | cmdclass=versioneer.get_cmdclass(), 15 | name='xym', 16 | description = ('A tool to fetch and extract YANG modules from IETF RFCs and Drafts'), 17 | long_description="xym is a simple tool for fetching and extracting YANG modules from IETF RFCs and drafts as local files and from URLs.", 18 | packages=['xym'], 19 | scripts=['bin/xym'], 20 | author='Jan Medved', 21 | author_email='jmedved@cisco.com', 22 | license='New-style BSD', 23 | url='https://github.com/xym-tool/xym', 24 | install_requires=install_reqs, 25 | include_package_data=True, 26 | keywords=['yang', 'extraction'], 27 | python_requires='>=3.4', 28 | classifiers=[], 29 | ) 30 | -------------------------------------------------------------------------------- /test/resources/test-file-no-file-after-code-begins: -------------------------------------------------------------------------------- 1 | Network Working Group B. Lengyel 2 | Request for Comments: 5717 Ericsson 3 | Category: Standards Track M. Bjorklund 4 | Tail-f Systems 5 | December 2009 6 | 7 | 8 | Partial Lock Remote Procedure Call (RPC) for NETCONF 9 | 10 | Abstract 11 | 12 | The Network Configuration protocol (NETCONF) defines the lock and 13 | unlock Remote Procedure Calls (RPCs), used to lock entire 14 | configuration datastores. In some situations, a way to lock only 15 | parts of a configuration datastore is required. This document 16 | defines a capability-based extension to the NETCONF protocol for 17 | locking portions of a configuration datastore. 18 | 19 | Status of This Memo 20 | 21 | This document specifies an Internet standards track protocol for the 22 | Internet community, and requests discussion and suggestions for 23 | improvements. Please refer to the current edition of the "Internet 24 | Official Protocol Standards" (STD 1) for the standardization state 25 | and status of this protocol. Distribution of this memo is unlimited. 26 | 27 | Copyright Notice 28 | 29 | Copyright (c) 2009 IETF Trust and the persons identified as the 30 | document authors. All rights reserved. 31 | 32 | This document is subject to BCP 78 and the IETF Trust's Legal 33 | Provisions Relating to IETF Documents 34 | (http://trustee.ietf.org/license-info) in effect on the date of 35 | publication of this document. Please review these documents 36 | carefully, as they describe your rights and restrictions with respect 37 | to this document. Code Components extracted from this document must 38 | include Simplified BSD License text as described in Section 4.e of 39 | the Trust Legal Provisions and are provided without warranty as 40 | described in the BSD License. 41 | 42 | This document may contain material from IETF Documents or IETF 43 | Contributions published or made publicly available before November 44 | 10, 2008. The person(s) controlling the copyright in some of this 45 | material may not have granted the IETF Trust the right to allow 46 | modifications of such material outside the IETF Standards Process. 47 | Without obtaining an adequate license from the person(s) controlling 48 | the copyright in such materials, this document may not be modified 49 | 50 | 51 | 52 | Lengyel & Bjorklund Standards Track [Page 1] 53 | 54 | 55 | RFC 5717 Partial Lock RPC for NETCONF December 2009 56 | 57 | 58 | outside the IETF Standards Process, and derivative works of it may 59 | not be created outside the IETF Standards Process, except to format 60 | it for publication as an RFC or to translate it into languages other 61 | than English. 62 | 63 | Table of Contents 64 | 65 | 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 3 66 | 1.1. Definition of Terms . . . . . . . . . . . . . . . . . . . 3 67 | 2. Partial Locking Capability . . . . . . . . . . . . . . . . . . 3 68 | 2.1. Overview . . . . . . . . . . . . . . . . . . . . . . . . . 3 69 | 2.1.1. Usage Scenarios . . . . . . . . . . . . . . . . . . . 4 70 | 2.2. Dependencies . . . . . . . . . . . . . . . . . . . . . . . 5 71 | 2.3. Capability Identifier . . . . . . . . . . . . . . . . . . 5 72 | 2.4. New Operations . . . . . . . . . . . . . . . . . . . . . . 5 73 | 2.4.1. . . . . . . . . . . . . . . . . . . . . 5 74 | 2.4.2. . . . . . . . . . . . . . . . . . . . 10 75 | 2.5. Modifications to Existing Operations . . . . . . . . . . . 10 76 | 2.6. Interactions with Other Capabilities . . . . . . . . . . . 11 77 | 2.6.1. Candidate Configuration Capability . . . . . . . . . . 11 78 | 2.6.2. Confirmed Commit Capability . . . . . . . . . . . . . 11 79 | 2.6.3. Distinct Startup Capability . . . . . . . . . . . . . 11 80 | 3. Security Considerations . . . . . . . . . . . . . . . . . . . 12 81 | 4. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 12 82 | 5. Acknowledgements . . . . . . . . . . . . . . . . . . . . . . . 13 83 | 6. References . . . . . . . . . . . . . . . . . . . . . . . . . . 13 84 | 6.1. Normative References . . . . . . . . . . . . . . . . . . . 13 85 | 6.2. Informative References . . . . . . . . . . . . . . . . . . 13 86 | Appendix A. XML Schema for Partial Locking (Normative) . . . . . 14 87 | Appendix B. YANG Module for Partial Locking (Non-Normative) . . . 17 88 | Appendix C. Usage Example - Reserving Nodes for Future 89 | Editing (Non-Normative) . . . . . . . . . . . . . . . 19 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | Lengyel & Bjorklund Standards Track [Page 2] 110 | 111 | 112 | RFC 5717 Partial Lock RPC for NETCONF December 2009 113 | 114 | 115 | 1. Introduction 116 | 117 | The [NETCONF] protocol describes the lock and unlock operations that 118 | operate on entire configuration datastores. Often, multiple 119 | management sessions need to be able to modify the configuration of a 120 | managed device in parallel. In these cases, locking only parts of a 121 | configuration datastore is needed. This document defines a 122 | capability-based extension to the NETCONF protocol to support partial 123 | locking of the NETCONF running datastore using a mechanism based on 124 | the existing XPath filtering mechanisms. 125 | 126 | 1.1. Definition of Terms 127 | 128 | The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", 129 | "SHOULD", "SHOULD NOT", "RECOMMENDED", "NOT RECOMMENDED", "MAY", and 130 | "OPTIONAL" in this document are to be interpreted as described in BCP 131 | 14, [RFC2119]. 132 | 133 | Additionally, the following terms are defined: 134 | 135 | o Instance Identifier: an XPath expression identifying a specific 136 | node in the conceptual XML datastore. It contains an absolute 137 | path expression in abbreviated syntax, where predicates are used 138 | only to specify values for nodes defined as keys to distinguish 139 | multiple instances. 140 | 141 | o Scope of the lock: initially, the set of nodes returned by the 142 | XPath expressions in a successful partial-lock operation. The set 143 | might be modified if some of the nodes are deleted by the session 144 | owning the lock. 145 | 146 | o Protected area: the set of nodes that are protected from 147 | modification by the lock. This set consists of nodes in the scope 148 | of the lock and nodes in subtrees under them. 149 | 150 | 2. Partial Locking Capability 151 | 152 | 2.1. Overview 153 | 154 | The :partial-lock capability indicates that the device supports the 155 | locking of its configuration with a more limited scope than a 156 | complete configuration datastore. The scope to be locked is 157 | specified by using restricted or full XPath expressions. Partial 158 | locking only affects configuration data and only the running 159 | datastore. The candidate or the start-up datastore are not affected. 160 | 161 | 162 | 163 | 164 | 165 | 166 | Lengyel & Bjorklund Standards Track [Page 3] 167 | 168 | 169 | RFC 5717 Partial Lock RPC for NETCONF December 2009 170 | 171 | 172 | The system MUST ensure that configuration resources covered by the 173 | lock are not modified by other NETCONF or non-NETCONF management 174 | operations such as Simple Network Management Protocol (SNMP) and the 175 | Command Line Interface (CLI). 176 | 177 | The duration of the partial lock begins when the partial lock is 178 | granted and lasts until (1) either the corresponding 179 | operation succeeds or (2) the NETCONF session terminates. 180 | 181 | A NETCONF session MAY have multiple parts of the running datastore 182 | locked using partial lock operations. 183 | 184 | The operation returns a lock-id to identify each 185 | successfully acquired lock. The lock-id is unique at any given time 186 | for a NETCONF server for all partial-locks granted to any NETCONF or 187 | non-NETCONF sessions. 188 | 189 | 2.1.1. Usage Scenarios 190 | 191 | In the following, we describe a few scenarios for partial locking. 192 | Besides the two described here, there are many other usage scenarios 193 | possible. 194 | 195 | 2.1.1.1. Multiple Managers Handling the Writable Running Datastore with 196 | Overlapping Sections 197 | 198 | Multiple managers are handling the same NETCONF agent simultaneously. 199 | The agent is handled via the writable running datastore. Each 200 | manager has his or her own task, which might involve the modification 201 | of overlapping sections of the datastore. 202 | 203 | After collecting and analyzing input and preparing the NETCONF 204 | operations off-line, the manager locks the areas that are important 205 | for his task using one single operation. The manager 206 | executes a number of operations to modify the 207 | configuration, then releases the partial-lock. The lock should be 208 | held for the shortest possible time (e.g., seconds rather than 209 | minutes). The manager should collect all human input before locking 210 | anything. As each manager locks only a part of the data model, 211 | usually multiple operators can execute the operations 212 | simultaneously. 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | Lengyel & Bjorklund Standards Track [Page 4] 224 | 225 | 226 | RFC 5717 Partial Lock RPC for NETCONF December 2009 227 | 228 | 229 | 2.1.1.2. Multiple Managers Handling the Writable Running Datastore, 230 | Distinct Management Areas 231 | 232 | Multiple managers are handling the same NETCONF agent simultaneously. 233 | The agent is handled via the writable running datastore. The agent's 234 | data model contains a number of well-defined separate areas that can 235 | be configured without impacting other areas. An example can be a 236 | server with multiple applications running on it, or a number of 237 | network elements with a common NETCONF agent for management. 238 | 239 | Each manager has his or her own task, which does not involve the 240 | modification of overlapping sections of the datastore. 241 | 242 | The manager locks his area with a operation, uses a 243 | number of commands to modify it, and later releases the 244 | lock. As each manager has his functional area assigned to him, and 245 | he locks only that area, multiple managers can edit the configuration 246 | simultaneously. Locks can be held for extended periods (e.g., 247 | minutes, hours), as this will not hinder other managers. 248 | 249 | This scenario assumes that the global lock operation from [NETCONF] 250 | is not used. 251 | 252 | 2.2. Dependencies 253 | 254 | The device MUST support restricted XPath expressions in the select 255 | element, as described in Section 2.4.1. Optionally, if the :xpath 256 | capability is also supported (as defined in [NETCONF], Section 8.9. 257 | "XPath Capability"), the device MUST also support using any XPath 1.0 258 | expression in the select element. 259 | 260 | 2.3. Capability Identifier 261 | 262 | urn:ietf:params:netconf:capability:partial-lock:1.0 263 | 264 | 2.4. New Operations 265 | 266 | 2.4.1. 267 | 268 | The operation allows the client to lock a portion of 269 | the running datastore. The portion to lock is specified with XPath 270 | expressions in the "select" elements in the operation. 271 | Each XPath expression MUST return a node set. 272 | 273 | When a NETCONF session holds a lock on a node, no other session or 274 | non-NETCONF mechanism of the system can change that node or any node 275 | in the hierarchy of nodes beneath it. 276 | 277 | 278 | 279 | 280 | Lengyel & Bjorklund Standards Track [Page 5] 281 | 282 | 283 | RFC 5717 Partial Lock RPC for NETCONF December 2009 284 | 285 | 286 | Locking a node protects the node itself and the complete subtree 287 | under the node from modification by others. The set of locked nodes 288 | is called the scope of the lock, while all the locked nodes and the 289 | nodes in the subtrees under them make up the protected area. 290 | 291 | The XPath expressions are evaluated only once: at lock time. 292 | Thereafter, the scope of the lock is maintained as a set of nodes, 293 | i.e., the returned nodeset, and not by the XPath expression. If the 294 | configuration data is later altered in a way that would make the 295 | original XPath expressions evaluate to a different set of nodes, this 296 | does not affect the scope of the partial lock. 297 | 298 | Let's say the agent's data model includes a list of interface nodes. 299 | If the XPath expression in the partial-lock operation covers all 300 | interface nodes at locking, the scope of the lock will be maintained 301 | as the list of interface nodes at the time when the lock was granted. 302 | If someone later creates a new interface, this new interface will not 303 | be included in the locked-nodes list created previously so the new 304 | interface will not be locked. 305 | 306 | A operation MUST be handled atomically by the NETCONF 307 | server. The server either locks all requested parts of the datastore 308 | or none. If during the operation one of the requested 309 | parts cannot be locked, the server MUST unlock all parts that have 310 | already been locked during that operation. 311 | 312 | If a node in the scope of the lock is deleted by the session owning 313 | the lock, it is removed from the scope of the lock, so any other 314 | session or non-NETCONF mechanism can recreate it. If all nodes in 315 | the scope of the lock are deleted, the lock will still be present. 316 | However, its scope will become empty (since the lock will not cover 317 | any nodes). 318 | 319 | A NETCONF server that supports partial locking MUST be able to grant 320 | multiple simultaneous partial locks to a single NETCONF session. If 321 | the protected area of the individual locks overlap, nodes in the 322 | common area MUST be protected until all of the overlapping locks are 323 | released. 324 | 325 | A operation MUST fail if: 326 | 327 | o Any NETCONF session (including the current session) owns the 328 | global lock on the running datastore. 329 | 330 | o Any part of the area to be protected is already locked (or 331 | protected by partial locking) by another management session, 332 | including other NETCONF sessions using or any other 333 | non-NETCONF management method. 334 | 335 | 336 | 337 | Lengyel & Bjorklund Standards Track [Page 6] 338 | 339 | 340 | RFC 5717 Partial Lock RPC for NETCONF December 2009 341 | 342 | 343 | o The requesting user is not successfully authenticated. 344 | 345 | o The NETCONF server implements access control and the locking user 346 | does not have sufficient access rights. The exact handling of 347 | access rights is outside the scope of this document, but it is 348 | assumed that there is an access control system that MAY deny or 349 | allow the operation. 350 | 351 | The operation is designed for simplicity, so when a 352 | partial lock is executed, you get what you asked for: a set of nodes 353 | that are locked for writing. 354 | 355 | As a consequence, users must observe the following: 356 | 357 | o Locking does not affect read operations. 358 | 359 | o If part of the running datastore is locked, this has no effect on 360 | any unlocked parts of the datastore. If this is a problem (e.g., 361 | changes depend on data values or nodes outside the protected part 362 | of the datastore), these nodes SHOULD be included in the protected 363 | area of the lock. 364 | 365 | o Configuration data can be edited both inside and outside the 366 | protected area of a lock. It is the responsibility of the NETCONF 367 | client application to lock all relevant parts of the datastore 368 | that are crucial for a specific management action. 369 | 370 | Note: The operation does not modify the global 371 | operation defined in the base NETCONF protocol [NETCONF]. If part of 372 | the running datastore is already locked by , then a 373 | global lock for the running datastore MUST fail even if the global 374 | lock is requested by the NETCONF session that owns the partial lock. 375 | 376 | 2.4.1.1. Parameters, Results, Examples 377 | 378 | Parameters: 379 | 380 | select: One or more 'select' elements, each containing an XPath 381 | expression. The XPath expression is evaluated in a context 382 | where the context node is the root of the server's 383 | conceptual data model, and the set of namespace declarations 384 | are those in scope on the select element. 385 | 386 | The nodes returned from the select expressions are reported in the 387 | rpc-reply message. 388 | 389 | Each select expression MUST return a node set, and at least one of 390 | the node sets MUST be non-empty. 391 | 392 | 393 | 394 | Lengyel & Bjorklund Standards Track [Page 7] 395 | 396 | 397 | RFC 5717 Partial Lock RPC for NETCONF December 2009 398 | 399 | 400 | If the device supports the :xpath capability, any valid XPath 1.0 401 | expression can be used. If the device does not support the 402 | :xpath capability, the XPath expression MUST be limited to an 403 | Instance Identifier expression. An Instance Identifier is an 404 | absolute path expression in abbreviated syntax, where predicates 405 | are used only to specify values for nodes defined as keys to 406 | distinguish multiple instances. 407 | 408 | Example: Lock virtual router 1 and interface eth1 409 | 410 | 414 | 415 | 418 | 421 | 422 | 423 | 424 | 428 | 127 429 | 430 | /rte:routing/rte:virtualRouter[rte:routerName='router1'] 431 | 432 | 433 | /if:interfaces/if:interface[if:id='eth1'] 434 | 435 | 436 | 437 | Note: The XML Schema in [NETCONF] has a known bug that requires the 438 | XML element in a . This means that the above 439 | examples will not validate using the XML Schema found in [NETCONF]. 440 | 441 | Positive Response: 442 | 443 | If the device was able to satisfy the request, an is sent 444 | with a element (lock identifier) in the 445 | element. A list of locked nodes is also returned in Instance 446 | Identifier format. 447 | 448 | 449 | 450 | 451 | Lengyel & Bjorklund Standards Track [Page 8] 452 | 453 | 454 | RFC 5717 Partial Lock RPC for NETCONF December 2009 455 | 456 | 457 | Negative Response: 458 | 459 | If any select expression is an invalid XPath expression, the is 'invalid-value'. 461 | 462 | If any select expression returns something other than a node set, the 463 | is 'invalid-value', and the is 'not-a- 464 | node-set'. 465 | 466 | If all the select expressions return an empty node set, the is 'operation-failed', and the is 'no-matches'. 468 | 469 | If the :xpath capability is not supported and the XPath expression is 470 | not an Instance Identifier, the is 'invalid-value', the 471 | is 'invalid-lock-specification'. 472 | 473 | If access control denies the partial lock, the is 474 | 'access-denied'. Access control SHOULD be checked before checking 475 | for conflicting locks to avoid giving out information about other 476 | sessions to an unauthorized client. 477 | 478 | If a lock is already held by another session on any node within the 479 | subtrees to be locked, the element is 'lock-denied' and 480 | the element includes the of the lock owner. 481 | If the lock is held by a non-NETCONF session, a of 0 482 | (zero) SHOULD be included. The same error response is returned if 483 | the requesting session already holds the (global) lock for the 484 | running datastore. 485 | 486 | If needed, the returned session-id may be used to the 487 | NETCONF session holding the lock. 488 | 489 | 2.4.1.2. Deadlock Avoidance 490 | 491 | As with most locking systems, it is possible that two management 492 | sessions trying to lock different parts of the configuration could 493 | become deadlocked. To avoid this situation, clients SHOULD lock 494 | everything they need in one operation. If locking fails, the client 495 | MUST back-off, release any previously acquired locks, and SHOULD 496 | retry the procedure after waiting some randomized time interval. 497 | 498 | 499 | 500 | 501 | 502 | 503 | 504 | 505 | 506 | 507 | 508 | Lengyel & Bjorklund Standards Track [Page 9] 509 | 510 | 511 | RFC 5717 Partial Lock RPC for NETCONF December 2009 512 | 513 | 514 | 2.4.2. 515 | 516 | The operation unlocks the parts of the running datastore that were 517 | previously locked using during the same session. The 518 | operation unlocks the parts that are covered by the lock identified 519 | by the lock-id parameter. In case of multiple potentially 520 | overlapping locks, only the lock identified by the lock-id is 521 | removed. 522 | 523 | Parameters: 524 | 525 | lock-id: Identity of the lock to be unlocked. This lock-id MUST 526 | have been received as a response to a lock request by the 527 | manager during the current session, and MUST NOT have been 528 | sent in a previous unlock request. 529 | 530 | Example: Unlock a previously created lock 531 | 532 | 535 | 536 | 127 537 | 538 | 539 | 540 | Positive Response: 541 | 542 | If the device was able to satisfy the request, an is sent 543 | that contains an element. A positive response MUST be sent even 544 | if all of the locked parts of the datastore have already been 545 | deleted. 546 | 547 | Negative Response: 548 | 549 | If the parameter does not identify a lock that is owned by 550 | the session, an 'invalid-value' error is returned. 551 | 552 | 2.5. Modifications to Existing Operations 553 | 554 | A successful partial lock will cause a subsequent operation to fail 555 | if that operation attempts to modify nodes in the protected area of 556 | the lock and is executed in a NETCONF session other than the session 557 | that has been granted the lock. The 'in-use' and the 558 | 'locked' is returned. All operations that modify the 559 | 560 | 561 | 562 | 563 | 564 | 565 | Lengyel & Bjorklund Standards Track [Page 10] 566 | 567 | 568 | RFC 5717 Partial Lock RPC for NETCONF December 2009 569 | 570 | 571 | running datastore are affected, including: , , , , and . If 573 | partial lock prevents from modifying some data, but the 574 | operation includes the continue-on-error option, modification of 575 | other parts of the datastore, which are not protected by partial 576 | locking, might still succeed. 577 | 578 | If the datastore contains nodes locked by partial lock, this will 579 | cause the (global) operation to fail. The element 580 | 'lock-denied' and an element including the 581 | of the lock owner will be returned. If the lock is held by a non- 582 | NETCONF session, a of 0 (zero) is returned. 583 | 584 | All of these operations are affected only if they are targeting the 585 | running datastore. 586 | 587 | 2.6. Interactions with Other Capabilities 588 | 589 | 2.6.1. Candidate Configuration Capability 590 | 591 | The candidate datastore cannot be locked using the 592 | operation. 593 | 594 | 2.6.2. Confirmed Commit Capability 595 | 596 | If: 597 | 598 | o a partial lock is requested for the running datastore, and 599 | 600 | o the NETCONF server implements the :confirmed-commit capability, 601 | and 602 | 603 | o there was a recent confirmed operation where the 604 | confirming operation has not been received 605 | 606 | then the lock MUST be denied, because if the confirmation does not 607 | arrive, the running datastore MUST be rolled back to its state before 608 | the commit. The NETCONF server might therefore need to modify the 609 | configuration. 610 | 611 | In this case, the 'in-use' and the 612 | 'outstanding-confirmed-commit' is returned. 613 | 614 | 2.6.3. Distinct Startup Capability 615 | 616 | The startup datastore cannot be locked using the 617 | operation. 618 | 619 | 620 | 621 | 622 | Lengyel & Bjorklund Standards Track [Page 11] 623 | 624 | 625 | RFC 5717 Partial Lock RPC for NETCONF December 2009 626 | 627 | 628 | 3. Security Considerations 629 | 630 | The same considerations are relevant as for the base NETCONF protocol 631 | [NETCONF]. and RPCs MUST only be 632 | allowed for an authenticated user. and RPCs SHOULD only be allowed for an authorized user. However, 634 | as NETCONF access control is not standardized and not a mandatory 635 | part of a NETCONF implementation, it is strongly recommended, but 636 | OPTIONAL (although nearly all implementations include some kind of 637 | access control). 638 | 639 | A lock (either a partial lock or a global lock) might prevent other 640 | users from configuring the system. The following mechanisms are in 641 | place to prevent the misuse of this possibility: 642 | 643 | A user, that is not successfully authenticated, MUST NOT be 644 | granted a partial lock. 645 | 646 | Only an authorized user SHOULD be able to request a partial lock. 647 | 648 | The partial lock is automatically released when a session is 649 | terminated regardless of how the session ends. 650 | 651 | The operation makes it possible to terminate other 652 | users' sessions. 653 | 654 | The NETCONF server MAY log partial lock requests in an audit 655 | trail. 656 | 657 | A lock that is hung for some reason (e.g., a broken TCP connection 658 | that the server has not yet recognized) can be released using another 659 | NETCONF session by explicitly killing the session owning that lock 660 | using the operation. 661 | 662 | Partial locking is not an authorization mechanism; it SHOULD NOT be 663 | used to provide security or access control. Partial locking SHOULD 664 | only be used as a mechanism for providing consistency when multiple 665 | managers are trying to configure the node. It is vital that users 666 | easily understand the exact scope of a lock. This is why the scope 667 | is determined when granting a lock and is not modified thereafter. 668 | 669 | 4. IANA Considerations 670 | 671 | This document registers one capability identifier URN from the 672 | "Network Configuration Protocol (NETCONF) Capability URNs" registry, 673 | and one URI for the NETCONF XML namespace in the "IETF XML registry" 674 | [RFC3688]. Note that the capability URN is compliant to [NETCONF], 675 | Section 10.3. 676 | 677 | 678 | 679 | Lengyel & Bjorklund Standards Track [Page 12] 680 | 681 | 682 | RFC 5717 Partial Lock RPC for NETCONF December 2009 683 | 684 | 685 | Index Capability Identifier 686 | ------------- --------------------------------------------------- 687 | :partial-lock urn:ietf:params:netconf:capability:partial-lock:1.0 688 | 689 | URI: urn:ietf:params:xml:ns:netconf:partial-lock:1.0 690 | 691 | Registrant Contact: The IESG. 692 | 693 | XML: N/A, the requested URI is an XML namespace. 694 | 695 | 5. Acknowledgements 696 | 697 | Thanks to Andy Bierman, Sharon Chisholm, Phil Shafer, David 698 | Harrington, Mehmet Ersue, Wes Hardaker, Juergen Schoenwaelder, Washam 699 | Fan, and many other members of the NETCONF WG for providing important 700 | input to this document. 701 | 702 | 6. References 703 | 704 | 6.1. Normative References 705 | 706 | [NETCONF] Enns, R., "NETCONF Configuration Protocol", RFC 4741, 707 | December 2006. 708 | 709 | [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate 710 | Requirement Levels", BCP 14, RFC 2119, March 1997. 711 | 712 | [RFC3688] Mealling, M., "The IETF XML Registry", BCP 81, RFC 3688, 713 | January 2004. 714 | 715 | 6.2. Informative References 716 | 717 | [YANG] Bjorklund, M., "YANG - A data modeling language for 718 | NETCONF", Work in Progress, December 2009. 719 | 720 | 721 | 722 | 723 | 724 | 725 | 726 | 727 | 728 | 729 | 730 | 731 | 732 | 733 | 734 | 735 | 736 | Lengyel & Bjorklund Standards Track [Page 13] 737 | 738 | 739 | RFC 5717 Partial Lock RPC for NETCONF December 2009 740 | 741 | 742 | Appendix A. XML Schema for Partial Locking (Normative) 743 | 744 | The following XML Schema defines the and operations: 746 | 747 | 748 | 749 | 750 | 755 | 756 | 757 | 758 | Schema defining the partial-lock and unlock operations. 759 | organization "IETF NETCONF Working Group" 760 | 761 | contact 762 | Netconf Working Group 763 | Mailing list: netconf@ietf.org 764 | Web: http://www.ietf.org/html.charters/netconf-charter.html 765 | 766 | Balazs Lengyel 767 | balazs.lengyel@ericsson.com 768 | 769 | revision 2009-10-19 770 | description Initial version, published as RFC 5717. 771 | 772 | 773 | 774 | 776 | 777 | 778 | 779 | 780 | A number identifying a specific 781 | partial-lock granted to a session. 782 | It is allocated by the system, and SHOULD 783 | be used in the unlock operation. 784 | 785 | 786 | 787 | 788 | 789 | 790 | 791 | 792 | 793 | Lengyel & Bjorklund Standards Track [Page 14] 794 | 795 | 796 | RFC 5717 Partial Lock RPC for NETCONF December 2009 797 | 798 | 799 | 800 | 801 | 802 | A NETCONF operation that locks parts of 803 | the running datastore. 804 | 805 | 806 | 807 | 808 | 809 | 811 | 812 | 813 | XPath expression that specifies the scope 814 | of the lock. An Instance Identifier 815 | expression must be used unless the :xpath 816 | capability is supported in which case any 817 | XPath 1.0 expression is allowed. 818 | 819 | 820 | 821 | 822 | 823 | 824 | 825 | 826 | 827 | 828 | 829 | A NETCONF operation that releases a previously acquired 830 | partial-lock. 831 | 832 | 833 | 834 | 835 | 836 | 837 | 838 | 839 | Identifies the lock to be released. MUST 840 | be the value received in the response to 841 | the partial-lock operation. 842 | 843 | 844 | 845 | 846 | 847 | 848 | 849 | 850 | Lengyel & Bjorklund Standards Track [Page 15] 851 | 852 | 853 | RFC 5717 Partial Lock RPC for NETCONF December 2009 854 | 855 | 856 | 857 | 858 | 859 | 860 | 862 | 863 | 864 | 866 | 867 | 868 | 869 | 870 | 871 | 872 | The content of the reply to a successful 873 | partial-lock request MUST conform to this complex type. 874 | 875 | 876 | 877 | 878 | 879 | 880 | Identifies the lock to be released. Must be the value 881 | received in the response to a partial-lock operation. 882 | 883 | 884 | 885 | 887 | 888 | 889 | List of locked nodes in the running datastore. 890 | 891 | 892 | 893 | 894 | 895 | 896 | 897 | 898 | 899 | 900 | 901 | 902 | 903 | 904 | 905 | 906 | 907 | Lengyel & Bjorklund Standards Track [Page 16] 908 | 909 | 910 | RFC 5717 Partial Lock RPC for NETCONF December 2009 911 | 912 | 913 | Appendix B. YANG Module for Partial Locking (Non-Normative) 914 | 915 | The following YANG module defines the and operations. The YANG language is defined in [YANG]. 917 | 918 | 919 | 920 | module ietf-netconf-partial-lock { 921 | 922 | namespace urn:ietf:params:xml:ns:netconf:partial-lock:1.0; 923 | prefix pl; 924 | 925 | organization "IETF Network Configuration (netconf) Working Group"; 926 | 927 | contact 928 | "Netconf Working Group 929 | Mailing list: netconf@ietf.org 930 | Web: http://www.ietf.org/html.charters/netconf-charter.html 931 | 932 | Balazs Lengyel 933 | Ericsson 934 | balazs.lengyel@ericsson.com"; 935 | 936 | description 937 | "This YANG module defines the and 938 | operations."; 939 | 940 | revision 2009-10-19 { 941 | description 942 | "Initial version, published as RFC 5717."; 943 | } 944 | 945 | typedef lock-id-type { 946 | type uint32; 947 | description 948 | "A number identifying a specific partial-lock granted to a session. 949 | It is allocated by the system, and SHOULD be used in the 950 | partial-unlock operation."; 951 | } 952 | 953 | rpc partial-lock { 954 | description 955 | "A NETCONF operation that locks parts of the running datastore."; 956 | input { 957 | leaf-list select { 958 | type string; 959 | min-elements 1; 960 | description 961 | 962 | 963 | 964 | Lengyel & Bjorklund Standards Track [Page 17] 965 | 966 | 967 | RFC 5717 Partial Lock RPC for NETCONF December 2009 968 | 969 | 970 | "XPath expression that specifies the scope of the lock. 971 | An Instance Identifier expression MUST be used unless the 972 | :xpath capability is supported, in which case any XPath 1.0 973 | expression is allowed."; 974 | } 975 | } 976 | output { 977 | leaf lock-id { 978 | type lock-id-type; 979 | description 980 | "Identifies the lock, if granted. The lock-id SHOULD be 981 | used in the partial-unlock rpc."; 982 | } 983 | leaf-list locked-node { 984 | type instance-identifier; 985 | min-elements 1; 986 | description 987 | "List of locked nodes in the running datastore"; 988 | } 989 | } 990 | } 991 | 992 | rpc partial-unlock { 993 | description 994 | "A NETCONF operation that releases a previously acquired 995 | partial-lock."; 996 | input { 997 | leaf lock-id { 998 | type lock-id-type; 999 | description 1000 | "Identifies the lock to be released. MUST be the value 1001 | received in the response to a partial-lock operation."; 1002 | } 1003 | } 1004 | } 1005 | } 1006 | 1007 | 1008 | 1009 | 1010 | 1011 | 1012 | 1013 | 1014 | 1015 | 1016 | 1017 | 1018 | 1019 | 1020 | 1021 | Lengyel & Bjorklund Standards Track [Page 18] 1022 | 1023 | 1024 | RFC 5717 Partial Lock RPC for NETCONF December 2009 1025 | 1026 | 1027 | Appendix C. Usage Example - Reserving Nodes for Future Editing 1028 | (Non-Normative) 1029 | 1030 | Partial lock cannot be used to lock non-existent nodes, which would 1031 | effectively attempt to reserve them for future use. To guarantee 1032 | that a node cannot be created by some other session, the parent node 1033 | should be locked, the top-level node of the new subtree created, and 1034 | then locked with another operation. After this, the 1035 | lock on the parent node should be removed. 1036 | 1037 | In this section, an example illustrating the above is given. 1038 | 1039 | We want to create Joe under , and start editing it. 1040 | Editing might take a number of minutes. We want to immediately lock 1041 | Joe so no one will touch it before we are finished with the editing. 1042 | 1043 | We also want to minimize locking other parts of the running datastore 1044 | as multiple managers might be adding users near simultaneously. 1045 | 1046 | First, we check what users are already defined. 1047 | 1048 | Step 1 - Read existing users 1049 | 1050 | 1052 | 1053 | 1054 | 1055 | 1056 | 1057 | 1058 | 1059 | 1060 | 1061 | 1062 | 1063 | 1064 | The NETCONF server sends the following reply. 1065 | 1066 | 1067 | 1068 | 1069 | 1070 | 1071 | 1072 | 1073 | 1074 | 1075 | 1076 | 1077 | 1078 | Lengyel & Bjorklund Standards Track [Page 19] 1079 | 1080 | 1081 | RFC 5717 Partial Lock RPC for NETCONF December 2009 1082 | 1083 | 1084 | Step 2 - Receiving existing data 1085 | 1086 | 1088 | 1089 | 1090 | 1091 | 1092 | fred 1093 | 8327 1094 | 1095 | 1096 | 1097 | 1098 | 1099 | 1100 | We want to add the new user Joe and immediately lock him using 1101 | partial locking. The way to do this, is to first lock all 1102 | nodes by locking the node. 1103 | 1104 | Note that if we would lock all the nodes using the select 1105 | expression '/usr:top/usr:users/usr:user'; this would not lock the new 1106 | user Joe, which we will create after locking. So we rather have to 1107 | lock the node. 1108 | 1109 | Step 3 - Lock users 1110 | 1111 | 1115 | 1116 | 1119 | 1120 | 1121 | 1122 | The NETCONF server grants the partial lock. The scope of the lock 1123 | includes only the node. The lock protects the node 1124 | and all nodes below it from modification (by other sessions). 1125 | 1126 | 1127 | 1128 | 1129 | 1130 | 1131 | 1132 | 1133 | 1134 | 1135 | Lengyel & Bjorklund Standards Track [Page 20] 1136 | 1137 | 1138 | RFC 5717 Partial Lock RPC for NETCONF December 2009 1139 | 1140 | 1141 | Step 4 - Receive lock 1142 | 1143 | 1147 | 1 1148 | 1149 | /usr:top/usr:users 1150 | 1151 | 1152 | 1153 | Next we create user Joe. Joe is protected by the lock received 1154 | above, as it is under the subtree rooted at the node. 1155 | 1156 | Step 5 - Create user Joe 1157 | 1158 | 1160 | 1161 | 1162 | 1163 | 1164 | 1165 | 1166 | 1167 | 1168 | Joe 1169 | 1170 | 1171 | 1172 | 1173 | 1174 | 1175 | 1176 | We receive a positive reply to the (not shown). Next 1177 | we request a lock, that locks only Joe, and release the lock 1178 | on the node. This will allow other managers to create 1179 | additional new users. 1180 | 1181 | 1182 | 1183 | 1184 | 1185 | 1186 | 1187 | 1188 | 1189 | 1190 | 1191 | 1192 | Lengyel & Bjorklund Standards Track [Page 21] 1193 | 1194 | 1195 | RFC 5717 Partial Lock RPC for NETCONF December 2009 1196 | 1197 | 1198 | Step 6 - Lock user Joe 1199 | 1200 | 1204 | 1205 | 1208 | 1209 | 1210 | 1211 | The NETCONF server grants the partial lock. The scope of this second 1212 | lock includes only the node with name Joe. The lock protects 1213 | all data below this particular node. 1214 | 1215 | Step 7 - Receive lock 1216 | 1217 | 1221 | 2 1222 | 1223 | /usr:top/usr:users/user[usr:name="Joe"]" 1224 | 1225 | 1226 | 1227 | The scope of the second lock is the node Joe. It protects 1228 | this node and any data below it (e.g., phone number). At this 1229 | point of time, these nodes are protected both by the first and second 1230 | lock. Next, we unlock the other s and the node, to 1231 | allow other managers to work on them. We still keep the second lock, 1232 | so the node Joe and the subtree below is still protected. 1233 | 1234 | Step 8 - Release lock on 1235 | 1236 | 1239 | 1240 | 1 1241 | 1242 | 1243 | 1244 | 1245 | 1246 | 1247 | 1248 | 1249 | Lengyel & Bjorklund Standards Track [Page 22] 1250 | 1251 | 1252 | RFC 5717 Partial Lock RPC for NETCONF December 2009 1253 | 1254 | 1255 | Authors' Addresses 1256 | 1257 | Balazs Lengyel 1258 | Ericsson 1259 | 1260 | EMail: balazs.lengyel@ericsson.com 1261 | 1262 | 1263 | Martin Bjorklund 1264 | Tail-f Systems 1265 | 1266 | EMail: mbj@tail-f.com 1267 | 1268 | 1269 | 1270 | 1271 | 1272 | 1273 | 1274 | 1275 | 1276 | 1277 | 1278 | 1279 | 1280 | 1281 | 1282 | 1283 | 1284 | 1285 | 1286 | 1287 | 1288 | 1289 | 1290 | 1291 | 1292 | 1293 | 1294 | 1295 | 1296 | 1297 | 1298 | 1299 | 1300 | 1301 | 1302 | 1303 | 1304 | 1305 | 1306 | Lengyel & Bjorklund Standards Track [Page 23] 1307 | -------------------------------------------------------------------------------- /test/resources/test-file-with-symbol: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Internet Engineering Task Force (IETF) A. Bierman 5 | Request for Comments: 6470 Brocade 6 | Category: Standards Track February 2012 7 | ISSN: 2070-1721 8 | 9 | 10 | Network Configuration Protocol (NETCONF) Base Notifications 11 | 12 | Abstract 13 | 14 | The Network Configuration Protocol (NETCONF) provides mechanisms to 15 | manipulate configuration datastores. However, client applications 16 | often need to be aware of common events, such as a change in NETCONF 17 | server capabilities, that may impact management applications. 18 | Standard mechanisms are needed to support the monitoring of the base 19 | system events within the NETCONF server. This document defines a 20 | YANG module that allows a NETCONF client to receive notifications for 21 | some common system events. 22 | 23 | Status of This Memo 24 | 25 | This is an Internet Standards Track document. 26 | 27 | This document is a product of the Internet Engineering Task Force 28 | (IETF). It represents the consensus of the IETF community. It has 29 | received public review and has been approved for publication by the 30 | Internet Engineering Steering Group (IESG). Further information on 31 | Internet Standards is available in Section 2 of RFC 5741. 32 | 33 | Information about the current status of this document, any errata, 34 | and how to provide feedback on it may be obtained at 35 | http://www.rfc-editor.org/info/rfc6470. 36 | 37 | Copyright Notice 38 | 39 | Copyright (c) 2012 IETF Trust and the persons identified as the 40 | document authors. All rights reserved. 41 | 42 | This document is subject to BCP 78 and the IETF Trust's Legal 43 | Provisions Relating to IETF Documents 44 | (http://trustee.ietf.org/license-info) in effect on the date of 45 | publication of this document. Please review these documents 46 | carefully, as they describe your rights and restrictions with respect 47 | to this document. Code Components extracted from this document must 48 | include Simplified BSD License text as described in Section 4.e of 49 | the Trust Legal Provisions and are provided without warranty as 50 | described in the Simplified BSD License. 51 | 52 | 53 | 54 | 55 | Bierman Standards Track [Page 1] 56 | 57 | 58 | RFC 6470 NETCONF Base Notifications February 2012 59 | 60 | 61 | Table of Contents 62 | 63 | 1. Introduction ....................................................2 64 | 1.1. Terminology ................................................2 65 | 2. YANG Module for NETCONF Base Notifications ......................3 66 | 2.1. Overview ...................................................3 67 | 2.2. Definitions ................................................4 68 | 3. IANA Considerations ............................................11 69 | 4. Security Considerations ........................................12 70 | 5. Acknowledgements ...............................................14 71 | 6. Normative References ...........................................14 72 | 73 | 1. Introduction 74 | 75 | The NETCONF protocol [RFC6241] provides mechanisms to manipulate 76 | configuration datastores. However, client applications often need to 77 | be aware of common events, such as a change in NETCONF server 78 | capabilities, that may impact management applications. Standard 79 | mechanisms are needed to support the monitoring of the base system 80 | events within the NETCONF server. This document defines a YANG 81 | module [RFC6020] that allows a NETCONF client to receive 82 | notifications for some common system events. 83 | 84 | 1.1. Terminology 85 | 86 | The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", 87 | "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this 88 | document are to be interpreted as described in [RFC2119]. 89 | 90 | The following terms are defined in [RFC6241]: 91 | 92 | o client 93 | o datastore 94 | o protocol operation 95 | o server 96 | 97 | The following terms are defined in [RFC5277]: 98 | 99 | o event 100 | o stream 101 | o subscription 102 | 103 | The following term is defined in [RFC6020]: 104 | 105 | o data node 106 | 107 | 108 | 109 | 110 | 111 | 112 | Bierman Standards Track [Page 2] 113 | 114 | 115 | RFC 6470 NETCONF Base Notifications February 2012 116 | 117 | 118 | 2. YANG Module for NETCONF Base Notifications 119 | 120 | 2.1. Overview 121 | 122 | The YANG module defined within this document specifies a small number 123 | of event notification messages for use within the 'NETCONF' stream, 124 | and accessible to clients via the subscription mechanism described in 125 | [RFC5277]. This module imports data types from the 'ietf-netconf' 126 | module defined in [RFC6241] and 'ietf-inet-types' module defined in 127 | [RFC6021]. 128 | 129 | These notifications pertain to configuration and monitoring portions 130 | of the managed system, not the entire system. A server MUST report 131 | events that are directly related to the NETCONF protocol. A server 132 | MAY report events for non-NETCONF management sessions, using the 133 | 'session-id' value of zero. 134 | 135 | This module defines the following notifications for the 'NETCONF' 136 | stream to notify a client application that the NETCONF server state 137 | has changed: 138 | 139 | netconf-config-change: 140 | Generated when the NETCONF server detects that the or 141 | configuration datastore has been changed by a management 142 | session. The notification summarizes the edits that have been 143 | detected. 144 | 145 | netconf-capability-change: 146 | Generated when the NETCONF server detects that the server 147 | capabilities have changed. Indicates which capabilities have been 148 | added, deleted, and/or modified. The manner in which a server 149 | capability is changed is outside the scope of this document. 150 | 151 | netconf-session-start: 152 | Generated when a NETCONF server detects that a NETCONF session has 153 | started. A server MAY generate this event for non-NETCONF 154 | management sessions. Indicates the identity of the user that 155 | started the session. 156 | 157 | netconf-session-end: 158 | Generated when a NETCONF server detects that a NETCONF session has 159 | terminated. A server MAY optionally generate this event for 160 | non-NETCONF management sessions. Indicates the identity of the 161 | user that owned the session, and why the session was terminated. 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | Bierman Standards Track [Page 3] 170 | 171 | 172 | RFC 6470 NETCONF Base Notifications February 2012 173 | 174 | 175 | netconf-confirmed-commit: 176 | Generated when a NETCONF server detects that a confirmed-commit 177 | event has occurred. Indicates the event and the current state of 178 | the confirmed-commit procedure in progress. 179 | 180 | 2.2. Definitions 181 | 182 | file="ietf-netconf-notifications@2011-12-09.yang" 183 | 184 | module ietf-netconf-notifications { 185 | 186 | namespace 187 | "urn:ietf:params:xml:ns:yang:ietf-netconf-notifications"; 188 | 189 | prefix ncn; 190 | 191 | import ietf-inet-types { prefix inet; } 192 | import ietf-netconf { prefix nc; } 193 | 194 | organization 195 | "IETF NETCONF (Network Configuration Protocol) Working Group"; 196 | 197 | contact 198 | "WG Web: 199 | WG List: 200 | 201 | WG Chair: Bert Wijnen 202 | 203 | 204 | WG Chair: Mehmet Ersue 205 | 206 | 207 | Editor: Andy Bierman 208 | "; 209 | 210 | description 211 | "This module defines a YANG data model for use with the 212 | NETCONF protocol that allows the NETCONF client to 213 | receive common NETCONF base event notifications. 214 | 215 | Copyright (c) 2012 IETF Trust and the persons identified as 216 | the document authors. All rights reserved. 217 | 218 | Redistribution and use in source and binary forms, with or 219 | without modification, is permitted pursuant to, and subject 220 | to the license terms contained in, the Simplified BSD License 221 | 222 | 223 | 224 | 225 | 226 | Bierman Standards Track [Page 4] 227 | 228 | 229 | RFC 6470 NETCONF Base Notifications February 2012 230 | 231 | 232 | set forth in Section 4.c of the IETF Trust's Legal Provisions 233 | Relating to IETF Documents 234 | (http://trustee.ietf.org/license-info). 235 | 236 | This version of this YANG module is part of RFC 6470; see 237 | the RFC itself for full legal notices."; 238 | 239 | revision "2012-02-06" { 240 | description 241 | "Initial version."; 242 | reference 243 | "RFC 6470: NETCONF Base Notifications"; 244 | } 245 | 246 | grouping common-session-parms { 247 | description 248 | "Common session parameters to identify a 249 | management session."; 250 | 251 | leaf username { 252 | type string; 253 | mandatory true; 254 | description 255 | "Name of the user for the session."; 256 | } 257 | 258 | leaf session-id { 259 | type nc:session-id-or-zero-type; 260 | mandatory true; 261 | description 262 | "Identifier of the session. 263 | A NETCONF session MUST be identified by a non-zero value. 264 | A non-NETCONF session MAY be identified by the value zero."; 265 | } 266 | 267 | leaf source-host { 268 | type inet:ip-address; 269 | description 270 | "Address of the remote host for the session."; 271 | } 272 | } 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | Bierman Standards Track [Page 5] 284 | 285 | 286 | RFC 6470 NETCONF Base Notifications February 2012 287 | 288 | 289 | grouping changed-by-parms { 290 | description 291 | "Common parameters to identify the source 292 | of a change event, such as a configuration 293 | or capability change."; 294 | 295 | container changed-by { 296 | description 297 | "Indicates the source of the change. 298 | If caused by internal action, then the 299 | empty leaf 'server' will be present. 300 | If caused by a management session, then 301 | the name, remote host address, and session ID 302 | of the session that made the change will be reported."; 303 | choice server-or-user { 304 | mandatory true; 305 | leaf server { 306 | type empty; 307 | description 308 | "If present, the change was caused 309 | by the server."; 310 | } 311 | 312 | case by-user { 313 | uses common-session-parms; 314 | } 315 | } // choice server-or-user 316 | } // container changed-by-parms 317 | } 318 | 319 | 320 | notification netconf-config-change { 321 | description 322 | "Generated when the NETCONF server detects that the 323 | or configuration datastore 324 | has been changed by a management session. 325 | The notification summarizes the edits that 326 | have been detected. 327 | 328 | The server MAY choose to also generate this 329 | notification while loading a datastore during the 330 | boot process for the device."; 331 | 332 | uses changed-by-parms; 333 | 334 | 335 | 336 | 337 | 338 | 339 | 340 | Bierman Standards Track [Page 6] 341 | 342 | 343 | RFC 6470 NETCONF Base Notifications February 2012 344 | 345 | 346 | leaf datastore { 347 | type enumeration { 348 | enum running { 349 | description "The datastore has changed."; 350 | } 351 | enum startup { 352 | description "The datastore has changed"; 353 | } 354 | } 355 | default "running"; 356 | description 357 | "Indicates which configuration datastore has changed."; 358 | } 359 | 360 | list edit { 361 | description 362 | "An edit record SHOULD be present for each distinct 363 | edit operation that the server has detected on 364 | the target datastore. This list MAY be omitted 365 | if the detailed edit operations are not known. 366 | The server MAY report entries in this list for 367 | changes not made by a NETCONF session (e.g., CLI)."; 368 | 369 | leaf target { 370 | type instance-identifier; 371 | description 372 | "Topmost node associated with the configuration change. 373 | A server SHOULD set this object to the node within 374 | the datastore that is being altered. A server MAY 375 | set this object to one of the ancestors of the actual 376 | node that was changed, or omit this object, if the 377 | exact node is not known."; 378 | } 379 | 380 | leaf operation { 381 | type nc:edit-operation-type; 382 | description 383 | "Type of edit operation performed. 384 | A server MUST set this object to the NETCONF edit 385 | operation performed on the target datastore."; 386 | } 387 | } // list edit 388 | } // notification netconf-config-change 389 | 390 | 391 | 392 | 393 | 394 | 395 | 396 | 397 | Bierman Standards Track [Page 7] 398 | 399 | 400 | RFC 6470 NETCONF Base Notifications February 2012 401 | 402 | 403 | notification netconf-capability-change { 404 | description 405 | "Generated when the NETCONF server detects that 406 | the server capabilities have changed. 407 | Indicates which capabilities have been added, deleted, 408 | and/or modified. The manner in which a server 409 | capability is changed is outside the scope of this 410 | document."; 411 | 412 | uses changed-by-parms; 413 | 414 | leaf-list added-capability { 415 | type inet:uri; 416 | description 417 | "List of capabilities that have just been added."; 418 | } 419 | 420 | leaf-list deleted-capability { 421 | type inet:uri; 422 | description 423 | "List of capabilities that have just been deleted."; 424 | } 425 | 426 | leaf-list modified-capability { 427 | type inet:uri; 428 | description 429 | "List of capabilities that have just been modified. 430 | A capability is considered to be modified if the 431 | base URI for the capability has not changed, but 432 | one or more of the parameters encoded at the end of 433 | the capability URI have changed. 434 | The new modified value of the complete URI is returned."; 435 | } 436 | } // notification netconf-capability-change 437 | 438 | 439 | notification netconf-session-start { 440 | description 441 | "Generated when a NETCONF server detects that a 442 | NETCONF session has started. A server MAY generate 443 | this event for non-NETCONF management sessions. 444 | Indicates the identity of the user that started 445 | the session."; 446 | uses common-session-parms; 447 | } // notification netconf-session-start 448 | 449 | 450 | 451 | 452 | 453 | 454 | Bierman Standards Track [Page 8] 455 | 456 | 457 | RFC 6470 NETCONF Base Notifications February 2012 458 | 459 | 460 | notification netconf-session-end { 461 | description 462 | "Generated when a NETCONF server detects that a 463 | NETCONF session has terminated. 464 | A server MAY optionally generate this event for 465 | non-NETCONF management sessions. Indicates the 466 | identity of the user that owned the session, 467 | and why the session was terminated."; 468 | 469 | uses common-session-parms; 470 | 471 | leaf killed-by { 472 | when "../termination-reason = 'killed'"; 473 | type nc:session-id-type; 474 | description 475 | "The ID of the session that directly caused this session 476 | to be abnormally terminated. If this session was abnormally 477 | terminated by a non-NETCONF session unknown to the server, 478 | then this leaf will not be present."; 479 | } 480 | 481 | leaf termination-reason { 482 | type enumeration { 483 | enum "closed" { 484 | description 485 | "The session was terminated by the client in normal 486 | fashion, e.g., by the NETCONF 487 | protocol operation."; 488 | } 489 | enum "killed" { 490 | description 491 | "The session was terminated in abnormal 492 | fashion, e.g., by the NETCONF 493 | protocol operation."; 494 | } 495 | enum "dropped" { 496 | description 497 | "The session was terminated because the transport layer 498 | connection was unexpectedly closed."; 499 | } 500 | enum "timeout" { 501 | description 502 | "The session was terminated because of inactivity, 503 | e.g., waiting for the message or 504 | messages."; 505 | } 506 | 507 | 508 | 509 | 510 | 511 | Bierman Standards Track [Page 9] 512 | 513 | 514 | RFC 6470 NETCONF Base Notifications February 2012 515 | 516 | 517 | enum "bad-hello" { 518 | description 519 | "The client's message was invalid."; 520 | } 521 | enum "other" { 522 | description 523 | "The session was terminated for some other reason."; 524 | } 525 | } 526 | mandatory true; 527 | description 528 | "Reason the session was terminated."; 529 | } 530 | } // notification netconf-session-end 531 | 532 | 533 | notification netconf-confirmed-commit { 534 | description 535 | "Generated when a NETCONF server detects that a 536 | confirmed-commit event has occurred. Indicates the event 537 | and the current state of the confirmed-commit procedure 538 | in progress."; 539 | reference 540 | "RFC 6241, Section 8.4"; 541 | 542 | uses common-session-parms { 543 | when "../confirm-event != 'timeout'"; 544 | } 545 | 546 | leaf confirm-event { 547 | type enumeration { 548 | enum "start" { 549 | description 550 | "The confirmed-commit procedure has started."; 551 | } 552 | enum "cancel" { 553 | description 554 | "The confirmed-commit procedure has been canceled, 555 | e.g., due to the session being terminated, or an 556 | explicit operation."; 557 | } 558 | enum "timeout" { 559 | description 560 | "The confirmed-commit procedure has been canceled 561 | due to the confirm-timeout interval expiring. 562 | The common session parameters will not be present 563 | in this sub-mode."; 564 | } 565 | 566 | 567 | 568 | Bierman Standards Track [Page 10] 569 | 570 | 571 | RFC 6470 NETCONF Base Notifications February 2012 572 | 573 | 574 | enum "extend" { 575 | description 576 | "The confirmed-commit timeout has been extended, 577 | e.g., by a new operation."; 578 | } 579 | enum "complete" { 580 | description 581 | "The confirmed-commit procedure has been completed."; 582 | } 583 | } 584 | mandatory true; 585 | description 586 | "Indicates the event that caused the notification."; 587 | } 588 | 589 | leaf timeout { 590 | when 591 | "../confirm-event = 'start' or ../confirm-event = 'extend'"; 592 | type uint32; 593 | units "seconds"; 594 | description 595 | "The configured timeout value if the event type 596 | is 'start' or 'extend'. This value represents 597 | the approximate number of seconds from the event 598 | time when the 'timeout' event might occur."; 599 | } 600 | } // notification netconf-confirmed-commit 601 | 602 | } 603 | 604 | 605 | 606 | 3. IANA Considerations 607 | 608 | This document registers one XML namespace URN in the 'IETF XML 609 | registry', following the format defined in [RFC3688]. 610 | 611 | URI: urn:ietf:params:xml:ns:yang:ietf-netconf-notifications 612 | 613 | Registrant Contact: The IESG. 614 | 615 | XML: N/A; the requested URI is an XML namespace. 616 | 617 | 618 | 619 | 620 | 621 | 622 | 623 | 624 | 625 | Bierman Standards Track [Page 11] 626 | 627 | 628 | RFC 6470 NETCONF Base Notifications February 2012 629 | 630 | 631 | This document registers one module name in the 'YANG Module Names' 632 | registry, defined in [RFC6020]. 633 | 634 | name: ietf-netconf-notifications 635 | prefix: ncn 636 | namespace: urn:ietf:params:xml:ns:yang:ietf-netconf-notifications 637 | RFC: 6470 638 | 639 | 4. Security Considerations 640 | 641 | The YANG module defined in this memo is designed to be accessed via 642 | the NETCONF protocol [RFC6241]. The lowest NETCONF layer is the 643 | secure transport layer and the mandatory-to-implement secure 644 | transport is SSH, defined in [RFC6242]. 645 | 646 | Some of the readable data nodes in this YANG module may be considered 647 | sensitive or vulnerable in some network environments. It is thus 648 | important to control read access (e.g., via get, get-config, or 649 | notification) to these data nodes. These are the subtrees and data 650 | nodes and their sensitivity/vulnerability: 651 | 652 | /netconf-config-change: 653 | Event type itself indicates that the system configuration has 654 | changed. This event could alert an attacker that specific 655 | configuration data nodes have been altered. 656 | /netconf-config-change/changed-by: 657 | Indicates whether the server or a specific user management session 658 | made the configuration change. Identifies the user name, 659 | session-id, and source host address associated with the 660 | configuration change, if any. 661 | /netconf-config-change/datastore: 662 | Indicates which datastore has been changed. This data can be used 663 | to determine if the non-volatile startup configuration data has 664 | been changed. 665 | /netconf-config-change/edit: 666 | Identifies the specific edit operations and specific datastore 667 | subtree(s) that have changed. This data could be used to 668 | determine if specific server vulnerabilities may now be present. 669 | 670 | 671 | 672 | 673 | 674 | 675 | 676 | 677 | 678 | 679 | 680 | 681 | 682 | Bierman Standards Track [Page 12] 683 | 684 | 685 | RFC 6470 NETCONF Base Notifications February 2012 686 | 687 | 688 | /netconf-capability-change: 689 | Event type itself indicates that the system capabilities have 690 | changed, and may now be vulnerable to unspecified attacks. An 691 | attacker will likely need to understand the content represented by 692 | specific capability URI strings. For example, knowing that a 693 | packet capture monitoring capability has been added to the system 694 | might help an attacker identify the device for possible 695 | unauthorized eavesdropping. 696 | /netconf-capability-change/changed-by: 697 | Indicates whether the server or a specific user management session 698 | made the capability change. Identifies the user name, session-id, 699 | and source host address associated with the capability change, if 700 | any. 701 | /netconf-capability-change/added-capability: 702 | Indicates the specific capability URIs that have been added. This 703 | data could be used to determine if specific server vulnerabilities 704 | may now be present. 705 | /netconf-capability-change/deleted-capability: 706 | Indicates the specific capability URIs that have been deleted. 707 | This data could be used to determine if specific server 708 | vulnerabilities may now be present. 709 | /netconf-capability-change/modified-capability: 710 | Indicates the specific capability URIs that have been modified. 711 | This data could be used to determine if specific server 712 | vulnerabilities may now be present. 713 | 714 | /netconf-session-start: 715 | Event type itself indicates that a NETCONF or other management 716 | session may start altering the device configuration and/or state. 717 | It may be possible for an attacker to alter the configuration by 718 | somehow taking advantage of another session concurrently editing 719 | an unlocked datastore. 720 | /netconf-session-start/username: 721 | Indicates the user name associated with the session. 722 | /netconf-session-start/source-host: 723 | Indicates the source host address associated with the session. 724 | 725 | /netconf-session-end: 726 | Event type itself indicates that a NETCONF or other management 727 | session may be finished altering the device configuration. This 728 | event could alert an attacker that a datastore may have been 729 | altered. 730 | /netconf-session-end/username: 731 | Indicates the user name associated with the session. 732 | /netconf-session-end/source-host: 733 | Indicates the source host address associated with the session. 734 | 735 | 736 | 737 | 738 | 739 | Bierman Standards Track [Page 13] 740 | 741 | 742 | RFC 6470 NETCONF Base Notifications February 2012 743 | 744 | 745 | /netconf-confirmed-commit: 746 | Event type itself indicates that the datastore may have 747 | changed. This event could alert an attacker that the device 748 | behavior has changed. 749 | /netconf-confirmed-commit/username: 750 | Indicates the user name associated with the session. 751 | /netconf-confirmed-commit/source-host: 752 | Indicates the source host address associated with the session. 753 | /netconf-confirmed-commit/confirm-event: 754 | Indicates the specific confirmed-commit state change that 755 | occurred. A value of 'complete' probably indicates that the 756 | datastore has changed. 757 | /netconf-confirmed-commit/timeout: 758 | Indicates the number of seconds in the future when the 759 | datastore may change, due to the server reverting to an older 760 | configuration. 761 | 762 | 5. Acknowledgements 763 | 764 | Thanks to Martin Bjorklund, Juergen Schoenwaelder, Kent Watsen, and 765 | many other members of the NETCONF WG for providing important input to 766 | this document. 767 | 768 | 6. Normative References 769 | 770 | [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate 771 | Requirement Levels", BCP 14, RFC 2119, March 1997. 772 | 773 | [RFC3688] Mealling, M., "The IETF XML Registry", BCP 81, RFC 3688, 774 | January 2004. 775 | 776 | [RFC5277] Chisholm, S. and H. Trevino, "NETCONF Event 777 | Notifications", RFC 5277, July 2008. 778 | 779 | [RFC6020] Bjorklund, M., Ed., "YANG - A Data Modeling Language for 780 | the Network Configuration Protocol (NETCONF)", RFC 6020, 781 | October 2010. 782 | 783 | [RFC6021] Schoenwaelder, J., Ed., "Common YANG Data Types", 784 | RFC 6021, October 2010. 785 | 786 | [RFC6241] Enns, R., Ed., Bjorklund, M., Ed., Schoenwaelder, J., Ed., 787 | and A. Bierman, Ed., "Network Configuration Protocol 788 | (NETCONF)", RFC 6241, June 2011. 789 | 790 | [RFC6242] Wasserman, M., "Using the NETCONF Protocol over Secure 791 | Shell (SSH)", RFC 6242, June 2011. 792 | 793 | 794 | 795 | 796 | Bierman Standards Track [Page 14] 797 | 798 | 799 | RFC 6470 NETCONF Base Notifications February 2012 800 | 801 | 802 | Author's Address 803 | 804 | Andy Bierman 805 | Brocade 806 | 807 | EMail: andy@netconfcentral.org 808 | 809 | 810 | 811 | 812 | 813 | 814 | 815 | 816 | 817 | 818 | 819 | 820 | 821 | 822 | 823 | 824 | 825 | 826 | 827 | 828 | 829 | 830 | 831 | 832 | 833 | 834 | 835 | 836 | 837 | 838 | 839 | 840 | 841 | 842 | 843 | 844 | 845 | 846 | 847 | 848 | 849 | 850 | 851 | 852 | 853 | Bierman Standards Track [Page 15] 854 | 855 | -------------------------------------------------------------------------------- /test/resources/test-file.txt: -------------------------------------------------------------------------------- 1 | This is a test file for xym.py. 2 | 3 | 4 | Test Case 1 5 | ----------- 6 | 7 | The following module SHOULD NOT generate an error. It has the prefix 8 | "example-" and is not inside a CODE BEGINS section. With --strict and 9 | --strict-examples, this is the only module that should be written out. 10 | 11 | module example-no-error { 12 | 13 | } 14 | 15 | 16 | Test Case 2 17 | ----------- 18 | 19 | The following module SHOULD generate an error. It doesn't have the 20 | prefix "example-" and is outside a CODE BEGINS section. 21 | 22 | module ex-error { 23 | 24 | } 25 | 26 | 27 | Test Case 3 28 | ----------- 29 | 30 | The following module SHOULD NOT generate an error. It doesn't have the 31 | prefix "example-" and is inside a CODE BEGINS section. 32 | 33 | file "ex-no-error.yang" 34 | module ex-no-error { 35 | 36 | } 37 | 38 | 39 | 40 | Test Case 4 41 | ----------- 42 | 43 | The following module SHOULD generate an error. It has the prefix 44 | "example-" and is inside a CODE BEGINS section. However, it will still 45 | be output under all cicumstances as it is not actually an example. 46 | 47 | file "example-error.yang" 48 | module example-error { 49 | 50 | } 51 | 52 | 53 | 54 | Test Case 5 55 | ----------- 56 | 57 | The following module SHOULD NOT generate an error. It doesn't have the 58 | prefix "example-" and is inside a CODE BEGINS section. 59 | 60 | file "test-valid.yang" 61 | module test-valid { 62 | 63 | } 64 | 65 | -------------------------------------------------------------------------------- /test/test-file.txt: -------------------------------------------------------------------------------- 1 | This is a test file for xym.py. 2 | 3 | 4 | Test Case 1 5 | ----------- 6 | 7 | The following module SHOULD NOT generate an error. It has the prefix 8 | "example-" and is not inside a CODE BEGINS section. With --strict and 9 | --strict-examples, this is the only module that should be written out. 10 | 11 | module example-no-error { 12 | 13 | } 14 | 15 | 16 | Test Case 2 17 | ----------- 18 | 19 | The following module SHOULD generate an error. It doesn't have the 20 | prefix "example-" and is outside a CODE BEGINS section. 21 | 22 | module ex-error { 23 | 24 | } 25 | 26 | 27 | Test Case 3 28 | ----------- 29 | 30 | The following module SHOULD NOT generate an error. It doesn't have the 31 | prefix "example-" and is inside a CODE BEGINS section. 32 | 33 | file "ex-no-error.yang" 34 | module ex-no-error { 35 | 36 | } 37 | 38 | 39 | 40 | Test Case 4 41 | ----------- 42 | 43 | The following module SHOULD generate an error. It has the prefix 44 | "example-" and is inside a CODE BEGINS section. However, it will still 45 | be output under all cicumstances as it is not actually an example. 46 | 47 | file "example-error.yang" 48 | module example-error { 49 | 50 | } 51 | 52 | 53 | 54 | Test Case 5 55 | ----------- 56 | 57 | The following module SHOULD NOT generate an error. It doesn't have the 58 | prefix "example-" and is inside a CODE BEGINS section. 59 | 60 | file "test-valid.yang" 61 | module test-valid { 62 | 63 | } 64 | 65 | -------------------------------------------------------------------------------- /test/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # 4 | # Let's try and integrate some unittest tests 5 | # 6 | 7 | import glob 8 | import os 9 | import unittest 10 | 11 | from xym import xym 12 | 13 | 14 | class TestCase_base(unittest.TestCase): 15 | def setUp(self): 16 | for y in glob.glob('*.yang'): 17 | os.remove(y) 18 | 19 | def tearDown(self): 20 | for y in glob.glob('*.yang'): 21 | os.remove(y) 22 | 23 | 24 | class TestCase_default(TestCase_base): 25 | def runTest(self): 26 | """Run a test that is the equivalent of: 27 | 28 | xym.py test-file.txt 29 | """ 30 | extracted_modules = xym.xym('resources/test-file.txt', './', './', strict=False, strict_examples=False, 31 | debug_level=0) 32 | self.assertTrue(len(extracted_modules) == 5) 33 | module_check = ['example-no-error.yang', 'ex-error.yang', 'ex-no-error.yang', 'example-error.yang', 34 | 'test-valid.yang'] 35 | for y in module_check: 36 | self.assertTrue(y in extracted_modules) 37 | 38 | 39 | class TestCase_strict(TestCase_base): 40 | def runTest(self): 41 | """Run a test that is the equivalent of: 42 | 43 | xym.py --strict test-file.txt 44 | """ 45 | extracted_modules = xym.xym('resources/test-file.txt', './', './', strict=True, strict_examples=False, 46 | debug_level=0) 47 | self.assertTrue(len(extracted_modules) == 3) 48 | module_check = ['ex-no-error.yang', 'example-error.yang', 'test-valid.yang'] 49 | for y in module_check: 50 | self.assertTrue(y in extracted_modules) 51 | 52 | 53 | class TestCase_strict_examples(TestCase_base): 54 | def runTest(self): 55 | """Run a test that is the equivalent of: 56 | 57 | xym.py --strict --strict-examples test-file.txt 58 | """ 59 | extracted_modules = xym.xym('resources/test-file.txt', './', './', strict=True, strict_examples=True, 60 | debug_level=0) 61 | self.assertTrue(len(extracted_modules) == 1) 62 | module_check = ['example-no-error.yang'] 63 | for y in module_check: 64 | self.assertTrue(y in extracted_modules) 65 | 66 | 67 | class TestCase_codeBegins_noFile(TestCase_base): 68 | def runTest(self): 69 | """Run a test that is the equivalent of: 70 | 71 | xym.py --strict --strict-examples test-file.txt 72 | """ 73 | print('startig test') 74 | extracted_modules = xym.xym('resources/test-file-no-file-after-code-begins', './', './', strict=True, 75 | strict_examples=False, debug_level=0, force_revision_regexp=True) 76 | print(extracted_modules) 77 | self.assertTrue(len(extracted_modules) == 1) 78 | module_check = ['ietf-netconf-partial-lock@2009-10-19.yang'] 79 | for y in module_check: 80 | self.assertTrue(y in extracted_modules) 81 | 82 | 83 | class TestCase_codeBegins_fileWithSymbol(TestCase_base): 84 | def runTest(self): 85 | """Run a test that is the equivalent of: 86 | 87 | xym.py --strict --strict-examples test-file.txt 88 | """ 89 | extracted_modules = xym.xym('resources/test-file-with-symbol', './', './', strict=True, strict_examples=False, 90 | debug_level=0, force_revision_regexp=True) 91 | self.assertTrue(len(extracted_modules) == 1) 92 | module_check = ['ietf-netconf-notifications@2012-02-06.yang'] 93 | for y in module_check: 94 | self.assertTrue(y in extracted_modules) 95 | 96 | 97 | class TestCase_forceRevisionPyang(TestCase_base): 98 | def runTest(self): 99 | """Run a test that is the equivalent of: 100 | 101 | xym.py --force-revision-pyang https://tools.ietf.org/rfc/rfc7223.txt 102 | """ 103 | print('start force_revision_pyang') 104 | extracted_modules = xym.xym("https://tools.ietf.org/rfc/rfc7223.txt", './', './', 105 | debug_level=0, force_revision_pyang=True) 106 | self.assertTrue(len(extracted_modules) == 4) 107 | module_check = ['ietf-interfaces@2014-05-08.yang', 'ex-ethernet.yang', 'ex-ethernet-bonding.yang', 108 | 'ex-vlan.yang'] 109 | for y in module_check: 110 | self.assertTrue(y in extracted_modules) 111 | 112 | 113 | class TestCase_forceRevisionRegexp(TestCase_base): 114 | def runTest(self): 115 | """Run a test that is the equivalent of: 116 | 117 | xym.py --force_revision_regexp https://tools.ietf.org/rfc/rfc7223.txt 118 | """ 119 | print('start force_revision_regexp') 120 | extracted_modules = xym.xym("https://tools.ietf.org/rfc/rfc7223.txt", './', './', 121 | debug_level=0, force_revision_regexp=True) 122 | self.assertTrue(len(extracted_modules) == 4) 123 | module_check = ['ietf-interfaces@2014-05-08.yang', 'ex-ethernet.yang', 'ex-ethernet-bonding.yang', 124 | 'ex-vlan.yang'] 125 | for y in module_check: 126 | self.assertTrue(y in extracted_modules) 127 | 128 | class TestCase_parseonlymodules(TestCase_base): 129 | def runTest(self): 130 | """Run a test that is the equivalent of: 131 | 132 | xym.py --parse-only-modules test-valid example-error.yang source test-file.txt 133 | """ 134 | extracted_modules = xym.xym('resources/test-file.txt', './', './', 135 | strict=False, 136 | strict_examples=False, 137 | debug_level=0, 138 | parse_only_modules=['test-valid', 'example-error.yang']) 139 | self.assertTrue(len(extracted_modules) == 2) 140 | module_check = ['test-valid.yang', 'example-error.yang'] 141 | for y in module_check: 142 | self.assertTrue(y in extracted_modules) 143 | 144 | class TestCase_skipmodules(TestCase_base): 145 | def runTest(self): 146 | """Run a test that is the equivalent of: 147 | 148 | xym.py --skip-modules test-valid ex-no-error.yang source test-file.txt 149 | """ 150 | extracted_modules = xym.xym('resources/test-file.txt', './', './', 151 | strict=False, 152 | strict_examples=False, 153 | debug_level=0, 154 | skip_modules=['test-valid', 'ex-no-error.yang']) 155 | self.assertTrue(len(extracted_modules) == 3) 156 | module_check = ['example-no-error.yang', 'ex-error.yang', 'example-error.yang'] 157 | for y in module_check: 158 | self.assertTrue(y in extracted_modules) 159 | 160 | class TestCase_strict_parseonlymodules(TestCase_base): 161 | def runTest(self): 162 | """Run a test that is the equivalent of: 163 | 164 | xym.py --strict --parse-only-modules example-no-error source test-file.txt 165 | """ 166 | extracted_modules = xym.xym('resources/test-file.txt', './', './', 167 | strict=True, 168 | strict_examples=False, 169 | debug_level=0, 170 | parse_only_modules=['example-no-error']) 171 | self.assertTrue(len(extracted_modules) == 0) 172 | 173 | class TestCase_strict_skipmodules(TestCase_base): 174 | def runTest(self): 175 | """Run a test that is the equivalent of: 176 | 177 | xym.py --strict --skip-modules ex-error ex-no-error example-error test-valid source test-file.txt 178 | """ 179 | extracted_modules = xym.xym('resources/test-file.txt', './', './', 180 | strict=True, 181 | strict_examples=False, 182 | debug_level=0, 183 | skip_modules=['ex-error', 'ex-no-error', 'example-error', 'test-valid']) 184 | self.assertTrue(len(extracted_modules) == 0) 185 | 186 | 187 | class TestCase_strict_examples_parseonlymodules(TestCase_base): 188 | def runTest(self): 189 | """Run a test that is the equivalent of: 190 | 191 | xym.py --strict --strict-examples --parse-only-modules ex-error ex-no-error example-error test-valid \ 192 | source test-file.txt 193 | """ 194 | extracted_modules = xym.xym('resources/test-file.txt', './', './', 195 | strict=True, 196 | strict_examples=True, 197 | debug_level=0, 198 | parse_only_modules=['ex-error', 'test-valid', 'ex-no-error', 'example-error']) 199 | self.assertTrue(len(extracted_modules) == 0) 200 | 201 | 202 | class TestCase_strict_examples_skipmodules(TestCase_base): 203 | def runTest(self): 204 | """Run a test that is the equivalent of: 205 | 206 | xym.py --strict --strict-examples --skip-modules example-no-error source test-file.txt 207 | """ 208 | extracted_modules = xym.xym('resources/test-file.txt', './', './', 209 | strict=True, 210 | strict_examples=True, 211 | debug_level=0, 212 | skip_modules=['example-no-error']) 213 | self.assertTrue(len(extracted_modules) == 0) -------------------------------------------------------------------------------- /xym/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ._version import get_versions 3 | __version__ = get_versions()['version'] 4 | del get_versions 5 | -------------------------------------------------------------------------------- /xym/_version.py: -------------------------------------------------------------------------------- 1 | 2 | # This file helps to compute a version number in source trees obtained from 3 | # git-archive tarball (such as those provided by githubs download-from-tag 4 | # feature). Distribution tarballs (built by setup.py sdist) and build 5 | # directories (produced by setup.py build) will contain a much shorter file 6 | # that just contains the computed version number. 7 | 8 | # This file is released into the public domain. Generated by 9 | # versioneer-0.19 (https://github.com/python-versioneer/python-versioneer) 10 | 11 | """Git implementation of _version.py.""" 12 | 13 | import errno 14 | import os 15 | import re 16 | import subprocess 17 | import sys 18 | 19 | 20 | def get_keywords(): 21 | """Get the keywords needed to look up the version information.""" 22 | # these strings will be replaced by git during git-archive. 23 | # setup.py/versioneer.py will grep for the variable names, so they must 24 | # each be defined on a line of their own. _version.py will just call 25 | # get_keywords(). 26 | git_refnames = " (HEAD -> master)" 27 | git_full = "98a673af9b7f4814a70553b345914c5f299f113c" 28 | git_date = "2024-11-18 09:57:21 +0000" 29 | keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} 30 | return keywords 31 | 32 | 33 | class VersioneerConfig: 34 | """Container for Versioneer configuration parameters.""" 35 | 36 | 37 | def get_config(): 38 | """Create, populate and return the VersioneerConfig() object.""" 39 | # these strings are filled in when 'setup.py versioneer' creates 40 | # _version.py 41 | cfg = VersioneerConfig() 42 | cfg.VCS = "git" 43 | cfg.style = "git-describe" 44 | cfg.tag_prefix = "v" 45 | cfg.parentdir_prefix = "None" 46 | cfg.versionfile_source = "xym/_version.py" 47 | cfg.verbose = False 48 | return cfg 49 | 50 | 51 | class NotThisMethod(Exception): 52 | """Exception raised if a method is not valid for the current scenario.""" 53 | 54 | 55 | LONG_VERSION_PY = {} 56 | HANDLERS = {} 57 | 58 | 59 | def register_vcs_handler(vcs, method): # decorator 60 | """Create decorator to mark a method as the handler of a VCS.""" 61 | def decorate(f): 62 | """Store f in HANDLERS[vcs][method].""" 63 | if vcs not in HANDLERS: 64 | HANDLERS[vcs] = {} 65 | HANDLERS[vcs][method] = f 66 | return f 67 | return decorate 68 | 69 | 70 | def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, 71 | env=None): 72 | """Call the given command(s).""" 73 | assert isinstance(commands, list) 74 | p = None 75 | for c in commands: 76 | try: 77 | dispcmd = str([c] + args) 78 | # remember shell=False, so use git.cmd on windows, not just git 79 | p = subprocess.Popen([c] + args, cwd=cwd, env=env, 80 | stdout=subprocess.PIPE, 81 | stderr=(subprocess.PIPE if hide_stderr 82 | else None)) 83 | break 84 | except EnvironmentError: 85 | e = sys.exc_info()[1] 86 | if e.errno == errno.ENOENT: 87 | continue 88 | if verbose: 89 | print("unable to run %s" % dispcmd) 90 | print(e) 91 | return None, None 92 | else: 93 | if verbose: 94 | print("unable to find command, tried %s" % (commands,)) 95 | return None, None 96 | stdout = p.communicate()[0].strip().decode() 97 | if p.returncode != 0: 98 | if verbose: 99 | print("unable to run %s (error)" % dispcmd) 100 | print("stdout was %s" % stdout) 101 | return None, p.returncode 102 | return stdout, p.returncode 103 | 104 | 105 | def versions_from_parentdir(parentdir_prefix, root, verbose): 106 | """Try to determine the version from the parent directory name. 107 | 108 | Source tarballs conventionally unpack into a directory that includes both 109 | the project name and a version string. We will also support searching up 110 | two directory levels for an appropriately named parent directory 111 | """ 112 | rootdirs = [] 113 | 114 | for i in range(3): 115 | dirname = os.path.basename(root) 116 | if dirname.startswith(parentdir_prefix): 117 | return {"version": dirname[len(parentdir_prefix):], 118 | "full-revisionid": None, 119 | "dirty": False, "error": None, "date": None} 120 | else: 121 | rootdirs.append(root) 122 | root = os.path.dirname(root) # up a level 123 | 124 | if verbose: 125 | print("Tried directories %s but none started with prefix %s" % 126 | (str(rootdirs), parentdir_prefix)) 127 | raise NotThisMethod("rootdir doesn't start with parentdir_prefix") 128 | 129 | 130 | @register_vcs_handler("git", "get_keywords") 131 | def git_get_keywords(versionfile_abs): 132 | """Extract version information from the given file.""" 133 | # the code embedded in _version.py can just fetch the value of these 134 | # keywords. When used from setup.py, we don't want to import _version.py, 135 | # so we do it with a regexp instead. This function is not used from 136 | # _version.py. 137 | keywords = {} 138 | try: 139 | f = open(versionfile_abs, "r") 140 | for line in f.readlines(): 141 | if line.strip().startswith("git_refnames ="): 142 | mo = re.search(r'=\s*"(.*)"', line) 143 | if mo: 144 | keywords["refnames"] = mo.group(1) 145 | if line.strip().startswith("git_full ="): 146 | mo = re.search(r'=\s*"(.*)"', line) 147 | if mo: 148 | keywords["full"] = mo.group(1) 149 | if line.strip().startswith("git_date ="): 150 | mo = re.search(r'=\s*"(.*)"', line) 151 | if mo: 152 | keywords["date"] = mo.group(1) 153 | f.close() 154 | except EnvironmentError: 155 | pass 156 | return keywords 157 | 158 | 159 | @register_vcs_handler("git", "keywords") 160 | def git_versions_from_keywords(keywords, tag_prefix, verbose): 161 | """Get version information from git keywords.""" 162 | if not keywords: 163 | raise NotThisMethod("no keywords at all, weird") 164 | date = keywords.get("date") 165 | if date is not None: 166 | # Use only the last line. Previous lines may contain GPG signature 167 | # information. 168 | date = date.splitlines()[-1] 169 | 170 | # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant 171 | # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 172 | # -like" string, which we must then edit to make compliant), because 173 | # it's been around since git-1.5.3, and it's too difficult to 174 | # discover which version we're using, or to work around using an 175 | # older one. 176 | date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) 177 | refnames = keywords["refnames"].strip() 178 | if refnames.startswith("$Format"): 179 | if verbose: 180 | print("keywords are unexpanded, not using") 181 | raise NotThisMethod("unexpanded keywords, not a git-archive tarball") 182 | refs = set([r.strip() for r in refnames.strip("()").split(",")]) 183 | # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of 184 | # just "foo-1.0". If we see a "tag: " prefix, prefer those. 185 | TAG = "tag: " 186 | tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) 187 | if not tags: 188 | # Either we're using git < 1.8.3, or there really are no tags. We use 189 | # a heuristic: assume all version tags have a digit. The old git %d 190 | # expansion behaves like git log --decorate=short and strips out the 191 | # refs/heads/ and refs/tags/ prefixes that would let us distinguish 192 | # between branches and tags. By ignoring refnames without digits, we 193 | # filter out many common branch names like "release" and 194 | # "stabilization", as well as "HEAD" and "master". 195 | tags = set([r for r in refs if re.search(r'\d', r)]) 196 | if verbose: 197 | print("discarding '%s', no digits" % ",".join(refs - tags)) 198 | if verbose: 199 | print("likely tags: %s" % ",".join(sorted(tags))) 200 | for ref in sorted(tags): 201 | # sorting will prefer e.g. "2.0" over "2.0rc1" 202 | if ref.startswith(tag_prefix): 203 | r = ref[len(tag_prefix):] 204 | if verbose: 205 | print("picking %s" % r) 206 | return {"version": r, 207 | "full-revisionid": keywords["full"].strip(), 208 | "dirty": False, "error": None, 209 | "date": date} 210 | # no suitable tags, so version is "0+unknown", but full hex is still there 211 | if verbose: 212 | print("no suitable tags, using unknown + full revision id") 213 | return {"version": "0+unknown", 214 | "full-revisionid": keywords["full"].strip(), 215 | "dirty": False, "error": "no suitable tags", "date": None} 216 | 217 | 218 | @register_vcs_handler("git", "pieces_from_vcs") 219 | def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): 220 | """Get version from 'git describe' in the root of the source tree. 221 | 222 | This only gets called if the git-archive 'subst' keywords were *not* 223 | expanded, and _version.py hasn't already been rewritten with a short 224 | version string, meaning we're inside a checked out source tree. 225 | """ 226 | GITS = ["git"] 227 | if sys.platform == "win32": 228 | GITS = ["git.cmd", "git.exe"] 229 | 230 | out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, 231 | hide_stderr=True) 232 | if rc != 0: 233 | if verbose: 234 | print("Directory %s not under git control" % root) 235 | raise NotThisMethod("'git rev-parse --git-dir' returned error") 236 | 237 | # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] 238 | # if there isn't one, this yields HEX[-dirty] (no NUM) 239 | describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", 240 | "--always", "--long", 241 | "--match", "%s*" % tag_prefix], 242 | cwd=root) 243 | # --long was added in git-1.5.5 244 | if describe_out is None: 245 | raise NotThisMethod("'git describe' failed") 246 | describe_out = describe_out.strip() 247 | full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) 248 | if full_out is None: 249 | raise NotThisMethod("'git rev-parse' failed") 250 | full_out = full_out.strip() 251 | 252 | pieces = {} 253 | pieces["long"] = full_out 254 | pieces["short"] = full_out[:7] # maybe improved later 255 | pieces["error"] = None 256 | 257 | # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] 258 | # TAG might have hyphens. 259 | git_describe = describe_out 260 | 261 | # look for -dirty suffix 262 | dirty = git_describe.endswith("-dirty") 263 | pieces["dirty"] = dirty 264 | if dirty: 265 | git_describe = git_describe[:git_describe.rindex("-dirty")] 266 | 267 | # now we have TAG-NUM-gHEX or HEX 268 | 269 | if "-" in git_describe: 270 | # TAG-NUM-gHEX 271 | mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) 272 | if not mo: 273 | # unparseable. Maybe git-describe is misbehaving? 274 | pieces["error"] = ("unable to parse git-describe output: '%s'" 275 | % describe_out) 276 | return pieces 277 | 278 | # tag 279 | full_tag = mo.group(1) 280 | if not full_tag.startswith(tag_prefix): 281 | if verbose: 282 | fmt = "tag '%s' doesn't start with prefix '%s'" 283 | print(fmt % (full_tag, tag_prefix)) 284 | pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" 285 | % (full_tag, tag_prefix)) 286 | return pieces 287 | pieces["closest-tag"] = full_tag[len(tag_prefix):] 288 | 289 | # distance: number of commits since tag 290 | pieces["distance"] = int(mo.group(2)) 291 | 292 | # commit: short hex revision ID 293 | pieces["short"] = mo.group(3) 294 | 295 | else: 296 | # HEX: no tags 297 | pieces["closest-tag"] = None 298 | count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], 299 | cwd=root) 300 | pieces["distance"] = int(count_out) # total number of commits 301 | 302 | # commit date: see ISO-8601 comment in git_versions_from_keywords() 303 | date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], 304 | cwd=root)[0].strip() 305 | # Use only the last line. Previous lines may contain GPG signature 306 | # information. 307 | date = date.splitlines()[-1] 308 | pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) 309 | 310 | return pieces 311 | 312 | 313 | def plus_or_dot(pieces): 314 | """Return a + if we don't already have one, else return a .""" 315 | if "+" in pieces.get("closest-tag", ""): 316 | return "." 317 | return "+" 318 | 319 | 320 | def render_pep440(pieces): 321 | """Build up version string, with post-release "local version identifier". 322 | 323 | Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you 324 | get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty 325 | 326 | Exceptions: 327 | 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] 328 | """ 329 | if pieces["closest-tag"]: 330 | rendered = pieces["closest-tag"] 331 | if pieces["distance"] or pieces["dirty"]: 332 | rendered += plus_or_dot(pieces) 333 | rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) 334 | if pieces["dirty"]: 335 | rendered += ".dirty" 336 | else: 337 | # exception #1 338 | rendered = "0+untagged.%d.g%s" % (pieces["distance"], 339 | pieces["short"]) 340 | if pieces["dirty"]: 341 | rendered += ".dirty" 342 | return rendered 343 | 344 | 345 | def render_pep440_pre(pieces): 346 | """TAG[.post0.devDISTANCE] -- No -dirty. 347 | 348 | Exceptions: 349 | 1: no tags. 0.post0.devDISTANCE 350 | """ 351 | if pieces["closest-tag"]: 352 | rendered = pieces["closest-tag"] 353 | if pieces["distance"]: 354 | rendered += ".post0.dev%d" % pieces["distance"] 355 | else: 356 | # exception #1 357 | rendered = "0.post0.dev%d" % pieces["distance"] 358 | return rendered 359 | 360 | 361 | def render_pep440_post(pieces): 362 | """TAG[.postDISTANCE[.dev0]+gHEX] . 363 | 364 | The ".dev0" means dirty. Note that .dev0 sorts backwards 365 | (a dirty tree will appear "older" than the corresponding clean one), 366 | but you shouldn't be releasing software with -dirty anyways. 367 | 368 | Exceptions: 369 | 1: no tags. 0.postDISTANCE[.dev0] 370 | """ 371 | if pieces["closest-tag"]: 372 | rendered = pieces["closest-tag"] 373 | if pieces["distance"] or pieces["dirty"]: 374 | rendered += ".post%d" % pieces["distance"] 375 | if pieces["dirty"]: 376 | rendered += ".dev0" 377 | rendered += plus_or_dot(pieces) 378 | rendered += "g%s" % pieces["short"] 379 | else: 380 | # exception #1 381 | rendered = "0.post%d" % pieces["distance"] 382 | if pieces["dirty"]: 383 | rendered += ".dev0" 384 | rendered += "+g%s" % pieces["short"] 385 | return rendered 386 | 387 | 388 | def render_pep440_old(pieces): 389 | """TAG[.postDISTANCE[.dev0]] . 390 | 391 | The ".dev0" means dirty. 392 | 393 | Exceptions: 394 | 1: no tags. 0.postDISTANCE[.dev0] 395 | """ 396 | if pieces["closest-tag"]: 397 | rendered = pieces["closest-tag"] 398 | if pieces["distance"] or pieces["dirty"]: 399 | rendered += ".post%d" % pieces["distance"] 400 | if pieces["dirty"]: 401 | rendered += ".dev0" 402 | else: 403 | # exception #1 404 | rendered = "0.post%d" % pieces["distance"] 405 | if pieces["dirty"]: 406 | rendered += ".dev0" 407 | return rendered 408 | 409 | 410 | def render_git_describe(pieces): 411 | """TAG[-DISTANCE-gHEX][-dirty]. 412 | 413 | Like 'git describe --tags --dirty --always'. 414 | 415 | Exceptions: 416 | 1: no tags. HEX[-dirty] (note: no 'g' prefix) 417 | """ 418 | if pieces["closest-tag"]: 419 | rendered = pieces["closest-tag"] 420 | if pieces["distance"]: 421 | rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) 422 | else: 423 | # exception #1 424 | rendered = pieces["short"] 425 | if pieces["dirty"]: 426 | rendered += "-dirty" 427 | return rendered 428 | 429 | 430 | def render_git_describe_long(pieces): 431 | """TAG-DISTANCE-gHEX[-dirty]. 432 | 433 | Like 'git describe --tags --dirty --always -long'. 434 | The distance/hash is unconditional. 435 | 436 | Exceptions: 437 | 1: no tags. HEX[-dirty] (note: no 'g' prefix) 438 | """ 439 | if pieces["closest-tag"]: 440 | rendered = pieces["closest-tag"] 441 | rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) 442 | else: 443 | # exception #1 444 | rendered = pieces["short"] 445 | if pieces["dirty"]: 446 | rendered += "-dirty" 447 | return rendered 448 | 449 | 450 | def render(pieces, style): 451 | """Render the given version pieces into the requested style.""" 452 | if pieces["error"]: 453 | return {"version": "unknown", 454 | "full-revisionid": pieces.get("long"), 455 | "dirty": None, 456 | "error": pieces["error"], 457 | "date": None} 458 | 459 | if not style or style == "default": 460 | style = "pep440" # the default 461 | 462 | if style == "pep440": 463 | rendered = render_pep440(pieces) 464 | elif style == "pep440-pre": 465 | rendered = render_pep440_pre(pieces) 466 | elif style == "pep440-post": 467 | rendered = render_pep440_post(pieces) 468 | elif style == "pep440-old": 469 | rendered = render_pep440_old(pieces) 470 | elif style == "git-describe": 471 | rendered = render_git_describe(pieces) 472 | elif style == "git-describe-long": 473 | rendered = render_git_describe_long(pieces) 474 | else: 475 | raise ValueError("unknown style '%s'" % style) 476 | 477 | return {"version": rendered, "full-revisionid": pieces["long"], 478 | "dirty": pieces["dirty"], "error": None, 479 | "date": pieces.get("date")} 480 | 481 | 482 | def get_versions(): 483 | """Get version information or return default if unable to do so.""" 484 | # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have 485 | # __file__, we can work backwards from there to the root. Some 486 | # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which 487 | # case we can only use expanded keywords. 488 | 489 | cfg = get_config() 490 | verbose = cfg.verbose 491 | 492 | try: 493 | return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, 494 | verbose) 495 | except NotThisMethod: 496 | pass 497 | 498 | try: 499 | root = os.path.realpath(__file__) 500 | # versionfile_source is the relative path from the top of the source 501 | # tree (where the .git directory might live) to this file. Invert 502 | # this to find the root from __file__. 503 | for i in cfg.versionfile_source.split('/'): 504 | root = os.path.dirname(root) 505 | except NameError: 506 | return {"version": "0+unknown", "full-revisionid": None, 507 | "dirty": None, 508 | "error": "unable to find root of source tree", 509 | "date": None} 510 | 511 | try: 512 | pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) 513 | return render(pieces, cfg.style) 514 | except NotThisMethod: 515 | pass 516 | 517 | try: 518 | if cfg.parentdir_prefix: 519 | return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) 520 | except NotThisMethod: 521 | pass 522 | 523 | return {"version": "0+unknown", "full-revisionid": None, 524 | "dirty": None, 525 | "error": "unable to compute version", "date": None} 526 | -------------------------------------------------------------------------------- /xym/xym.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function # Must be at the beginning of the file 3 | 4 | import argparse 5 | import io 6 | import os 7 | import os.path 8 | import re 9 | import sys 10 | from lxml import etree as ET 11 | from collections import Counter 12 | 13 | import requests 14 | from pyang import plugin, error 15 | from pyang.plugins.name import emit_name 16 | from requests.packages.urllib3 import disable_warnings 17 | 18 | from .yangParser import create_context 19 | 20 | __author__ = 'jmedved@cisco.com, calle@tail-f.com, bclaise@cisco.com, einarnn@gmail.com' 21 | __copyright__ = "Copyright(c) 2015, 2016, 2017, 2020 Cisco Systems, Inc." 22 | __license__ = "New-style BSD" 23 | __email__ = "einarnn@cisco.com" 24 | 25 | if sys.version_info < (2, 7, 9): 26 | disable_warnings() 27 | 28 | try: 29 | xrange 30 | except Exception: 31 | xrange = range 32 | 33 | 34 | URL_PATTERN = re.compile( 35 | r'^(?:http|ftp)s?://' # http:// or https:// 36 | r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain 37 | r'localhost|' # localhost... 38 | r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip 39 | r'(?::\d+)?' # optional port 40 | r'(?:/?|[/?]\S+)$', 41 | re.IGNORECASE 42 | ) 43 | 44 | 45 | def hexdump(src, length=16, sep='.'): 46 | """ 47 | Hexdump function by sbz and 7h3rAm on Github: 48 | (https://gist.github.com/7h3rAm/5603718). 49 | :param src: Source, the string to be shown in hexadecimal format 50 | :param length: Number of hex characters to print in one row 51 | :param sep: Unprintable characters representation 52 | :return: 53 | """ 54 | filtr = ''.join([(len(repr(chr(x))) == 3) and chr(x) or sep for x in range(256)]) 55 | lines = [] 56 | for c in xrange(0, len(src), length): 57 | chars = src[c:c + length] 58 | hexstring = ' '.join(["%02x" % ord(x) for x in chars]) 59 | if len(hexstring) > 24: 60 | hexstring = "%s %s" % (hexstring[:24], hexstring[24:]) 61 | printable = ''.join(["%s" % ((ord(x) <= 127 and filtr[ord(x)]) or sep) for x in chars]) 62 | lines.append(" %02x: %-*s |%s|\n" % (c, length * 3, hexstring, printable)) 63 | print(''.join(lines)) 64 | 65 | 66 | def finalize_model(input_model): 67 | """ 68 | Extracts string from the model data. This function is always the last 69 | stage in the model post-processing pipeline. 70 | :param input_model: Model to be processed 71 | :return: list of strings, ready to be written to a module file 72 | """ 73 | finalized_output = [] 74 | for mline in input_model: 75 | finalized_output.append(mline[0]) 76 | return finalized_output 77 | 78 | 79 | class YangModuleExtractor: 80 | """ 81 | Extract YANG modules from IETF RFC or draft text string. 82 | """ 83 | MODULE_STATEMENT = re.compile(r'''^[ \t]*(sub)?module +(["'])?([-A-Za-z0-9]*(@[0-9-]*)?)(["'])? *\{.*$''') 84 | PAGE_TAG = re.compile(r'.*\[Page [0-9]*\].*') 85 | CODE_ENDS_TAG = re.compile(r'^[} \t]*.*$') 86 | CODE_BEGINS_TAG = re.compile(r'^[ \t\r\n]*( *file[\s=]+(?:(".*")|(\S*)))?[ \t\r\n]*$') 87 | EXAMPLE_TAG = re.compile(r'^(example-)') 88 | 89 | def __init__(self, src_id, dst_dir, strict=True, strict_examples=True, strict_name=False, add_line_refs=False, 90 | debug_level=0, skip_modules=None, parse_only_modules=None, extract_code_snippets=False, 91 | code_snippets_dir=None): 92 | """ 93 | Initializes class-global variables. 94 | :param src_id: text string containing the draft or RFC text from which YANG 95 | module(s) are to be extracted 96 | :param dst_dir: Directory where to put the extracted YANG module(s) 97 | :param strict: Mode - if 'True', enforce / ; 98 | if 'False', just look for 'module {' and '}' 99 | :param strict_examples: Only output valid examples when in strict mode 100 | :param strict_name: enforce name from module 101 | :param debug_level: If > 0 print some debug statements to the console 102 | :param skip_modules: it will skip modules in this list 103 | :param parse_only_modules: it will parse only modules in this list 104 | :return: 105 | """ 106 | self.src_id = src_id 107 | self.dst_dir = dst_dir 108 | self.strict = strict 109 | self.strict_examples = strict_examples 110 | self.strict_name = strict_name 111 | self.add_line_refs = add_line_refs 112 | self.debug_level = debug_level 113 | self.skip_modules = skip_modules or [] 114 | self.parse_only_modules = parse_only_modules or [] 115 | self.max_line_len = 0 116 | self.extracted_models = [] 117 | self.extract_code_snippets = extract_code_snippets and (not URL_PATTERN.match(src_id) or code_snippets_dir) 118 | self.code_snippets_dir = ( 119 | code_snippets_dir 120 | or os.path.join(self.dst_dir, 'code-snippets', os.path.splitext(self.src_id)[0]) 121 | ) if self.extract_code_snippets else None 122 | self.code_snippets = [] 123 | 124 | def warning(self, s): 125 | """ 126 | Prints out a warning message to stderr. 127 | :param s: The warning string to print 128 | :return: None 129 | """ 130 | print(" WARNING: '%s', %s" % (self.src_id, s), file=sys.stderr) 131 | 132 | def error(self, s): 133 | """ 134 | Prints out an error message to stderr. 135 | :param s: The error string to print 136 | :return: None 137 | """ 138 | print(" ERROR: '%s', %s" % (self.src_id, s), file=sys.stderr) 139 | 140 | def get_mod_rev(self, module): 141 | mname = '' 142 | mrev = '' 143 | bt = '' 144 | 145 | with open(module, 'r') as ym: 146 | for line in ym: 147 | if mname != '' and mrev != '' and bt != '': 148 | return mname + '@' + mrev + ' (belongs-to {})'.format(bt) 149 | 150 | if mname == '': 151 | m = re.search(r'''^\s*(sub)?module\s+['"]?([\w\-\d]+)['"]?''', line) 152 | if m: 153 | mname = m.group(2) 154 | continue 155 | 156 | if mrev == '': 157 | m = re.search(r'''^\s*revision\s+['"]?([\d\-]+)['"]?''', line) 158 | if m: 159 | mrev = m.group(1) 160 | continue 161 | 162 | if bt == '': 163 | m = re.search(r'''^\s*belongs-to\s+['"]?([\w\-\d]+)['"]?''', line) 164 | if m: 165 | bt = m.group(1) 166 | continue 167 | 168 | if mrev is not None and mrev.rstrip() != '': 169 | mrev = '@' + mrev 170 | if bt != '': 171 | return mname + mrev + ' (belongs-to {})'.format(bt) 172 | 173 | return mname + mrev 174 | 175 | def get_extracted_models(self, force_revision_pyang, force_revision_regexp): 176 | if force_revision_pyang or force_revision_regexp: 177 | models = [] 178 | models.extend(self.extracted_models) 179 | for model in models: 180 | if force_revision_pyang: 181 | if isinstance(self.dst_dir, list): 182 | path = ':'.join(self.dst_dir) 183 | else: 184 | path = self.dst_dir 185 | ctx = create_context(path) 186 | ctx.opts.print_revision = True 187 | for p in plugin.plugins: 188 | p.setup_ctx(ctx) 189 | with open(self.dst_dir + '/' + model, 'r', encoding="utf-8") as yang_file: 190 | module = yang_file.read() 191 | m = ctx.add_module(self.dst_dir + '/' + model, module) 192 | if m is None: 193 | m = [] 194 | else: 195 | m = [m] 196 | ctx.validate() 197 | 198 | f = io.StringIO() 199 | emit_name(ctx, m, f) 200 | out = f.getvalue() 201 | 202 | if out.rstrip() == '': 203 | 204 | def __print_pyang_output(ctx): 205 | err = '' 206 | for (epos, etag, eargs) in ctx.errors: 207 | elevel = error.err_level(etag) 208 | if error.is_warning(elevel): 209 | kind = "warning" 210 | else: 211 | kind = "error" 212 | 213 | err += str(epos) + ': %s: ' % kind + \ 214 | error.err_to_str(etag, eargs) + '\n' 215 | return err 216 | 217 | err = __print_pyang_output(ctx) 218 | if err: 219 | self.error('extracting revision from file with pyang ' + self.dst_dir + 220 | '/' + model + ' has following errors:\n' + err) 221 | else: 222 | out = self.get_mod_rev(self.dst_dir + '/' + model) 223 | 224 | real_model_name_revision = out.rstrip() 225 | if real_model_name_revision != '': 226 | if '@' in real_model_name_revision: 227 | real_model_revision = '@' + real_model_name_revision.split('@')[1][0:10] 228 | else: 229 | real_model_revision = '' 230 | real_model_name = real_model_name_revision.split('@')[0] 231 | real_model_name_revision = real_model_name + real_model_revision 232 | if force_revision_regexp: 233 | missing_revision_symbol = '' 234 | else: 235 | missing_revision_symbol = '@' 236 | if real_model_revision == missing_revision_symbol: 237 | self.warning('yang module ' + model.split('@')[0] + ' does not contain a revision statement') 238 | if real_model_name != model.split('@')[0].split('.')[0]: 239 | self.error(model.split('@')[0] + ' model name is wrong') 240 | self.change_model_name(model, real_model_name + '.yang') 241 | 242 | else: 243 | if '@' in model: 244 | existing_model = model.split('@') 245 | existing_model_revision = existing_model[1].split('.')[0] 246 | existing_model_name = existing_model[0] 247 | 248 | switch_items = False 249 | # check for suffix .yang 250 | if '.yang' not in model: 251 | self.error(existing_model_name + ' is missing .yang suffix') 252 | switch_items = True 253 | 254 | # check for model revision if correct 255 | if real_model_revision.strip('@') != existing_model_revision: 256 | self.error(existing_model_name + ' model revision ' + existing_model_revision 257 | + ' is wrong or has incorrect format') 258 | switch_items = True 259 | 260 | # check for model name if correct 261 | if real_model_name != existing_model_name: 262 | self.warning( 263 | 'file name ' + existing_model_name + ' does not match model name ' + real_model_name 264 | ) 265 | switch_items = True 266 | 267 | # if any of above are not correct change file 268 | if switch_items: 269 | self.change_model_name(model, real_model_name_revision + '.yang') 270 | else: 271 | self.warning(real_model_name + ' revision not specified in file name') 272 | self.change_model_name(model, real_model_name_revision + '.yang') 273 | return self.extracted_models 274 | 275 | def change_model_name(self, old_model_name, new_model_name): 276 | self.extracted_models.remove(old_model_name) 277 | self.extracted_models.append(new_model_name) 278 | os.rename(self.dst_dir + '/' + old_model_name, self.dst_dir + '/' + new_model_name) 279 | 280 | def remove_leading_spaces(self, input_model): 281 | """ 282 | This function is a part of the model post-processing pipeline. It 283 | removes leading spaces from an extracted module; depending on the 284 | formatting of the draft/rfc text, may have multiple spaces prepended 285 | to each line. The function also determines the length of the longest 286 | line in the module - this value can be used by later stages of the 287 | model post-processing pipeline. 288 | :param input_model: The YANG model to be processed 289 | :return: YANG model lines with leading spaces removed 290 | """ 291 | leading_spaces = 1024 292 | output_model = [] 293 | for mline in input_model: 294 | line = mline[0] 295 | if line.rstrip(' \r\n') != '': 296 | leading_spaces = min(leading_spaces, len(line) - len(line.lstrip(' '))) 297 | output_model.append([line[leading_spaces:], mline[1]]) 298 | 299 | line_len = len(line[leading_spaces:]) 300 | if line_len > self.max_line_len: 301 | self.max_line_len = line_len 302 | else: 303 | output_model.append(['\n', mline[1]]) 304 | return output_model 305 | 306 | def add_line_references(self, input_model): 307 | """ 308 | This function is a part of the model post-processing pipeline. For 309 | each line in the module, it adds a reference to the line number in 310 | the original draft/RFC from where the module line was extracted. 311 | :param input_model: The YANG model to be processed 312 | :return: Modified YANG model, where line numbers from the RFC/Draft 313 | text file are added as comments at the end of each line in 314 | the modified model 315 | """ 316 | output_model = [] 317 | for ln in input_model: 318 | line_len = len(ln[0]) 319 | line_ref = ('// %4d' % ln[1]).rjust((self.max_line_len - line_len + 7), ' ') 320 | new_line = '%s %s\n' % (ln[0].rstrip(' \r\n\t\f'), line_ref) 321 | output_model.append([new_line, ln[1]]) 322 | return output_model 323 | 324 | def remove_extra_empty_lines(self, input_model): 325 | """ 326 | Removes superfluous newlines from a YANG model that was extracted 327 | from a draft or RFC text. Newlines are removed whenever 2 or more 328 | consecutive empty lines are found in the model. This function is a 329 | part of the model post-processing pipeline. 330 | :param input_model: The YANG model to be processed 331 | :return: YANG model with superfluous newlines removed 332 | """ 333 | ncnt = 0 334 | output_model = [] 335 | for ln in input_model: 336 | if ln[0].strip(' \n\r') == '': 337 | if ncnt == 0: 338 | output_model.append(ln) 339 | elif self.debug_level > 1: 340 | self.debug_print_strip_msg(ln[1] - 1, ln[0]) 341 | ncnt += 1 342 | else: 343 | output_model.append(ln) 344 | ncnt = 0 345 | if self.debug_level > 0: 346 | print(' Removed %d empty lines' % (len(input_model) - len(output_model))) 347 | return output_model 348 | 349 | def post_process_model(self, input_model, add_line_refs): 350 | """ 351 | This function defines the order and execution logic for actions 352 | that are performed in the model post-processing pipeline. 353 | :param input_model: The YANG model to be processed in the pipeline 354 | :param add_line_refs: Flag that controls whether line number 355 | references should be added to the model. 356 | :return: List of strings that constitute the final YANG model to 357 | be written to its module file. 358 | """ 359 | intermediate_model = self.remove_leading_spaces(input_model) 360 | intermediate_model = self.remove_extra_empty_lines(intermediate_model) 361 | if add_line_refs: 362 | intermediate_model = self.add_line_references(intermediate_model) 363 | return finalize_model(intermediate_model) 364 | 365 | def write_model_to_file(self, mdl, fn): 366 | """ 367 | Write a YANG model that was extracted from a source identifier 368 | (URL or source .txt file) to a .yang destination file 369 | :param mdl: YANG model, as a list of lines 370 | :param fn: Name of the YANG model file 371 | :return: 372 | """ 373 | # Write the model to file 374 | output = ''.join(self.post_process_model(mdl, self.add_line_refs)) 375 | if fn: 376 | fqfn = self.dst_dir + '/' + fn 377 | if os.path.isfile(fqfn): 378 | self.error("File '%s' exists" % fqfn) 379 | return 380 | with open(fqfn, 'w') as of: 381 | of.write(output) 382 | of.close() 383 | self.extracted_models.append(fn) 384 | else: 385 | self.error("Output file name can not be determined; YANG file not created") 386 | 387 | def debug_print_line(self, i, level, line): 388 | """ 389 | Debug print of the currently parsed line 390 | :param i: The line number of the line that is being currently parsed 391 | :param level: Parser level 392 | :param line: the line that is currently being parsed 393 | :return: None 394 | """ 395 | if self.debug_level == 2: 396 | print("Line %d (%d): '%s'" % (i + 1, level, line.rstrip(' \r\n\t\f'))) 397 | if self.debug_level > 2: 398 | print("Line %d (%d):" % (i + 1, level)) 399 | hexdump(line) 400 | 401 | def debug_print_strip_msg(self, i, line): 402 | """ 403 | Debug print indicating that an empty line is being skipped 404 | :param i: The line number of the line that is being currently parsed 405 | :param line: the parsed line 406 | :return: None 407 | """ 408 | if self.debug_level == 2: 409 | print(" Stripping Line %d: '%s'" % (i + 1, line.rstrip(' \r\n\t\f'))) 410 | elif self.debug_level > 2: 411 | print(" Stripping Line %d:" % (i + 1)) 412 | hexdump(line) 413 | 414 | def strip_empty_lines_forward(self, lines, i): 415 | """ 416 | Skip over empty lines 417 | :param lines: parsed text 418 | :param i: current parsed line 419 | :return: number of skipped lined 420 | """ 421 | while i < len(lines): 422 | line = lines[i].strip(' \r\n\t\f') 423 | if line != '': 424 | break 425 | self.debug_print_strip_msg(i, lines[i]) 426 | i += 1 # Strip an empty line 427 | return i 428 | 429 | def strip_empty_lines_backward(self, model, max_lines_to_strip): 430 | """ 431 | Strips empty lines preceding the line that is currently being parsed. This 432 | function is called when the parser encounters a Footer. 433 | :param model: lines that were added to the model up to this point 434 | :param line_num: the number of teh line being parsed 435 | :param max_lines_to_strip: max number of lines to strip from the model 436 | :return: None 437 | """ 438 | for l in range(0, max_lines_to_strip): 439 | if model[-1][0].strip(' \r\n\t\f') != '': 440 | return 441 | self.debug_print_strip_msg(model[-1][1] - 1, model[-1][0]) 442 | model.pop() 443 | 444 | def check_edge_cases(self, example_match, in_code): 445 | """ 446 | Checks for edge cases and set level to appropriate value. 447 | 448 | :param example_match: if example is matched in module name 449 | :param in_code: if module in CODE BEGINS section 450 | :return: level value 451 | """ 452 | # skip all "not example" modules in strict-examples mode 453 | if self.strict_examples and not example_match: 454 | if self.parse_only_modules: 455 | self.warning("Unable to parse not example module in strict-example mode") 456 | return 0 457 | # skip all example modules in section in strict-example mode 458 | elif self.strict_examples and example_match and in_code: 459 | if self.parse_only_modules: 460 | self.warning("Unable to parse example module in section in strict-example mode") 461 | return 0 462 | # check if we are not parsing example module in strict mode 463 | elif self.strict and not self.strict_examples and example_match and not in_code: 464 | if self.parse_only_modules: 465 | self.warning("Unable to parse example module in strict mode") 466 | return 0 467 | # skip all modules outside section in strict mode 468 | elif self.strict and not self.strict_examples and not in_code: 469 | return 0 470 | # enable to parse this module 471 | return 1 472 | 473 | def should_parse_module(self, module_name, output_file): 474 | if self.parse_only_modules: 475 | if module_name not in self.parse_only_modules and output_file not in self.parse_only_modules: 476 | return False 477 | elif self.skip_modules: 478 | if module_name in self.skip_modules or output_file in self.skip_modules: 479 | print("\nSkipping '%s'" % module_name) 480 | return False 481 | return True 482 | 483 | def change_output_file_name(self, output_file, module_name): 484 | ret = output_file 485 | if self.strict_name or not output_file: 486 | if output_file: 487 | revision = output_file.split('@')[-1].split('.')[0] 488 | print("\nrewriting filename from '%s' to '%s@%s.yang'" % (output_file, module_name, revision)) 489 | ret = '{}@{}.yang'.format(module_name, revision) 490 | else: 491 | ret = '%s.yang' % module_name 492 | if self.debug_level > 0: 493 | print(' Getting YANG file name from module name: %s' % ret) 494 | return ret 495 | 496 | def extract_yang_model_text(self, content): 497 | """ 498 | Extracts one or more YANG models from an RFC or draft text string in 499 | which the models are specified. The function skips over page 500 | formatting (Page Headers and Footers) and performs basic YANG module 501 | syntax checking. In strict mode, the function also enforces the 502 | / tags - a model is not extracted unless 503 | the tags are present. 504 | :return: None 505 | """ 506 | model = [] 507 | current_code_snippet = [] 508 | in_code_snippet = False 509 | output_file = None 510 | in_code = False 511 | code_section_start = None 512 | i = 0 513 | level = 0 514 | quotes = 0 515 | lines = content.splitlines(True) 516 | while i < len(lines): 517 | line = lines[i] 518 | 519 | # Try to match '' 520 | if self.CODE_ENDS_TAG.match(line): 521 | if in_code is False: 522 | self.warning("Line %d: misplaced " % i) 523 | if level != 0: 524 | self.error('Line %d - encountered while parsing model' % i) 525 | if '}' in line: 526 | last_line_character = line.rfind('}') + 1 527 | last_line_text = line[:last_line_character] 528 | line = last_line_text 529 | level = 0 530 | in_code = False 531 | output_file = None 532 | code_section_start = None 533 | if in_code_snippet: 534 | self.code_snippets.append(current_code_snippet.copy()) 535 | current_code_snippet.clear() 536 | in_code_snippet = False 537 | 538 | if level != 0 and "\"" in line: 539 | if line.count("\"") % 2 == 0: 540 | quotes = 0 541 | else: 542 | if quotes == 1: 543 | quotes = 0 544 | else: 545 | quotes = 1 546 | 547 | if in_code_snippet and line.strip(' \r\n\t\f') != '': 548 | if self.PAGE_TAG.match(line): 549 | i += 1 550 | # Strip empty lines between the Footer and the next page Header 551 | i = self.strip_empty_lines_forward(lines, i) 552 | if i < len(lines): 553 | i += 1 # Strip the next page Header 554 | else: 555 | self.error(" - EOF encountered while parsing the code snippet") 556 | return 557 | # Strip empty lines between the page Header and real content on the page 558 | i = self.strip_empty_lines_forward(lines, i) - 1 559 | if i >= len(lines): 560 | self.error(" - EOF encountered while parsing the code snippet") 561 | return 562 | else: 563 | current_code_snippet.append(line) 564 | 565 | # Try to match '(sub)module {' 566 | match = self.MODULE_STATEMENT.match(line) 567 | if not in_code_snippet and match and quotes == 0: 568 | # We're already parsing a module 569 | if level > 0: 570 | self.error("Line %d - 'module' statement within another module" % i) 571 | return 572 | 573 | if in_code and output_file is None: 574 | self.warning('Line %d - Missing file name in ' % code_section_start) 575 | 576 | # Check if we should enforce / 577 | # if we do enforce, we ignore models not enclosed in / 578 | if match.group(2) or match.group(5): 579 | self.warning('Line %d - Module name should not be enclosed in quotes' % i) 580 | 581 | module_name = match.group(3) 582 | 583 | # do the module name checking, etc. 584 | example_match = self.EXAMPLE_TAG.match(module_name) 585 | if in_code and example_match: 586 | self.error("Line %d - YANG module '%s' with and starting with 'example-'" % 587 | (i, module_name)) 588 | elif not in_code and not example_match: 589 | if module_name.startswith('ex-'): 590 | self.warning("Line %d - example YANG module '%s' not starting with 'example-'" % 591 | (i, module_name)) 592 | else: 593 | self.error("Line %d - YANG module '%s' with no and not starting with 'example-'" % 594 | (i, module_name)) 595 | 596 | # check for parse only modules list 597 | if self.should_parse_module(module_name, output_file): 598 | level = self.check_edge_cases(example_match, in_code) 599 | else: 600 | level = 0 601 | 602 | if level == 1: 603 | print("\nExtracting '%s'" % module_name) 604 | if quotes == 0: 605 | output_file = self.change_output_file_name(output_file, module_name.strip('"\'')) 606 | 607 | if level > 0: 608 | self.debug_print_line(i, level, lines[i]) 609 | # Try to match the Footer ('[Page ]') 610 | # If match found, skip over page headers and footers 611 | if self.PAGE_TAG.match(line): 612 | self.strip_empty_lines_backward(model, 3) 613 | self.debug_print_strip_msg(i, lines[i]) 614 | i += 1 # Strip the 615 | # Strip empty lines between the Footer and the next page Header 616 | i = self.strip_empty_lines_forward(lines, i) 617 | if i < len(lines): 618 | self.debug_print_strip_msg(i, lines[i]) 619 | i += 1 # Strip the next page Header 620 | else: 621 | self.error(" - EOF encountered while parsing the model") 622 | return 623 | # Strip empty lines between the page Header and real content on the page 624 | i = self.strip_empty_lines_forward(lines, i) - 1 625 | if i >= len(lines): 626 | self.error(" - EOF encountered while parsing the model") 627 | return 628 | else: 629 | model.append([line, i + 1]) 630 | counter = Counter(line) 631 | if quotes == 0: 632 | if "\"" in line and "}" in line: 633 | if line.index("}") > line.rindex("\"") or line.index("}") < line.index("\""): 634 | level += (counter['{'] - counter['}']) 635 | else: 636 | level += (counter['{'] - counter['}']) 637 | if level == 1: 638 | self.write_model_to_file(model, output_file) 639 | self.max_line_len = 0 640 | model = [] 641 | output_file = None 642 | level = 0 643 | 644 | # Try to match '' 645 | match = self.CODE_BEGINS_TAG.match(line) 646 | if match: 647 | next_line_is_module_declaration = False 648 | code_section_start = i 649 | if in_code: 650 | self.error("Line %d - Misplaced or missing " % i) 651 | if level: 652 | self.error("Line %d - within a model" % i) 653 | return 654 | in_code = True 655 | j = i 656 | # If we matched 'CODE BEGINS', but not the file name, look on following lines for a complete match 657 | while match: 658 | if match.group(2): 659 | output_file = match.group(2).strip('"') 660 | break 661 | if match.group(3) and match.group(3).endswith('.yang'): 662 | # this is our best guess at whether we have the whole file name if it was unquoted 663 | self.warning("Line %d - Unquoted file name in " % i) 664 | output_file = match.group(3) 665 | break 666 | next_line_is_module_declaration = self.MODULE_STATEMENT.match(lines[j + 1]) 667 | if next_line_is_module_declaration: 668 | break 669 | j += 1 670 | if j >= len(lines): 671 | break 672 | line = line.rstrip() + lines[j].lstrip() 673 | match = self.CODE_BEGINS_TAG.match(line) 674 | 675 | if self.extract_code_snippets and not output_file and not next_line_is_module_declaration: 676 | in_code_snippet = True 677 | j = code_section_start 678 | # if we ended up with an actual match, update our line counter; 679 | # otherwise forget the scan for the file name/code snippet 680 | if (match and output_file) or in_code_snippet: 681 | i = j 682 | i += 1 683 | if level > 0: 684 | self.error(" - EOF encountered while parsing the model") 685 | return 686 | if in_code is True: 687 | self.error("Line %d - Missing " % i) 688 | 689 | def write_code_snippets_to_files(self): 690 | if not self.code_snippets: 691 | return 692 | os.makedirs(self.code_snippets_dir, exist_ok=True) 693 | for index, code_snippet in enumerate(self.code_snippets): 694 | filename = str(index) + '.txt' 695 | full_file_path = os.path.join(self.code_snippets_dir, filename) 696 | with open(full_file_path, 'w') as code_snippet_file: 697 | for line in code_snippet: 698 | line = line[line.count(' ', 0, 3):] # removing leading spaces from line 699 | code_snippet_file.write(line) 700 | 701 | def should_parse_xml(self, sourcecode): 702 | # Do some checks to see if we should parse this block of source code. 703 | sctype = sourcecode.get("type") 704 | markers = sourcecode.get("markers") 705 | if not markers: 706 | markers = "false" 707 | if not sctype: 708 | # If the sourcecode block doesn't have any type, then check if there 709 | # is a strict YANG module in it. This may be older XML. 710 | if re.search(r"[<]CODE BEGINS[>] file .+\.yang", sourcecode.text): 711 | return True 712 | 713 | elif sctype.lower() == "yang": 714 | # If the type is explicitly set, and it's "yang", then 715 | # check to see if it will render with strict markers. 716 | if self.strict_examples and markers.lower() == "false": 717 | return True 718 | 719 | if not self.strict_examples and markers.lower() == "true": 720 | return True 721 | 722 | # Else this is not for us. 723 | return False 724 | 725 | def extract_yang_model_xml(self, content): 726 | doc_parser = ET.XMLParser( 727 | resolve_entities=False, recover=True, ns_clean=True, encoding="utf-8" 728 | ) 729 | root = ET.fromstring(content.encode("utf-8"), doc_parser) 730 | for sourcecode in root.iter("sourcecode"): 731 | if not sourcecode.text: 732 | continue 733 | 734 | if not self.should_parse_xml(sourcecode): 735 | continue 736 | 737 | lines = sourcecode.text.splitlines(True) 738 | if "" in sourcecode.text: 739 | # First try and just get the text. 740 | matches = re.search( 741 | r""" file "(([^@]+)@[^"]+)"\n(.+)\n""", 742 | sourcecode.text, 743 | re.M | re.DOTALL, 744 | ) 745 | if matches: 746 | print("\nExtracting '%s'" % matches[2]) 747 | 748 | output_file = self.change_output_file_name(matches[1], matches[2]) 749 | self.write_model_to_file( 750 | [[line, -1] for line in matches[3].splitlines(True)], 751 | output_file, 752 | ) 753 | continue 754 | 755 | # If the regex doesn't match, then attempt to extract the module as text. 756 | self.extract_yang_model_text(sourcecode.text) 757 | continue 758 | output_file = sourcecode.get("name") 759 | match = None 760 | i = 0 761 | for i, line in enumerate(lines): 762 | if match: 763 | break 764 | match = self.MODULE_STATEMENT.match(line) 765 | if match is None: 766 | continue 767 | mstart = i - 1 768 | lines = lines[mstart:] 769 | if not output_file: 770 | self.warning('Missing file name in ') 771 | if match.group(2) or match.group(5): 772 | self.warning('Module name should not be enclosed in quotes') 773 | module_name = match.group(3) 774 | example_match = self.EXAMPLE_TAG.match(module_name) 775 | if module_name.startswith('ex-'): 776 | self.warning("example YANG module '%s' not starting with 'example-'" % module_name) 777 | example_match = True 778 | if sourcecode.get('markers') == 'true' and example_match: 779 | self.error("YANG module '%s' with markers and starting with 'example-'" % module_name) 780 | elif sourcecode.get('markers') in ('false', None) and not example_match: 781 | self.error("YANG module '%s' with no markers and not starting with 'example-'" % module_name) 782 | if not self.should_parse_module(module_name, output_file): 783 | continue 784 | print("\nExtracting '%s'" % module_name) 785 | 786 | output_file = self.change_output_file_name(output_file, module_name) 787 | self.write_model_to_file([[line, -1] for line in lines], output_file) 788 | 789 | 790 | def xym(source_id, srcdir, dstdir, strict=False, strict_name=False, strict_examples=False, debug_level=0, 791 | add_line_refs=False, force_revision_pyang=False, force_revision_regexp=False, skip_modules=None, 792 | parse_only_modules=None, rfcxml=False, extract_code_snippets=False, code_snippets_dir=None): 793 | """ 794 | Extracts YANG model from an IETF RFC or draft text file. 795 | This is the main (external) API entry for the module. 796 | 797 | :param add_line_refs: 798 | :param source_id: identifier (file name or URL) of a draft or RFC file containing 799 | one or more YANG models 800 | :param srcdir: If source_id points to a file, the optional parameter identifies 801 | the directory where the file is located 802 | :param dstdir: Directory where to put the extracted YANG models 803 | :param strict: Strict syntax enforcement 804 | :param strict_name: Strict name enforcement - name resolved from module name and not from the document 805 | after code begins 806 | :param strict_examples: Only output valid examples when in strict mode 807 | :param debug_level: Determines how much debug output is printed to the console 808 | :param force_revision_regexp: Whether it should create a @.yang even on error using regexp 809 | :param force_revision_pyang: Whether it should create a @.yang even on error using pyang 810 | :param skip_modules: it will skip modules in this list 811 | :param parse_only_modules: it will parse only modules in this list 812 | :param rfcxml: Whether the input file is in RFCXMLv3 format 813 | :param extract_code_snippets: If True then code examples from the draft will be extracted as well 814 | :param code_snippets_dir: Directory where to store code snippets 815 | :return: None 816 | """ 817 | 818 | if force_revision_regexp and force_revision_pyang: 819 | print('Can not use both methods for parsing name and revision - using regular expression method only') 820 | force_revision_pyang = False 821 | 822 | rqst_hdrs = {'Accept': 'text/plain', 'Accept-Charset': 'utf-8'} 823 | 824 | ye = YangModuleExtractor(source_id, dstdir, strict, strict_examples, strict_name, add_line_refs, debug_level, 825 | skip_modules, parse_only_modules, extract_code_snippets, code_snippets_dir) 826 | is_url = URL_PATTERN.match(source_id) 827 | if is_url: 828 | r = requests.get(source_id, headers=rqst_hdrs) 829 | if r.status_code == 200: 830 | if sys.version_info >= (3, 4): 831 | content = r.text 832 | else: 833 | content = r.text.encode('utf8') 834 | if rfcxml: 835 | ye.extract_yang_model_xml(content) 836 | else: 837 | ye.extract_yang_model_text(content) 838 | else: 839 | print("Failed to fetch file from URL '%s', error '%d'" % (source_id, r.status_code), file=sys.stderr) 840 | else: 841 | try: 842 | if sys.version_info >= (3, 4): 843 | with open(os.path.join(srcdir, source_id), encoding='latin-1', errors='ignore') as sf: 844 | if rfcxml: 845 | ye.extract_yang_model_xml(sf.read()) 846 | else: 847 | ye.extract_yang_model_text(sf.read()) 848 | else: 849 | with open(os.path.join(srcdir, source_id)) as sf: 850 | if rfcxml: 851 | ye.extract_yang_model_xml(sf.read()) 852 | else: 853 | ye.extract_yang_model_text(sf.read()) 854 | except IOError as ioe: 855 | print(ioe) 856 | if ye.extract_code_snippets: 857 | ye.write_code_snippets_to_files() 858 | return ye.get_extracted_models(force_revision_pyang, force_revision_regexp) 859 | 860 | 861 | if __name__ == "__main__": 862 | """ 863 | Command line utility / test 864 | """ 865 | parser = argparse.ArgumentParser(description="Extracts one or more YANG " 866 | "models from an IETF RFC/draft text file") 867 | parser.add_argument("source", 868 | help="The URL or file name of the RFC/draft text from " 869 | "which to get the model") 870 | parser.add_argument( 871 | "--rfcxml", 872 | action="store_true", 873 | default=False, 874 | help="Parse a file in RFCXMLv3 format", 875 | ) 876 | parser.add_argument("--srcdir", default='.', 877 | help="Optional: directory where to find the source " 878 | "text; default is './'") 879 | parser.add_argument("--dstdir", default='.', 880 | help="Optional: directory where to put the extracted " 881 | "YANG module(s); default is './'") 882 | parser.add_argument("--strict-name", action='store_true', default=False, 883 | help="Optional flag that determines name enforcement; " 884 | "If set to 'True', name will be resolved from module " 885 | "itself and not from name given in the document;" 886 | " default is 'False'") 887 | parser.add_argument("--strict", action='store_true', default=False, 888 | help="Optional flag that determines syntax enforcement; " 889 | "If set to 'True', the / " 890 | "tags are required; default is 'False'") 891 | parser.add_argument("--strict-examples", action='store_true', default=False, 892 | help="Only output valid examples when in strict mode") 893 | parser.add_argument("--debug", type=int, default=0, 894 | help="Optional: debug level - determines the amount of " 895 | "debug information printed to console; default is 0 (no " 896 | "debug info printed). Debug level 2 prints every parsed " 897 | "line from the original Draft/RFC text. Debug level 3 " 898 | "hexdumps every parsed line. ") 899 | parser.add_argument("--add-line-refs", action='store_true', default=False, 900 | help="Optional: if present, comments are added to each " 901 | "line in the extracted YANG module that contain " 902 | "the reference to the line number in the " 903 | "original RFC/Draft text file from which the " 904 | "line was extracted.") 905 | parser.add_argument("--force-revision-pyang", action='store_true', default=False, 906 | help="Optional: if True it will check if file contains correct revision in file name." 907 | "If it doesnt it will automatically add the correct revision to the filename using pyang") 908 | parser.add_argument("--force-revision-regexp", action='store_true', default=False, 909 | help="Optional: if True it will check if file contains correct revision in file name." 910 | "If it doesnt it will automatically add the correct revision to the filename using regular" 911 | " expression") 912 | parser.add_argument("--extract-code-snippets", action="store_true", default=False, 913 | help="Optional: if True all the code snippets from the RFC/draft will be extracted. " 914 | "If the source argument is a URL and this argument is set to True, " 915 | "please be sure that the code-snippets-dir argument is provided, " 916 | "otherwise this value would be overwritten to False.") 917 | parser.add_argument("--code-snippets-dir", type=str, default='', 918 | help="Optional: Directory where to store code snippets extracted from the RFC/draft." 919 | "If this argument isn't provided and the source argument isn't a URL, " 920 | "then it will be set to the dstdir + 'code-snippets' + source(without file extension). " 921 | "If this argument isn't provided and the source argument is a URL, " 922 | "then code snippets wouldn't be extracted") 923 | group = parser.add_mutually_exclusive_group() 924 | group.add_argument( 925 | "--parse-only-modules", nargs='+', 926 | help="Optional: it will parse only modules added in the list in arguments." 927 | ) 928 | group.add_argument( 929 | "--skip-modules", nargs='+', 930 | help="Optional: it will skip modules added in the list in arguments." 931 | ) 932 | 933 | args = parser.parse_args() 934 | 935 | extracted_models = xym(args.source, 936 | args.srcdir, 937 | args.dstdir, 938 | args.strict, 939 | args.strict_name, 940 | args.strict_examples, 941 | args.debug, 942 | args.add_line_refs, 943 | args.force_revision_pyang, 944 | args.force_revision_regexp, 945 | skip_modules=args.skip_modules, 946 | parse_only_modules=args.parse_only_modules, 947 | rfcxml=args.rfcxml, 948 | extract_code_snippets=args.extract_code_snippets, 949 | code_snippets_dir=args.code_snippets_dir, 950 | ) 951 | if len(extracted_models) > 0: 952 | if args.strict: 953 | print("\nCreated the following models that conform to the strict guidelines::") 954 | else: 955 | print("\nCreated the following models::") 956 | for em in extracted_models: 957 | print('%s : %s ' % (em, args.source)) 958 | print('') 959 | else: 960 | print('\nNo models created.\n') 961 | -------------------------------------------------------------------------------- /xym/yangParser.py: -------------------------------------------------------------------------------- 1 | """Utility belt for working with ``pyang`` and ``pyangext``.""" 2 | 3 | __author__ = "Miroslav Kovac" 4 | __copyright__ = "Copyright 2018 Cisco and its affiliates, Copyright The IETF Trust 2019, All Rights Reserved" 5 | __license__ = "Apache License, Version 2.0" 6 | __email__ = "miroslav.kovac@pantheon.tech" 7 | 8 | import codecs 9 | import io 10 | from os.path import isfile 11 | 12 | from pyang.context import Context 13 | from pyang.error import error_codes 14 | from pyang.repository import FileRepository 15 | from pyang.yang_parser import YangParser 16 | 17 | DEFAULT_OPTIONS = { 18 | 'path': [], 19 | 'deviations': [], 20 | 'features': [], 21 | 'format': 'yang', 22 | 'keep_comments': True, 23 | 'no_path_recurse': False, 24 | 'trim_yin': False, 25 | 'yang_canonical': False, 26 | 'yang_remove_unused_imports': False, 27 | # -- errors 28 | 'ignore_error_tags': [], 29 | 'ignore_errors': [], 30 | 'list_errors': True, 31 | 'print_error_code': False, 32 | 'errors': [], 33 | 'warnings': [code for code, desc in error_codes.items() if desc[0] > 4], 34 | 'verbose': True, 35 | } 36 | """Default options for pyang command line""" 37 | 38 | _COPY_OPTIONS = [ 39 | 'canonical', 40 | 'max_line_len', 41 | 'max_identifier_len', 42 | 'trim_yin', 43 | 'lax_xpath_checks', 44 | 'strict', 45 | ] 46 | """copy options to pyang context options""" 47 | 48 | 49 | class objectify(object): # pylint: disable=invalid-name 50 | """Utility for providing object access syntax (.attr) to dicts""" 51 | 52 | def __init__(self, *args, **kwargs): 53 | for entry in args: 54 | self.__dict__.update(entry) 55 | 56 | self.__dict__.update(kwargs) 57 | 58 | def __getattr__(self, _): 59 | return None 60 | 61 | def __setattr__(self, attr, value): 62 | self.__dict__[attr] = value 63 | 64 | 65 | def _parse_features_string(feature_str): 66 | if feature_str.find(':') == -1: 67 | return (feature_str, []) 68 | 69 | [module_name, rest] = feature_str.split(':', 1) 70 | if rest == '': 71 | return (module_name, []) 72 | 73 | features = rest.split(',') 74 | return (module_name, features) 75 | 76 | 77 | def create_context(path='.', *options, **kwargs): 78 | """Generates a pyang context. 79 | 80 | The dict options and keyword arguments are similar to the command 81 | line options for ``pyang``. For ``plugindir`` use env var 82 | ``PYANG_PLUGINPATH``. For ``path`` option use the argument with the 83 | same name, or ``PYANG_MODPATH`` env var. 84 | 85 | Arguments: 86 | path (str): location of YANG modules. 87 | (Join string with ``os.pathsep`` for multiple locations). 88 | Default is the current working dir. 89 | *options: list of dicts, with options to be passed to context. 90 | See bellow. 91 | **kwargs: similar to ``options`` but have a higher precedence. 92 | See bellow. 93 | 94 | Keyword Arguments: 95 | print_error_code (bool): On errors, print the error code instead 96 | of the error message. Default ``False``. 97 | warnings (list): If contains ``error``, treat all warnings 98 | as errors, except any other error code in the list. 99 | If contains ``none``, do not report any warning. 100 | errors (list): Treat each error code container as an error. 101 | ignore_error_tags (list): Ignore error code. 102 | (For a list of error codes see ``pyang --list-errors``). 103 | ignore_errors (bool): Ignore all errors. Default ``False``. 104 | canonical (bool): Validate the module(s) according to the 105 | canonical YANG order. Default ``False``. 106 | yang_canonical (bool): Print YANG statements according to the 107 | canonical order. Default ``False``. 108 | yang_remove_unused_imports (bool): Remove unused import statements 109 | when printing YANG. Default ``False``. 110 | trim_yin (bool): In YIN input modules, trim whitespace 111 | in textual arguments. Default ``False``. 112 | lax_xpath_checks (bool): Lax check of XPath expressions. 113 | Default ``False``. 114 | strict (bool): Force strict YANG compliance. Default ``False``. 115 | max_line_len (int): Maximum line length allowed. Disabled by default. 116 | max_identifier_len (int): Maximum identifier length allowed. 117 | Disabled by default. 118 | features (list): Features to support, default all. 119 | Format ``:[,]*``. 120 | keep_comments (bool): Do not discard comments. Default ``True``. 121 | no_path_recurse (bool): Do not recurse into directories 122 | in the yang path. Default ``False``. 123 | 124 | Returns: 125 | pyang.Context: Context object for ``pyang`` usage 126 | """ 127 | # deviations (list): Deviation module (NOT CURRENTLY WORKING). 128 | 129 | opts = objectify(DEFAULT_OPTIONS, *options, **kwargs) 130 | repo = FileRepository(path, no_path_recurse=opts.no_path_recurse) 131 | 132 | ctx = Context(repo) 133 | ctx.opts = opts 134 | 135 | for attr in _COPY_OPTIONS: 136 | setattr(ctx, attr, getattr(opts, attr)) 137 | 138 | # make a map of features to support, per module (taken from pyang bin) 139 | for feature_name in opts.features: 140 | (module_name, features) = _parse_features_string(feature_name) 141 | ctx.features[module_name] = features 142 | 143 | # apply deviations (taken from pyang bin) 144 | for file_name in opts.deviations: 145 | with io.open(file_name, "r", encoding="utf-8") as fd: 146 | module = ctx.add_module(file_name, fd.read()) 147 | if module is not None: 148 | ctx.deviation_modules.append(module) 149 | 150 | return ctx 151 | 152 | 153 | def parse(text, ctx = None): 154 | """Parse a YANG statement into an Abstract Syntax subtree. 155 | 156 | Arguments: 157 | text (str): file name for a YANG module or text 158 | ctx (optional pyang.Context): context used to validate text 159 | 160 | Returns: 161 | pyang.statements.Statement: Abstract syntax subtree 162 | 163 | Note: 164 | The ``parse`` function can be used to parse small amounts of text. 165 | If yout plan to parse an entire YANG (sub)module, please use instead:: 166 | 167 | ast = ctx.add_module(module_name, text_contet) 168 | 169 | It is also well known that ``parse`` function cannot solve 170 | YANG deviations yet. 171 | """ 172 | parser = YangParser() # Similar names, but, this one is from PYANG library 173 | 174 | filename = 'parser-input' 175 | 176 | ctx_ = ctx or create_context() 177 | 178 | if isfile(text): 179 | filename = text 180 | text = codecs.open(filename, encoding="utf-8").read() 181 | 182 | # ensure reported errors are just from parsing 183 | # old_errors = ctx_.errors 184 | ctx_.errors = [] 185 | 186 | ast = parser.parse(ctx_, filename, text) 187 | 188 | return ast 189 | --------------------------------------------------------------------------------