├── .github ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── build.yml │ ├── deploy.yml │ └── pr-tests.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── _config.yml ├── crosspm ├── __init__.py ├── __main__.py ├── adapters │ ├── __init__.py │ ├── artifactoryaql.py │ ├── common.py │ └── files.py ├── cmakepm.cmake ├── config.py ├── cpm.py ├── crosspm.cmake ├── helpers │ ├── __init__.py │ ├── archive.py │ ├── cache.py │ ├── config.py │ ├── content.py │ ├── downloader.py │ ├── exceptions.py │ ├── locker.py │ ├── output.py │ ├── package.py │ ├── parser.py │ ├── promoter.py │ ├── python.py │ ├── source.py │ └── usedby.py └── template │ ├── __init__.py │ ├── all_yaml.j2 │ └── gus.j2 ├── docs ├── FAQ.md ├── README.md ├── config │ ├── CONFIG.md │ ├── Environment-variables.md │ ├── IMPORT.md │ ├── OUTPUT.md │ └── output-template.md ├── cpmconfig.md └── usage │ ├── USAGE-CMAKE.md │ ├── USAGE-CREDS.md │ ├── USAGE-PYTHON.md │ ├── USAGE-USEDBY.md │ └── USAGE.md ├── gh-md-toc ├── requirements-ci.txt ├── requirements-dev.txt ├── requirements-test.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── conftest.py ├── data │ ├── config_deps.yaml │ ├── config_deps_depslock.yaml │ ├── config_depslock.yaml │ ├── config_en_cp1251.yaml │ ├── config_en_utf8_bom.yaml │ ├── config_import_with_comment.yaml │ ├── config_import_without_comment.yaml │ ├── config_recursive_off.yaml │ ├── config_recursive_on.yaml │ ├── config_ru_utf8_bom.yaml │ ├── config_stub.yaml │ └── test_cred.yaml ├── test_artifactoryaql.py ├── test_config.py ├── test_cpm.py ├── test_helpers_python.py ├── test_output.py ├── test_package.py └── test_parser.py └── toc.sh /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Todos 2 | - [ ] Documentation 3 | - [ ] Code Review, all remarks corrected 4 | - [ ] Unit (preferred) or internal `crosspm-integration-tests` added 5 | - [ ] Internal `crosspm-integration-tests` passed on dev-version `crosspm` 6 | - [ ] Internal `cpmconfig` passed on dev-version `crosspm` 7 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | py_version: 7 | description: 'Python version' 8 | default: '3.8' 9 | required: false 10 | type: string 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v4 17 | with: 18 | fetch-depth: 1 19 | - uses: actions/setup-python@v5 20 | with: 21 | python-version: ${{ inputs.py_version }} 22 | - name: Install dependencies 23 | run: | 24 | python -m pip install --upgrade pip 25 | python -m pip install build wheel 26 | python -m pip install -r requirements.txt 27 | - name: Build package 28 | run: python setup.py sdist bdist_wheel 29 | - name: Store dist for 2w 30 | uses: actions/upload-artifact@v4 31 | with: 32 | name: CrossPM packages 33 | path: dist/ 34 | if-no-files-found: error 35 | retention-days: 14 36 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Deploy 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build-packages: 10 | uses: ./.github/workflows/build.yml 11 | # Specify python version with this argument (same level as "uses"): 12 | # with: 13 | # py_version: '3.7' 14 | deploy: 15 | runs-on: ubuntu-latest 16 | needs: build-packages 17 | steps: 18 | - name: Download Artifacts 19 | uses: actions/download-artifact@v4 20 | with: 21 | name: CrossPM packages 22 | path: dist/ 23 | - name: Publish package 24 | uses: pypa/gh-action-pypi-publish@release/v1 25 | with: 26 | user: __token__ 27 | password: ${{ secrets.PYPI_API_TOKEN }} 28 | -------------------------------------------------------------------------------- /.github/workflows/pr-tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches-ignore: 6 | - master 7 | # Pull Requests can not reuse workflows from the source branch: 8 | # https://github.com/actions/toolkit/issues/932 9 | pull_request: 10 | branches: 11 | - '*' 12 | 13 | jobs: 14 | tests: 15 | runs-on: ubuntu-latest 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | python-version: 20 | - '3.8' 21 | - '3.9' 22 | - '3.10' 23 | - '3.11' 24 | 25 | steps: 26 | - uses: actions/checkout@v4 27 | with: 28 | fetch-depth: 1 29 | - uses: actions/setup-python@v5 30 | with: 31 | python-version: ${{ matrix.python-version }} 32 | - name: Install dependencies 33 | run: | 34 | python -m pip install --upgrade pip wheel 35 | python -m pip install -r requirements-ci.txt 36 | - name: Run Tests on Python ${{ matrix.python-version }} 37 | run: | 38 | python -mflake8 crosspm 39 | python -mcoverage run -m pytest tests 40 | build-artifact: 41 | uses: ./.github/workflows/build.yml 42 | needs: tests 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .idea/ 3 | build/ 4 | crosspm.egg-info/ 5 | dist/ 6 | tests/pt/ 7 | docs/_build/ 8 | /tests_old/ 9 | .cache 10 | .eggs 11 | .tests 12 | README.rst 13 | /htmlcov 14 | .coverage 15 | coverage.xml 16 | test.py 17 | token.txt 18 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: ^tests/data 2 | repos: 3 | - repo: https://github.com/pre-commit/pre-commit-hooks 4 | rev: v2.3.0 5 | hooks: 6 | - id: check-yaml 7 | - id: end-of-file-fixer 8 | - id: trailing-whitespace 9 | 10 | - repo: https://github.com/PyCQA/flake8 11 | rev: 4.0.1 12 | hooks: 13 | - id: flake8 14 | exclude: ^tests/ 15 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ##### 1.0.10 2 | - Support /*/ in path 3 | - old artifactory adapter deprecated, now it is `artifactory-aql` 4 | 5 | ##### 1.0.9 6 | - Output order like order in dep-file 7 | 8 | ##### 1.0.8 9 | - Added **prefer-local** flag 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016-2017 DevOpsHQ http://devopshq.github.io/crosspm 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | CrossPM 2 | ======= 3 | 4 | [![Deploy](https://github.com/devopshq/crosspm/actions/workflows/deploy.yml/badge.svg?branch=master)](https://github.com/devopshq/crosspm/actions/workflows/deploy.yml) 5 | [![codacy](https://api.codacy.com/project/badge/Grade/7a9ed2e6bb3e445f9e4a776e9b7f7886)](https://www.codacy.com/app/devopshq/crosspm/dashboard) 6 | [![pypi](https://img.shields.io/pypi/v/crosspm.svg)](https://pypi.python.org/pypi/crosspm) 7 | [![license](https://img.shields.io/pypi/l/crosspm.svg)](https://github.com/devopshq/crosspm/blob/master/LICENSE) 8 | 9 | Documentation 10 | ------------- 11 | Actual version always here: http://devopshq.github.io/crosspm 12 | 13 | Introduction 14 | ------------ 15 | 16 | CrossPM (Cross Package Manager) is a universal extensible package manager. 17 | It lets you download and as a next step - manage packages of different types from different repositories. 18 | 19 | Out-of-the-box modules: 20 | 21 | - Adapters 22 | - Artifactory 23 | - [Artifactory-AQL](https://www.jfrog.com/confluence/display/RTF/Artifactory+Query+Language) (supported since artifactory 3.5.0): 24 | - files (simple repository on your local filesystem) 25 | 26 | - Package file formats 27 | - zip 28 | - tar.gz 29 | - nupkg (treats like simple zip archive for now) 30 | 31 | Modules planned to implement: 32 | 33 | - Adapters 34 | - git 35 | - smb 36 | - sftp/ftp 37 | 38 | - Package file formats 39 | - nupkg (nupkg dependencies support) 40 | - 7z 41 | 42 | We also need your feedback to let us know which repositories and package formats do you need, 43 | so we could plan its implementation. 44 | 45 | The biggest feature of CrossPM is flexibility. It is fully customizable, i.e. repository structure, package formats, 46 | packages version templates, etc. 47 | 48 | To handle all the power it have, you need to write configuration file (**crosspm.yaml**) 49 | and manifest file with the list of packages you need to download. 50 | 51 | Configuration file format is YAML, as you could see from its filename, so you free to use yaml hints and tricks, 52 | as long, as main configuration parameters remains on their levels :) 53 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-cayman 2 | -------------------------------------------------------------------------------- /crosspm/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from crosspm.config import __version__ 3 | 4 | version = str(__version__).split('.') 5 | if len(version) < 4: 6 | version = __version__ 7 | else: 8 | if version[3].lower().startswith('dev'): 9 | version = __version__ 10 | else: 11 | version = '{} build {}'.format('.'.join(version[:3]), version[3]) 12 | 13 | # import not in top, because use version 14 | from crosspm.cpm import CrossPM # noqa 15 | -------------------------------------------------------------------------------- /crosspm/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from crosspm.cpm import CrossPM 4 | 5 | 6 | def main(): 7 | app = CrossPM() 8 | app.run() 9 | 10 | 11 | if __name__ == '__main__': 12 | main() 13 | -------------------------------------------------------------------------------- /crosspm/adapters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devopshq/crosspm/f081203edc5c4df99b0f44b7e79ed2da31aa630a/crosspm/adapters/__init__.py -------------------------------------------------------------------------------- /crosspm/adapters/common.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import logging 3 | from crosspm.helpers.config import Config 4 | 5 | 6 | class BaseAdapter: 7 | def __init__(self, config: Config): 8 | self._config = config 9 | self._log = logging.getLogger('crosspm') 10 | 11 | def get_packages(self, source, parser, downloader, list_or_file_path, property_validate=True): 12 | # Here must be function 13 | self._log.info('Get packages') 14 | return None 15 | 16 | def download_package(self, package, dest_path): 17 | # Here must be function 18 | self._log.info('Download package') 19 | return None, False 20 | -------------------------------------------------------------------------------- /crosspm/adapters/files.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import json 3 | import shutil 4 | import pathlib 5 | from crosspm.adapters.common import BaseAdapter 6 | from crosspm.helpers.exceptions import * # noqa 7 | from crosspm.helpers.package import Package 8 | from crosspm.helpers.config import CROSSPM_DEPENDENCY_LOCK_FILENAME 9 | import os 10 | 11 | CHUNK_SIZE = 1024 12 | 13 | setup = { 14 | "name": [ 15 | "files", 16 | ], 17 | } 18 | 19 | 20 | # class PureFilesPath(pathlib.PurePath): 21 | # """ 22 | # A class to work with Files paths that doesn't connect 23 | # to any server. I.e. it supports only basic path 24 | # operations. 25 | # """ 26 | # _flavour = pathlib._posix_flavour 27 | # __slots__ = () 28 | # 29 | 30 | class FilesPath(pathlib.Path): # , PureFilesPath): 31 | def glob(self, pattern): 32 | return super(FilesPath, self).glob(pattern) 33 | 34 | def rglob(self, pattern): 35 | return super(FilesPath, self).rglob(pattern) 36 | 37 | def __new__(cls, *args, **kwargs): 38 | cls = WindowsPath if os.name == 'nt' else PosixPath 39 | obj = pathlib.Path.__new__(cls, *args, **kwargs) 40 | return obj 41 | 42 | @property 43 | def properties(self): 44 | """ 45 | Fetch artifact properties 46 | """ 47 | return {} 48 | 49 | @properties.setter 50 | def properties(self, properties): 51 | self.del_properties(self.properties, recursive=False) 52 | self.set_properties(properties, recursive=False) 53 | 54 | @properties.deleter 55 | def properties(self): 56 | self.del_properties(self.properties, recursive=False) 57 | 58 | def set_properties(self, properties, recursive=True): 59 | """ 60 | Adds new or modifies existing properties listed in properties 61 | 62 | properties - is a dict which contains the property names and values to set. 63 | Property values can be a list or tuple to set multiple values 64 | for a key. 65 | recursive - on folders property attachment is recursive by default. It is 66 | possible to force recursive behavior. 67 | """ 68 | if not properties: 69 | return False 70 | 71 | return True # self._accessor.set_properties(self, properties, recursive) 72 | 73 | def del_properties(self, properties, recursive=None): 74 | """ 75 | Delete properties listed in properties 76 | 77 | properties - iterable contains the property names to delete. If it is an 78 | str it will be casted to tuple. 79 | recursive - on folders property attachment is recursive by default. It is 80 | possible to force recursive behavior. 81 | """ 82 | return True # self._accessor.del_properties(self, properties, recursive) 83 | 84 | 85 | class PosixPath(FilesPath, pathlib.PurePosixPath): 86 | __slots__ = () 87 | 88 | 89 | class WindowsPath(FilesPath, pathlib.PureWindowsPath): 90 | __slots__ = () 91 | 92 | def owner(self): 93 | raise NotImplementedError("Path.owner() is unsupported on this system") 94 | 95 | def group(self): 96 | raise NotImplementedError("Path.group() is unsupported on this system") 97 | 98 | 99 | class Adapter(BaseAdapter): 100 | def get_packages(self, source, parser, downloader, list_or_file_path): 101 | _pkg_name_col = self._config.name_column 102 | _packages_found = {} 103 | _pkg_name_old = "" 104 | for _paths in parser.get_paths(list_or_file_path, source): 105 | _packages = [] 106 | _params_found = {} 107 | _params_found_raw = {} 108 | last_error = '' 109 | _pkg_name = _paths['params'][_pkg_name_col] 110 | if _pkg_name != _pkg_name_old: 111 | _pkg_name_old = _pkg_name 112 | self._log.info( 113 | '{}: {}'.format(_pkg_name, 114 | {k: v for k, v in _paths['params'].items() if 115 | k not in (_pkg_name_col, 'repo')})) 116 | for _sub_paths in _paths['paths']: 117 | self._log.info('repo: {}'.format(_sub_paths['repo'])) 118 | for _path in _sub_paths['paths']: 119 | _tmp_params = dict(_paths['params']) 120 | _tmp_params['repo'] = _sub_paths['repo'] 121 | 122 | _path_fixed, _path_pattern = parser.split_fixed_pattern(_path) 123 | _repo_paths = FilesPath(_path_fixed) 124 | try: 125 | for _repo_path in _repo_paths.glob(_path_pattern): 126 | _mark = 'found' 127 | _matched, _params, _params_raw = parser.validate_path(str(_repo_path), _tmp_params) 128 | if _matched: 129 | _params_found[_repo_path] = {k: v for k, v in _params.items()} 130 | _params_found_raw[_repo_path] = {k: v for k, v in _params_raw.items()} 131 | _mark = 'match' 132 | _valid, _params = parser.validate(_repo_path.properties, 'properties', _tmp_params, 133 | return_params=True) 134 | if _valid: 135 | _mark = 'valid' 136 | _packages += [_repo_path] 137 | _params_found[_repo_path].update({k: v for k, v in _params.items()}) 138 | _params_found[_repo_path]['filename'] = str(_repo_path.name) 139 | self._log.debug(' {}: {}'.format(_mark, str(_repo_path))) 140 | except RuntimeError as e: 141 | try: 142 | err = json.loads(e.args[0]) 143 | except Exception: 144 | err = {} 145 | if isinstance(err, dict): 146 | # TODO: Check errors 147 | # e.args[0] = '''{ 148 | # "errors" : [ { 149 | # "status" : 404, 150 | # "message" : "Not Found" 151 | # } ] 152 | # }''' 153 | for error in err.get('errors', []): 154 | err_status = error.get('status', -1) 155 | err_msg = error.get('message', '') 156 | if err_status == 401: 157 | msg = 'Authentication error[{}]{}'.format(err_status, 158 | (': {}'.format( 159 | err_msg)) if err_msg else '') 160 | elif err_status == 404: 161 | msg = last_error 162 | else: 163 | msg = 'Error[{}]{}'.format(err_status, 164 | (': {}'.format(err_msg)) if err_msg else '') 165 | if last_error != msg: 166 | self._log.error(msg) 167 | last_error = msg 168 | 169 | _package = None 170 | if _packages: 171 | _packages = parser.filter_one(_packages, _paths['params'], _params_found) 172 | if isinstance(_packages, dict): 173 | _packages = [_packages] 174 | 175 | if len(_packages) == 1: 176 | _stat_pkg = self.pkg_stat(_packages[0]['path']) 177 | 178 | _params_raw = _params_found_raw.get(_packages[0]['path'], {}) 179 | _params_tmp = _params_found.get(_packages[0]['path'], {}) 180 | _params_tmp.update({k: v for k, v in _packages[0]['params'].items() if k not in _params_tmp}) 181 | _package = Package(_pkg_name, _packages[0]['path'], _paths['params'], downloader, self, parser, 182 | _params_tmp, _params_raw, _stat_pkg) 183 | _mark = 'chosen' 184 | self._log.info(' {}: {}'.format(_mark, str(_packages[0]['path']))) 185 | 186 | elif len(_packages) > 1: 187 | raise CrosspmException( 188 | CROSSPM_ERRORCODE_MULTIPLE_DEPS, 189 | 'Multiple instances found for package [{}] not found.'.format(_pkg_name) 190 | ) 191 | else: 192 | # Package not found: may be error, but it could be in other source. 193 | pass 194 | else: 195 | # Package not found: may be error, but it could be in other source. 196 | pass 197 | 198 | if (_package is not None) or (not self._config.no_fails): 199 | _added, _package = downloader.add_package(_pkg_name, _package) 200 | else: 201 | _added = False 202 | if _package is not None: 203 | _pkg_name = _package.name 204 | if _added or (_package is not None): 205 | if (_package is not None) or (not self._config.no_fails): 206 | if (_package is not None) or (_packages_found.get(_pkg_name, None) is None): 207 | _packages_found[_pkg_name] = _package 208 | 209 | if _added and (_package is not None): 210 | if downloader.do_load: 211 | _package.download() 212 | _deps_file = _package.get_file(self._config.deps_lock_file_name) 213 | if not _deps_file: 214 | _deps_file = _package.get_file(CROSSPM_DEPENDENCY_LOCK_FILENAME) 215 | if _deps_file: 216 | _package.find_dependencies(_deps_file) 217 | elif self._config.deps_file_name: 218 | _deps_file = _package.get_file(self._config.deps_file_name) 219 | if _deps_file and os.path.isfile(_deps_file): 220 | _package.find_dependencies(_deps_file) 221 | 222 | return _packages_found 223 | 224 | @staticmethod 225 | def pkg_stat(package): 226 | _stat_attr = {'ctime': 'st_atime', 227 | 'mtime': 'st_mtime', 228 | 'size': 'st_size'} 229 | _stat_pkg = os.stat(str(package)) 230 | _stat_pkg = {k: getattr(_stat_pkg, v, None) for k, v in _stat_attr.items()} 231 | return _stat_pkg 232 | 233 | def download_package(self, package, dest_path): 234 | dest_dir = os.path.dirname(dest_path) 235 | 236 | if not os.path.exists(dest_dir): 237 | os.makedirs(dest_dir) 238 | elif os.path.exists(dest_path): 239 | os.remove(dest_path) 240 | 241 | try: 242 | _stat_pkg = self.pkg_stat(package) 243 | shutil.copyfile(str(package), dest_path) 244 | os.utime(dest_path, (_stat_pkg['ctime'], _stat_pkg['mtime'])) 245 | 246 | except Exception as e: 247 | code = CROSSPM_ERRORCODE_SERVER_CONNECT_ERROR 248 | msg = 'FAILED to download package {} at url: [{}]'.format( 249 | package.name, 250 | str(package), 251 | ) 252 | raise CrosspmException(code, msg) from e 253 | 254 | return dest_path 255 | 256 | @staticmethod 257 | def get_package_filename(package): 258 | if isinstance(package, FilesPath): 259 | return package.name 260 | return '' 261 | 262 | @staticmethod 263 | def get_package_path(package): 264 | if isinstance(package, FilesPath): 265 | return str(package) 266 | return '' 267 | -------------------------------------------------------------------------------- /crosspm/cmakepm.cmake: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.11) 2 | 3 | include(ExternalProject) 4 | 5 | set(Python_ADDITIONAL_VERSIONS 3.0 3.1 3.2 3.3 3.4 3.5) 6 | 7 | find_package(PythonInterp) 8 | 9 | 10 | set(CPM_DIR ${CMAKE_CURRENT_LIST_DIR}) 11 | 12 | 13 | function(pm_download_dependencies) 14 | 15 | _pm_download_dependencies( ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR} ) 16 | 17 | endfunction() 18 | 19 | 20 | function(pm_download_current_dependencies) 21 | 22 | _pm_download_dependencies( ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ) 23 | 24 | endfunction() 25 | 26 | 27 | function(_pm_download_dependencies SOURCE_DIR BINARY_DIR) 28 | 29 | if(NOT DEFINED CPM_TARGET_OS) 30 | if("x$ENV{TargetOS}" STREQUAL "x") 31 | message(FATAL_ERROR "Env var TargetOS not set!") 32 | else() 33 | set(CPM_TARGET_OS $ENV{TargetOS} CACHE STRING "CMAKE Package Manager variable" FORCE) 34 | endif() 35 | endif() 36 | 37 | if(NOT DEFINED CPM_TARGET_ARCH) 38 | if("x$ENV{TargetArchitecture}" STREQUAL "x") 39 | message(FATAL_ERROR "Env var TargetArchitecture not set!") 40 | else() 41 | set(CPM_TARGET_ARCH $ENV{TargetArchitecture} CACHE STRING "CMAKE Package Manager variable" FORCE) 42 | endif() 43 | endif() 44 | 45 | if(NOT DEFINED CPM_PLATFORM_TOOLSET) 46 | if("x$ENV{PlatformToolset}" STREQUAL "x") 47 | message(FATAL_ERROR "Env var PlatformToolset not set!") 48 | else() 49 | set(CPM_PLATFORM_TOOLSET $ENV{PlatformToolset} CACHE STRING "CMAKE Package Manager variable" FORCE) 50 | endif() 51 | endif() 52 | 53 | execute_process( 54 | COMMAND 55 | "${PYTHON_EXECUTABLE}" 56 | "-u" 57 | "-m" "crosspm" 58 | "download" 59 | "-o" 60 | "os=${CPM_TARGET_OS},arch=${CPM_TARGET_ARCH},cl=${CPM_PLATFORM_TOOLSET}" 61 | WORKING_DIRECTORY 62 | "${SOURCE_DIR}" 63 | OUTPUT_FILE 64 | "${BINARY_DIR}/tmp_packages.deps" 65 | RESULT_VARIABLE 66 | RESULT_CODE_VALUE 67 | ) 68 | 69 | if(NOT "${RESULT_CODE_VALUE}" STREQUAL "0") 70 | message(FATAL_ERROR "process failed RESULT_CODE='${RESULT_CODE_VALUE}'") 71 | endif() 72 | 73 | # read file to variable 74 | file(STRINGS "${BINARY_DIR}/tmp_packages.deps" FILE_CONTENT) 75 | 76 | # count lines 77 | list(LENGTH FILE_CONTENT FILE_CONTENT_LENGTH) 78 | 79 | # calc: max_i = n - 1 80 | math(EXPR FILE_CONTENT_N "${FILE_CONTENT_LENGTH} - 1") 81 | 82 | 83 | # iterate over lines 84 | foreach(FILE_CONTENT_I RANGE ${FILE_CONTENT_N}) 85 | list(GET FILE_CONTENT ${FILE_CONTENT_I} LINE_VALUE) 86 | 87 | string(STRIP "${LINE_VALUE}" LINE_VALUE) 88 | 89 | if(NOT "${LINE_VALUE}" STREQUAL "") 90 | # split line to list 91 | string(REPLACE ": " ";" LINE_VARS_LIST ${LINE_VALUE}) 92 | 93 | # get vars from list 94 | list(GET LINE_VARS_LIST 0 VAR_PACKAGE_NAME) 95 | list(GET LINE_VARS_LIST 1 VAR_PACKAGE_PATH) 96 | 97 | string(STRIP "${VAR_PACKAGE_NAME}" VAR_PACKAGE_NAME) 98 | string(STRIP "${VAR_PACKAGE_PATH}" VAR_PACKAGE_PATH) 99 | 100 | # add cmake package 101 | _pm_add_package(${VAR_PACKAGE_NAME} ${VAR_PACKAGE_PATH}) 102 | endif() 103 | endforeach() 104 | 105 | 106 | 107 | 108 | endfunction() 109 | 110 | 111 | function(_pm_add_package PACKAGE_NAME PACKAGE_PATH) 112 | 113 | message(STATUS "Load package '${PACKAGE_NAME}' from '${PACKAGE_PATH}'") 114 | 115 | if(CPM_PACKAGE_${PACKAGE_NAME} AND NOT CPM_PACKAGE_${PACKAGE_NAME} STREQUAL ${PACKAGE_PATH}) 116 | message(FATAL_ERROR "CPM package '${PACKAGE_NAME}' already loaded from ${CPM_PACKAGE_${PACKAGE_NAME}}") 117 | else() 118 | set(CPM_PACKAGE_${PACKAGE_NAME} ${PACKAGE_PATH} CACHE STRING "CMAKE Package Manager guard" FORCE) 119 | endif() 120 | 121 | if (EXISTS "${PACKAGE_PATH}/_pm_package.cmake") 122 | include( "${PACKAGE_PATH}/_pm_package.cmake" NO_POLICY_SCOPE ) 123 | pm_add_lib( ${PACKAGE_PATH} ) 124 | else() 125 | message(STATUS "There is no _pm_package.cmake try to autogenerate target") 126 | _pm_add_package_auto(${PACKAGE_NAME} ${PACKAGE_PATH}) 127 | endif() 128 | endfunction() 129 | 130 | function(_pm_add_package_auto PACKAGE_NAME PACKAGE_PATH) 131 | 132 | string(TOUPPER ${PACKAGE_NAME} TARGET_IMPORTED_NAME) 133 | 134 | if(NOT TARGET ${TARGET_IMPORTED_NAME}) 135 | add_library(${TARGET_IMPORTED_NAME} INTERFACE) 136 | target_include_directories(${TARGET_IMPORTED_NAME} INTERFACE 137 | $ 138 | ) 139 | endif() 140 | 141 | endfunction() 142 | -------------------------------------------------------------------------------- /crosspm/config.py: -------------------------------------------------------------------------------- 1 | __version__ = '1.0.15' 2 | -------------------------------------------------------------------------------- /crosspm/cpm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | {app_name} 6 | Usage: 7 | crosspm download [options] 8 | crosspm lock [DEPS] [DEPSLOCK] [options] 9 | crosspm usedby [DEPS] [options] 10 | crosspm pack [options] 11 | crosspm cache [size | age | clear [hard]] 12 | crosspm -h | --help 13 | crosspm --version 14 | 15 | Options: 16 | Output file. 17 | Source directory path. 18 | -h, --help Show this screen. 19 | --version Show version. 20 | -L, --list Do not load packages and its dependencies. Just show what's found. 21 | -v LEVEL, --verbose=LEVEL Set output verbosity: ({verb_level}) [default: ]. 22 | -l LOGFILE, --log=LOGFILE File name for log output. Log level is '{log_default}' if set when verbose doesn't. 23 | -c FILE, --config=FILE Path to configuration file. 24 | -o OPTIONS, --options OPTIONS Extra options. 25 | --deps-path=FILE Path to file with dependencies [./{deps_default}] 26 | --depslock-path=FILE Path to file with locked dependencies [./{deps_lock_default}] 27 | --dependencies-content=CONTENT Content for dependencies.txt file 28 | --dependencies-lock-content=CONTENT Content for dependencies.txt.lock file 29 | --lock-on-success Save file with locked dependencies next to original one if download succeeds 30 | --out-format=TYPE Output data format. Available formats:({out_format}) [default: {out_format_default}] 31 | --output=FILE Output file name (required if --out_format is not stdout) 32 | --output-template=FILE Template path, e.g. nuget.packages.config.j2 (required if --out_format=jinja) 33 | --no-fails Ignore fails config if possible. 34 | --recursive=VALUE Process all packages recursively to find and lock all dependencies 35 | --prefer-local Do not search package if exist in cache 36 | --stdout Print info and debug message to STDOUT, error to STDERR. Otherwise - all messages to STDERR 37 | 38 | """ # noqa 39 | 40 | import logging 41 | import os 42 | import shlex 43 | import sys 44 | import time 45 | 46 | from typing import Optional 47 | 48 | from docopt import docopt 49 | 50 | from crosspm import version 51 | from crosspm.helpers.archive import Archive 52 | from crosspm.helpers.config import ( 53 | CROSSPM_DEPENDENCY_LOCK_FILENAME, 54 | CROSSPM_DEPENDENCY_FILENAME, 55 | Config, 56 | ) 57 | from crosspm.helpers.content import DependenciesContent 58 | from crosspm.helpers.downloader import Downloader 59 | from crosspm.helpers.exceptions import * # noqa 60 | from crosspm.helpers.locker import Locker 61 | from crosspm.helpers.output import Output 62 | from crosspm.helpers.python import get_object_from_string 63 | from crosspm.helpers.usedby import Usedby 64 | 65 | app_name = 'CrossPM (Cross Package Manager) version: {version} The MIT License (MIT)'.format(version=version) 66 | 67 | 68 | def do_run(func): 69 | def wrapper(self, *args, **kwargs): 70 | try: 71 | res = func(self, *args, **kwargs) 72 | except CrosspmExceptionWrongArgs as e: 73 | print(__doc__) 74 | return self.exit(e.error_code, e.msg) 75 | 76 | except CrosspmException as e: 77 | return self.exit(e.error_code, e.msg) 78 | 79 | except Exception as e: 80 | self._log.exception(e) 81 | return self.exit(CROSSPM_ERRORCODE_UNKNOWN_ERROR, 'Unknown error occurred!') 82 | return 0, res 83 | 84 | return wrapper 85 | 86 | 87 | class CrossPM: 88 | _ready = False 89 | 90 | def __init__(self, args=None, throw_exceptions=None, return_result=False): 91 | self._config: Optional[Config] = None 92 | self._output: Optional[Output] = None 93 | self._return_result = return_result 94 | 95 | if throw_exceptions is None: 96 | # legacy behavior 97 | if self._return_result: 98 | self._throw_exceptions = False 99 | else: 100 | self._throw_exceptions = True 101 | else: 102 | self._throw_exceptions = throw_exceptions 103 | 104 | self._log = logging.getLogger('crosspm') 105 | 106 | args = self.prepare_args(args) 107 | docopt_str = __doc__.format(app_name=app_name, 108 | verb_level=Config.get_verbosity_level(), 109 | log_default=Config.get_verbosity_level(0, True), 110 | deps_default=CROSSPM_DEPENDENCY_FILENAME, 111 | deps_lock_default=CROSSPM_DEPENDENCY_LOCK_FILENAME, 112 | out_format=Output.get_output_types(), 113 | out_format_default='stdout', 114 | ) 115 | self._args = docopt(docopt_str, 116 | argv=args, 117 | version=version) 118 | 119 | if self._args['--recursive']: 120 | recursive_str = self._args['--recursive'] 121 | if recursive_str.lower() == 'true': 122 | self._args['--recursive'] = True 123 | elif recursive_str.lower() == 'false': 124 | self._args['--recursive'] = False 125 | else: 126 | raise Exception("Unknown value to --recursive: {}".format(recursive_str)) 127 | 128 | if isinstance(self._args, str): 129 | if self._throw_exceptions: 130 | print(app_name) 131 | print(self._args) 132 | exit() 133 | 134 | self._ready = True 135 | 136 | if self._args['download']: 137 | self.command_ = Downloader 138 | elif self._args['lock']: 139 | self.command_ = Locker 140 | elif self._args['usedby']: 141 | self.command_ = Usedby 142 | else: 143 | self.command_ = None 144 | 145 | @property 146 | def stdout(self): 147 | """ 148 | Флаг --stdout может быть взят из переменной окружения CROSSPM_STDOUT. 149 | Если есть любое значение в CROSSPM_STDOUT - оно понимается как True 150 | :return: 151 | """ 152 | # --stdout 153 | stdout = self._args['--stdout'] 154 | if stdout: 155 | return True 156 | 157 | # CROSSPM_STDOUT 158 | stdout_env = os.getenv('CROSSPM_STDOUT', None) 159 | if stdout_env is not None: 160 | return True 161 | 162 | return False 163 | 164 | @staticmethod 165 | def prepare_args(args, windows=None): 166 | """ 167 | Prepare args - add support for old interface, e.g: 168 | - --recursive was "flag" and for now it support True or False value 169 | :param args: 170 | :return: 171 | """ 172 | if windows is None: 173 | windows = "win" in sys.platform 174 | 175 | if isinstance(args, str): 176 | args = shlex.split(args, posix=not windows) 177 | elif isinstance(args, list): 178 | pass 179 | elif args is None: 180 | args = sys.argv[1:] 181 | else: 182 | raise Exception("Unknown args type: {}".format(type(args))) 183 | 184 | # --recursive => --recursive=True|False convert 185 | for position, argument in enumerate(args): 186 | # Normal way, skip change 187 | if argument.lower() in ('--recursive=true', '--recursive=false'): 188 | return args 189 | 190 | elif argument.lower() == '--recursive': 191 | if len(args) > position + 1 and args[position + 1].lower() in ["true", "false"]: 192 | # --recursive true | false 193 | return args 194 | else: 195 | # legacy way, convert --recursive to --recursive=true 196 | args[position] = "--recursive=True" 197 | return args 198 | return args 199 | 200 | @do_run 201 | def read_config(self): 202 | _deps_path = self._args['--deps-path'] 203 | # Передаём содержимое напрямую 204 | _deps_content = DependenciesContent(self._args['--dependencies-content']) \ 205 | if _deps_path is None and self._args['--dependencies-content'] is not None else None 206 | _depslock_path = self._args['--depslock-path'] 207 | _depslock_content = DependenciesContent(self._args['--dependencies-lock-content']) \ 208 | if _depslock_path is None and self._args['--dependencies-lock-content'] is not None else None 209 | if self._args['lock']: 210 | if self._args['DEPS']: 211 | _deps_path = self._args['DEPS'] 212 | if self._args['DEPSLOCK']: 213 | _depslock_path = self._args['DEPSLOCK'] 214 | self._config = Config(self._args['--config'], self._args['--options'], self._args['--no-fails'], _depslock_path, 215 | _deps_path, self._args['--lock-on-success'], 216 | self._args['--prefer-local'], 217 | deps_content=_deps_content, 218 | deps_lock_content=_depslock_content, 219 | recursive=self._args['--recursive']) 220 | self._output = Output(self._config.output('result', None), self._config.name_column, self._config) 221 | 222 | def exit(self, code, msg): 223 | self._log.critical(msg) 224 | if self._throw_exceptions: 225 | sys.exit(code) 226 | else: 227 | return code, msg 228 | 229 | @do_run 230 | def check_common_args(self): 231 | if self._args['--output']: 232 | output = self._args['--output'].strip().strip("'").strip('"') 233 | output_abs = os.path.abspath(output) 234 | if os.path.isdir(output_abs): 235 | raise CrosspmExceptionWrongArgs( 236 | '"%s" is a directory - can\'t write to it' 237 | ) 238 | self._args['--output'] = output 239 | 240 | @do_run 241 | def set_logging_level(self): 242 | level_str = self._args['--verbose'].strip().lower() 243 | 244 | log = self._args['--log'] 245 | if log: 246 | log = log.strip().strip("'").strip('"') 247 | log_abs = os.path.abspath(log) 248 | if os.path.isdir(log_abs): 249 | raise CrosspmExceptionWrongArgs( 250 | '"%s" is a directory - can\'t write log to it' 251 | ) 252 | else: 253 | log_dir = os.path.dirname(log_abs) 254 | if not os.path.exists(log_dir): 255 | os.makedirs(log_dir) 256 | else: 257 | log_abs = None 258 | 259 | level = Config.get_verbosity_level(level_str or 'console') 260 | self._log.handlers = [] 261 | if level or log_abs: 262 | self._log.setLevel(level) 263 | format_str = '%(asctime)-19s [%(levelname)-9s] %(message)s' 264 | if level_str == 'debug': 265 | format_str = '%(asctime)-19s [%(levelname)-9s] %(name)-12s: %(message)s' 266 | formatter = logging.Formatter(format_str, datefmt="%Y-%m-%d %H:%M:%S") 267 | 268 | if level: 269 | # legacy way - Cmake catch message from stdout and parse PACKAGE_ROOT 270 | # So, crosspm print debug and info message to stderr for debug purpose 271 | if not self.stdout: 272 | sh = logging.StreamHandler(stream=sys.stderr) 273 | sh.setLevel(level) 274 | self._log.addHandler(sh) 275 | # If --stdout flag enabled 276 | else: 277 | sh = logging.StreamHandler(stream=sys.stderr) 278 | sh.setLevel(logging.WARNING) 279 | self._log.addHandler(sh) 280 | sh = logging.StreamHandler(stream=sys.stdout) 281 | sh.setLevel(level) 282 | self._log.addHandler(sh) 283 | 284 | if log_abs: 285 | if not level_str: 286 | level = Config.get_verbosity_level(0) 287 | fh = logging.FileHandler(filename=log_abs) 288 | fh.setLevel(level) 289 | fh.setFormatter(formatter) 290 | self._log.addHandler(fh) 291 | 292 | def run(self): 293 | time_start = time.time() 294 | if self._ready: 295 | 296 | errorcode, msg = self.set_logging_level() 297 | self._log.info(app_name) 298 | errorcode, msg = self.check_common_args() 299 | if errorcode == 0: 300 | errorcode, msg = self.read_config() 301 | 302 | if errorcode == 0: 303 | if self._args['download']: 304 | errorcode, msg = self.command(self.command_) 305 | elif self._args['lock']: 306 | errorcode, msg = self.command(self.command_) 307 | elif self._args['usedby']: 308 | errorcode, msg = self.command(self.command_) 309 | elif self._args['pack']: 310 | errorcode, msg = self.pack() 311 | elif self._args['cache']: 312 | errorcode, msg = self.cache() 313 | else: 314 | errorcode, msg = CROSSPM_ERRORCODE_WRONG_ARGS, self._args 315 | time_end = time.time() 316 | self._log.info('Done in %2.2f sec' % (time_end - time_start)) 317 | return errorcode, msg 318 | 319 | @do_run 320 | def command(self, command_): 321 | if self._return_result: 322 | params = {} 323 | else: 324 | if self._args['--out-format'] == 'stdout': 325 | if self._args['--output']: 326 | raise CrosspmExceptionWrongArgs( 327 | "unwanted argument '--output' while argument '--out-format={}'".format( 328 | self._args['--out-format'], 329 | )) 330 | elif not self._args['--output']: 331 | raise CrosspmExceptionWrongArgs( 332 | "argument '--output' required when argument '--out-format={}'".format( 333 | self._args['--out-format'], 334 | )) 335 | 336 | params = { 337 | 'out_format': ['--out-format', ''], 338 | 'output': ['--output', ''], 339 | 'output_template': ['--output-template', ''], 340 | # 'out_prefix': ['--out-prefix', ''], 341 | # 'depslock_path': ['--depslock-path', ''], 342 | } 343 | 344 | for k, v in params.items(): 345 | params[k] = self._args[v[0]] if v[0] in self._args else v[1] 346 | if isinstance(params[k], str): 347 | params[k] = params[k].strip('"').strip("'") 348 | 349 | # try to dynamic load --output-template from python module 350 | output_template = params['output_template'] 351 | if output_template: 352 | # Try to load from python module 353 | module_template = get_object_from_string(output_template) 354 | if module_template is not None: 355 | self._log.debug( 356 | "Found output template path '{}' from '{}'".format(module_template, output_template)) 357 | params['output_template'] = module_template 358 | else: 359 | self._log.debug("Output template '{}' use like file path".format(output_template)) 360 | 361 | # check template exist 362 | output_template = params['output_template'] 363 | if output_template and not os.path.exists(output_template): 364 | raise CrosspmException(CROSSPM_ERRORCODE_CONFIG_NOT_FOUND, 365 | "Can not find template '{}'".format(output_template)) 366 | 367 | do_load = not self._args['--list'] 368 | # hack for Locker 369 | if command_ is Locker: 370 | do_load = self._config.recursive 371 | 372 | cpm_ = command_(self._config, do_load) 373 | cpm_.entrypoint() 374 | 375 | if self._return_result: 376 | return self._return(cpm_) 377 | else: 378 | # self._output.write(params, packages) 379 | self._output.write_output(params, cpm_.get_tree_packages()) 380 | return '' 381 | 382 | def _return(self, cpm_downloader): 383 | if str(self._return_result).lower() == 'raw': 384 | return cpm_downloader.get_raw_packages() 385 | if str(self._return_result).lower() == 'tree': 386 | return cpm_downloader.get_tree_packages() 387 | else: 388 | return self._output.output_type_module(cpm_downloader.get_tree_packages()) 389 | 390 | @do_run 391 | def pack(self): 392 | Archive.create(self._args[''], self._args['']) 393 | 394 | @do_run 395 | def cache(self): 396 | if self._args['clear']: 397 | self._config.cache.clear(self._args['hard']) 398 | elif self._args['size']: 399 | self._config.cache.size() 400 | elif self._args['age']: 401 | self._config.cache.age() 402 | else: 403 | self._config.cache.info() 404 | 405 | 406 | if __name__ == '__main__': 407 | app = CrossPM() 408 | app.run() 409 | -------------------------------------------------------------------------------- /crosspm/crosspm.cmake: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.11) 2 | 3 | include(ExternalProject) 4 | 5 | set(Python_ADDITIONAL_VERSIONS 3.0 3.1 3.2 3.3 3.4 3.5) 6 | 7 | find_package(PythonInterp) 8 | 9 | 10 | set(CPM_DIR ${CMAKE_CURRENT_LIST_DIR}) 11 | 12 | 13 | function(pm_download_dependencies) 14 | 15 | _pm_download_dependencies( ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR} ) 16 | 17 | endfunction() 18 | 19 | 20 | function(pm_download_current_dependencies) 21 | 22 | _pm_download_dependencies( ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ) 23 | 24 | endfunction() 25 | 26 | 27 | function(_pm_download_dependencies SOURCE_DIR BINARY_DIR) 28 | 29 | if(NOT DEFINED CPM_TARGET_OS) 30 | if("x$ENV{TargetOS}" STREQUAL "x") 31 | message(FATAL_ERROR "Env var TargetOS not set!") 32 | else() 33 | set(CPM_TARGET_OS $ENV{TargetOS} CACHE STRING "CMAKE Package Manager variable" FORCE) 34 | endif() 35 | endif() 36 | 37 | if(NOT DEFINED CPM_TARGET_ARCH) 38 | if("x$ENV{TargetArchitecture}" STREQUAL "x") 39 | message(FATAL_ERROR "Env var TargetArchitecture not set!") 40 | else() 41 | set(CPM_TARGET_ARCH $ENV{TargetArchitecture} CACHE STRING "CMAKE Package Manager variable" FORCE) 42 | endif() 43 | endif() 44 | 45 | if(NOT DEFINED CPM_PLATFORM_TOOLSET) 46 | if("x$ENV{PlatformToolset}" STREQUAL "x") 47 | message(FATAL_ERROR "Env var PlatformToolset not set!") 48 | else() 49 | set(CPM_PLATFORM_TOOLSET $ENV{PlatformToolset} CACHE STRING "CMAKE Package Manager variable" FORCE) 50 | endif() 51 | endif() 52 | 53 | execute_process( 54 | COMMAND 55 | "${PYTHON_EXECUTABLE}" 56 | "-u" 57 | "-m" "crosspm" 58 | "download" 59 | "-o" 60 | "os=${CPM_TARGET_OS},arch=${CPM_TARGET_ARCH},cl=${CPM_PLATFORM_TOOLSET}" 61 | WORKING_DIRECTORY 62 | "${SOURCE_DIR}" 63 | OUTPUT_FILE 64 | "${BINARY_DIR}/tmp_packages.deps" 65 | RESULT_VARIABLE 66 | RESULT_CODE_VALUE 67 | ) 68 | 69 | if(NOT "${RESULT_CODE_VALUE}" STREQUAL "0") 70 | message(FATAL_ERROR "process failed RESULT_CODE='${RESULT_CODE_VALUE}'") 71 | endif() 72 | 73 | # read file to variable 74 | file(STRINGS "${BINARY_DIR}/tmp_packages.deps" FILE_CONTENT) 75 | 76 | # count lines 77 | list(LENGTH FILE_CONTENT FILE_CONTENT_LENGTH) 78 | 79 | # calc: max_i = n - 1 80 | math(EXPR FILE_CONTENT_N "${FILE_CONTENT_LENGTH} - 1") 81 | 82 | 83 | # iterate over lines 84 | foreach(FILE_CONTENT_I RANGE ${FILE_CONTENT_N}) 85 | list(GET FILE_CONTENT ${FILE_CONTENT_I} LINE_VALUE) 86 | 87 | string(STRIP "${LINE_VALUE}" LINE_VALUE) 88 | 89 | if(NOT "${LINE_VALUE}" STREQUAL "") 90 | # split line to list 91 | string(REPLACE ": " ";" LINE_VARS_LIST ${LINE_VALUE}) 92 | 93 | # get vars from list 94 | list(GET LINE_VARS_LIST 0 VAR_PACKAGE_NAME) 95 | list(GET LINE_VARS_LIST 1 VAR_PACKAGE_PATH) 96 | 97 | string(STRIP "${VAR_PACKAGE_NAME}" VAR_PACKAGE_NAME) 98 | string(STRIP "${VAR_PACKAGE_PATH}" VAR_PACKAGE_PATH) 99 | 100 | # add cmake package 101 | _pm_add_package(${VAR_PACKAGE_NAME} ${VAR_PACKAGE_PATH}) 102 | endif() 103 | endforeach() 104 | 105 | 106 | 107 | 108 | endfunction() 109 | 110 | 111 | function(_pm_add_package PACKAGE_NAME PACKAGE_PATH) 112 | 113 | message(STATUS "Load package '${PACKAGE_NAME}' from '${PACKAGE_PATH}'") 114 | 115 | if(CPM_PACKAGE_${PACKAGE_NAME} AND NOT CPM_PACKAGE_${PACKAGE_NAME} STREQUAL ${PACKAGE_PATH}) 116 | message(FATAL_ERROR "CPM package '${PACKAGE_NAME}' already loaded from ${CPM_PACKAGE_${PACKAGE_NAME}}") 117 | else() 118 | set(CPM_PACKAGE_${PACKAGE_NAME} ${PACKAGE_PATH} CACHE STRING "CMAKE Package Manager guard" FORCE) 119 | endif() 120 | 121 | if (EXISTS "${PACKAGE_PATH}/_pm_package.cmake") 122 | include( "${PACKAGE_PATH}/_pm_package.cmake" NO_POLICY_SCOPE ) 123 | pm_add_lib( ${PACKAGE_PATH} ) 124 | else() 125 | message(STATUS "There is no _pm_package.cmake try to autogenerate target") 126 | _pm_add_package_auto(${PACKAGE_NAME} ${PACKAGE_PATH}) 127 | endif() 128 | endfunction() 129 | 130 | function(_pm_add_package_auto PACKAGE_NAME PACKAGE_PATH) 131 | 132 | string(TOUPPER ${PACKAGE_NAME} TARGET_IMPORTED_NAME) 133 | 134 | if(NOT TARGET ${TARGET_IMPORTED_NAME}) 135 | add_library(${TARGET_IMPORTED_NAME} INTERFACE) 136 | target_include_directories(${TARGET_IMPORTED_NAME} INTERFACE 137 | $ 138 | ) 139 | endif() 140 | 141 | endfunction() 142 | -------------------------------------------------------------------------------- /crosspm/helpers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devopshq/crosspm/f081203edc5c4df99b0f44b7e79ed2da31aa630a/crosspm/helpers/__init__.py -------------------------------------------------------------------------------- /crosspm/helpers/archive.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import contextlib 3 | import os 4 | import shutil 5 | import sys 6 | import tarfile 7 | import zipfile 8 | 9 | from crosspm.helpers.exceptions import CrosspmException, CROSSPM_ERRORCODE_NO_FILES_TO_PACK, \ 10 | CROSSPM_ERRORCODE_UNKNOWN_ARCHIVE 11 | 12 | WINDOWS = True if sys.platform == 'win32' else False 13 | 14 | 15 | class Archive: 16 | @staticmethod 17 | def create(archive_name, src_dir_path): 18 | archive_name = os.path.realpath(archive_name) 19 | src_dir_path = os.path.realpath(src_dir_path) 20 | files_to_pack = [] 21 | archive_name_tmp = os.path.join( 22 | os.path.dirname(archive_name), 23 | 'tmp_{}'.format(os.path.basename(archive_name)), 24 | ) 25 | 26 | for name in os.listdir(src_dir_path): 27 | current_path = os.path.join(src_dir_path, name) 28 | real_path = os.path.realpath(current_path) 29 | rel_path = os.path.relpath(current_path, src_dir_path) 30 | 31 | files_to_pack.append((real_path, rel_path,)) 32 | 33 | if not files_to_pack: 34 | raise CrosspmException( 35 | CROSSPM_ERRORCODE_NO_FILES_TO_PACK, 36 | 'no files to pack, directory [{}] is empty'.format( 37 | src_dir_path 38 | ), 39 | ) 40 | 41 | with contextlib.closing(tarfile.TarFile.open(archive_name_tmp, 'w:gz')) as tf: 42 | for real_path, rel_path in files_to_pack: 43 | tf.add(real_path, arcname=rel_path) 44 | 45 | os.renames(archive_name_tmp, archive_name) 46 | 47 | @staticmethod 48 | def extract(archive_name, dst_dir_path, file_name=None): 49 | # HACK for https://bugs.python.org/issue18199 50 | if WINDOWS: 51 | dst_dir_path = "\\\\?\\" + dst_dir_path 52 | 53 | dst_dir_path_tmp = '{}_tmp'.format(dst_dir_path) 54 | 55 | # remove temp dir 56 | if os.path.exists(dst_dir_path_tmp): 57 | shutil.rmtree(dst_dir_path_tmp) 58 | if os.path.exists(dst_dir_path): 59 | os.renames(dst_dir_path, dst_dir_path_tmp) 60 | 61 | # TODO: remove hack for deb-package 62 | # ------- START deb-package ------- 63 | if archive_name.endswith(".deb"): 64 | # На Windows: 65 | # deb-пакет содержит в себе data.tar файл 66 | # Распаковываем deb в первый раз, сохраняя data.tar рядом с deb-файлом 67 | # Затем, подменяет имя архива и пускаем по обычному пути распаковки, 68 | # чтобы он нашел tar и распаковал его как обычно 69 | if WINDOWS: 70 | datatar_dir = os.path.dirname(archive_name) 71 | if not os.path.exists(datatar_dir): 72 | os.makedirs(datatar_dir) 73 | from pyunpack import Archive 74 | Archive(archive_name).extractall(datatar_dir) 75 | # Подмена имени 76 | archive_name = os.path.join(datatar_dir, 'data.tar') 77 | # На linux - распаковываем сразу 78 | else: 79 | from pyunpack import Archive 80 | if not os.path.exists(dst_dir_path): 81 | os.makedirs(dst_dir_path) 82 | Archive(archive_name).extractall(dst_dir_path) 83 | # ------- END deb-package ------- 84 | 85 | if tarfile.is_tarfile(archive_name): 86 | with contextlib.closing(tarfile.TarFile.open(archive_name, 'r:*')) as tf: 87 | if file_name: 88 | tf.extract(file_name, path=dst_dir_path) 89 | else: 90 | # Quick hack for unpack tgz on Windows with PATH like \\?\C: 91 | # Internal issue: CM-39214 92 | try: 93 | tf.extractall(path=dst_dir_path) 94 | except OSError: 95 | dst_dir_path = dst_dir_path.replace("\\\\?\\", '') 96 | tf.extractall(path=dst_dir_path) 97 | 98 | elif zipfile.is_zipfile(archive_name): 99 | with contextlib.closing(zipfile.ZipFile(archive_name, mode='r')) as zf: 100 | if file_name: 101 | zf.extract(file_name, path=dst_dir_path) 102 | else: 103 | zf.extractall(path=dst_dir_path) 104 | 105 | elif archive_name.endswith('.rar'): 106 | if not os.path.exists(dst_dir_path): 107 | os.makedirs(dst_dir_path) 108 | from pyunpack import Archive 109 | Archive(archive_name).extractall(dst_dir_path) 110 | 111 | elif archive_name.endswith('.deb'): 112 | # We unpack above 113 | pass 114 | 115 | else: 116 | tries = 0 117 | while tries < 3: 118 | tries += 1 119 | try: 120 | if os.path.exists(dst_dir_path): 121 | shutil.rmtree(dst_dir_path) 122 | if os.path.exists(dst_dir_path_tmp): 123 | os.renames(dst_dir_path_tmp, dst_dir_path) 124 | tries = 3 125 | except Exception: 126 | pass 127 | raise CrosspmException( 128 | CROSSPM_ERRORCODE_UNKNOWN_ARCHIVE, 129 | 'unknown archive type. File: [{}]'.format(archive_name), 130 | ) 131 | 132 | # remove temp dir 133 | if os.path.exists(dst_dir_path_tmp): 134 | shutil.rmtree(dst_dir_path_tmp) 135 | 136 | @staticmethod 137 | def extract_file(archive_name, dst_dir_path, file_name): 138 | 139 | try: 140 | Archive.extract(archive_name, dst_dir_path, file_name) 141 | _dest_file = os.path.join(dst_dir_path, file_name) 142 | except Exception: 143 | _dest_file = None 144 | return _dest_file 145 | -------------------------------------------------------------------------------- /crosspm/helpers/cache.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import logging 3 | import os 4 | import time 5 | from crosspm.helpers.package import md5sum 6 | from datetime import datetime, timedelta 7 | 8 | 9 | class Cache: 10 | def __init__(self, config, cache_data): 11 | self._log = logging.getLogger('crosspm') 12 | self._config = config 13 | self._cache_path = config.crosspm_cache_root 14 | if not os.path.exists(self._cache_path): 15 | os.makedirs(self._cache_path) 16 | 17 | self.path = { 18 | 'packed': os.path.realpath(os.path.join(self._cache_path, 'archive')), 19 | 'unpacked': os.path.realpath(os.path.join(self._cache_path, 'cache')), 20 | } 21 | self.temp_path = os.path.realpath(os.path.join(self._cache_path, 'tmp')) 22 | 23 | self._clear = cache_data.get('clear', {}) 24 | self._clear['auto'] = self._clear.get('auto', False) 25 | if isinstance(self._clear['auto'], str): 26 | if self._clear['auto'].lower() in ['0', 'no', '-', 'false']: 27 | self._clear['auto'] = False 28 | try: 29 | self._clear['auto'] = bool(self._clear['auto']) 30 | except Exception: 31 | self._clear['auto'] = False 32 | 33 | self._clear['size'] = self.str_to_size(self._clear.get('size', '0')) 34 | 35 | try: 36 | self._clear['days'] = int(self._clear.get('days', '0')) 37 | except Exception: 38 | self._clear['days'] = 0 39 | 40 | _unique = config.get_fails('unique', None) 41 | self._storage = cache_data.get('storage', {'packed': '', 'unpacked': ''}) 42 | if not self._storage['packed']: 43 | if _unique and isinstance(_unique, (list, tuple)): 44 | self._storage['packed'] = '/'.join('{%s}' % x for x in _unique) 45 | if self._storage['packed']: 46 | self._storage['packed'] += '/{filename}' 47 | 48 | if not self._storage['packed']: 49 | self._storage['packed'] = '{%s}/{filename}' % config.name_column 50 | 51 | if not self._storage['unpacked']: 52 | if _unique and isinstance(_unique, (list, tuple)): 53 | self._storage['unpacked'] = '/'.join('{%s}' % x for x in _unique) 54 | 55 | if not self._storage['unpacked']: 56 | self._storage['unpacked'] = '{%s}' % config.name_column 57 | 58 | def str_to_size(self, size): 59 | _size = str(size).replace(' ', '').lower() 60 | try: 61 | _size0 = float(_size[:-2]) 62 | except Exception: 63 | _size0 = 0 64 | _measure = _size[-2:] 65 | if _measure == 'kb': 66 | _size = _size0 * 1024 67 | elif _measure == 'mb': 68 | _size = _size0 * 1024 * 1024 69 | elif _measure == 'gb': 70 | _size = _size0 * 1024 * 1024 * 1024 71 | else: 72 | if _size[-1] == 'b': 73 | _size = _size[:-1] 74 | try: 75 | _size = float(_size) 76 | except Exception: 77 | _size = 0 78 | return _size 79 | 80 | def size_to_str(self, size, dec=0): 81 | try: 82 | _size = float(size) 83 | except Exception: 84 | _size = 0 85 | _measure = 'b ' 86 | _size0 = round(_size / 1024, 3) 87 | if _size0 >= 1.0: 88 | _measure = 'Kb' 89 | _size = _size0 90 | _size0 = round(_size / 1024, 3) 91 | if _size0 >= 1.0: 92 | _measure = 'Mb' 93 | _size = _size0 94 | _size0 = round(_size / 1024, 3) 95 | if _size0 >= 1.0: 96 | _measure = 'Gb' 97 | _size = _size0 98 | _size = round(_size, dec) 99 | if dec == 0: 100 | _size = int(_size) 101 | return '{:,} {}'.format(_size, _measure) 102 | 103 | def get_info(self, get_files=True): 104 | def get_dir(_dir_name, sub_dirs=True): 105 | res = { 106 | 'path': _dir_name, 107 | 'size': 0, 108 | 'time': 0, 109 | 'files': [], 110 | 'folders': [], 111 | } 112 | if os.path.isdir(_dir_name): 113 | for item in os.listdir(_dir_name): 114 | item_name = os.path.realpath(os.path.join(_dir_name, item)) 115 | item_stat = os.stat(item_name) 116 | _size = getattr(item_stat, 'st_size', 0) 117 | _now = datetime.now() 118 | _time = getattr(item_stat, 'st_ctime', 119 | time.mktime(_now.timetuple()) + float(_now.microsecond) / 1000000.0) 120 | if os.path.isfile(os.path.join(_dir_name, item)): 121 | res['size'] += _size 122 | res['time'] = max(res['time'], _time) 123 | res['files'].append({'size': _size, 124 | 'time': _time, 125 | 'path': item_name}) 126 | elif sub_dirs: 127 | _folder = get_dir(item_name) 128 | res['folders'].append(_folder) 129 | res['size'] += _folder['size'] 130 | res['time'] = max(res['time'], _folder['time']) 131 | return res 132 | 133 | total = { 134 | 'packed': get_dir(self.path['packed']), 135 | 'unpacked': get_dir(self.path['unpacked']), 136 | 'temp': get_dir(self.temp_path), 137 | 'other': get_dir(self._cache_path, False), 138 | } 139 | return total 140 | 141 | def _sort(self, item): 142 | return item['time'] 143 | 144 | def info(self): 145 | print("Cache info:") 146 | 147 | def _delete_dir(self, _dirs, max_time=None, del_size=None): 148 | cleared = [0] 149 | 150 | def do_delete_dir(_dir): 151 | _size = 0 152 | folders_to_delete = [] 153 | files_to_delete = [] 154 | for _folder in _dir['folders']: 155 | _size += do_delete_dir(_folder) 156 | if len(_folder['folders']) + len(_folder['files']) == 0: 157 | os.rmdir(_folder['path']) 158 | folders_to_delete += [_folder] 159 | for _file in _dir['files']: 160 | do_clear = False 161 | if max_time: 162 | if max_time > _file['time']: 163 | do_clear = True 164 | if del_size: 165 | if del_size > cleared[0]: 166 | do_clear = True 167 | if not (max_time or del_size): 168 | do_clear = True 169 | if do_clear: 170 | _size += _file['size'] 171 | cleared[0] += _file['size'] 172 | os.remove(_file['path']) 173 | files_to_delete += [_file] 174 | _dir['size'] -= _size 175 | for _folder in folders_to_delete: 176 | _dir['folders'].remove(_folder) 177 | for _file in files_to_delete: 178 | _dir['files'].remove(_file) 179 | return _size 180 | 181 | return do_delete_dir(_dirs) 182 | 183 | def auto_clear(self): 184 | if self._clear['auto']: 185 | self.clear(False, True) 186 | 187 | def clear(self, hard=False, auto=False): 188 | 189 | # TODO: Check if given path is really crosspm cache before deleting anything! 190 | _now = datetime.now().timestamp() 191 | self._log.info('Read cache...') 192 | max_size = self._clear['size'] 193 | if self._clear['days']: 194 | max_time = _now - timedelta(days=self._clear['days']).total_seconds() 195 | else: 196 | max_time = 0 197 | 198 | total = self.get_info() 199 | 200 | total_size = sum(total[x]['size'] for x in total) 201 | oldest = min(total[x]['time'] if total[x]['time'] else _now for x in total) 202 | 203 | if auto and not hard: 204 | do_clear = False 205 | if max_size and max_size < total_size: 206 | do_clear = True 207 | if max_time and max_time > oldest: 208 | do_clear = True 209 | if not do_clear: 210 | return 211 | 212 | self._log.info('Clear cache{}'.format(' HARD! >:-E' if hard else ':')) 213 | 214 | self.size(total) 215 | 216 | # TEMP 217 | cleared = self._delete_dir(total['temp']) 218 | 219 | # UNPACKED 220 | if hard: 221 | cleared += self._delete_dir(total['unpacked']) 222 | else: 223 | folders_to_delete = [] 224 | _size = 0 225 | for folder in sorted(total['unpacked']['folders'], key=self._sort): 226 | do_clear = False 227 | if max_size: 228 | if total_size - cleared - _size > max_size: 229 | do_clear = True 230 | if max_time: 231 | if folder['time'] < max_time: 232 | do_clear = True 233 | if do_clear: 234 | _size += self._delete_dir(folder) 235 | if len(folder['folders']) + len(folder['files']) == 0: 236 | os.rmdir(folder['path']) 237 | folders_to_delete += [folder] 238 | # if (not do_clear) and (not max_time): 239 | # break 240 | total['unpacked']['size'] -= _size 241 | cleared += _size 242 | for folder in folders_to_delete: 243 | total['unpacked']['folders'].remove(folder) 244 | 245 | # PACKED 246 | if hard: 247 | cleared += self._delete_dir(total['packed']) 248 | else: 249 | del_size = total_size - cleared - max_size 250 | if del_size < 0: 251 | del_size = 0 252 | if del_size or max_time: 253 | cleared += self._delete_dir(total['packed'], max_time, del_size) 254 | 255 | self._log.info('') 256 | self._log.info('Очищено: {}'.format(self.size_to_str(cleared, 1))) 257 | self._log.info('') 258 | self.size(total) 259 | 260 | def size(self, total=None): 261 | if not total: 262 | total = self.get_info(False) 263 | self._log.info('Cache size:') 264 | self._log.info(' packed: {:>15}'.format(self.size_to_str(total['packed']['size'], 1))) 265 | self._log.info(' unpacked: {:>15}'.format(self.size_to_str(total['unpacked']['size'], 1))) 266 | self._log.info(' temp: {:>15}'.format(self.size_to_str(total['temp']['size'], 1))) 267 | self._log.info(' other: {:>15}'.format(self.size_to_str(total['other']['size'], 1))) 268 | self._log.info('---------------------------') 269 | self._log.info(' total: {:>15}'.format(self.size_to_str(sum(total[x]['size'] for x in total), 1))) 270 | 271 | def age(self, total=None): 272 | if not total: 273 | total = self.get_info(False) 274 | oldest = min((x for x in ( 275 | total['packed']['time'], total['unpacked']['time'], total['temp']['time'], total['other']['time']) if x)) 276 | self._log.info('Cache oldest timestamp:') 277 | self._log.info(' packed: {}'.format(datetime.fromtimestamp(total['packed']['time']))) 278 | self._log.info(' unpacked: {}'.format(datetime.fromtimestamp(total['unpacked']['time']))) 279 | self._log.info(' temp: {}'.format(datetime.fromtimestamp(total['temp']['time']))) 280 | self._log.info(' other: {}'.format(datetime.fromtimestamp(total['other']['time']))) 281 | self._log.info('---------------------------') 282 | self._log.info(' oldest: {}'.format(datetime.fromtimestamp(oldest))) 283 | 284 | def path_packed(self, package=None, params=None): 285 | return self.path_any('packed', package, params) 286 | 287 | def path_unpacked(self, package=None, params=None): 288 | return self.path_any('unpacked', package, params) 289 | 290 | def path_any(self, name, package=None, params=None): 291 | if params: 292 | tmp_params = {} 293 | for k, v in params.items(): 294 | if isinstance(v, list): 295 | v = ".".join([x for x in v if x is not None and x != '']) 296 | tmp_params[k] = v 297 | res = self._storage[name].format(**tmp_params) 298 | elif package: 299 | res = self._storage[name].format(**(package.get_params(merged=True))) 300 | else: 301 | res = '' 302 | res = os.path.realpath(os.path.join(self.path[name], res)) 303 | return res 304 | 305 | def exists_packed(self, package=None, params=None, pkg_path='', check_stat=True): 306 | # Check if file exists and size and time match 307 | path = self.path_packed(package, params) if not pkg_path else pkg_path 308 | res = os.path.exists(path) 309 | if res and package and check_stat: 310 | md5file = md5sum(filename=path) 311 | if package.md5 != md5file: 312 | res = False 313 | os.remove(path) 314 | return res, path 315 | 316 | def exists_unpacked(self, package, filename=None, params=None, pkg_path=''): 317 | # TODO: Check if file exists and size and time match 318 | path = self.path_unpacked(package, params) if not pkg_path else pkg_path 319 | if os.path.isdir(path): 320 | # check that dir is not empty 321 | res = bool(os.listdir(path)) 322 | else: 323 | res = os.path.exists(path) 324 | 325 | return res, path 326 | -------------------------------------------------------------------------------- /crosspm/helpers/content.py: -------------------------------------------------------------------------------- 1 | class DependenciesContent(str): 2 | """ 3 | Используется только для определения что мы подали на вход не конкретный путь до файла, а уже готовый контент 4 | """ 5 | pass 6 | -------------------------------------------------------------------------------- /crosspm/helpers/downloader.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import logging 3 | import os 4 | from collections import OrderedDict, defaultdict 5 | from typing import Optional 6 | 7 | from crosspm.helpers.config import CROSSPM_DEPENDENCY_FILENAME, CROSSPM_DEPENDENCY_LOCK_FILENAME, Config # noqa 8 | from crosspm.helpers.exceptions import * 9 | from crosspm.helpers.package import Package 10 | from crosspm.helpers.parser import Parser 11 | 12 | 13 | class Command(object): 14 | def entrypoint(self, *args, **kwargs): 15 | assert NotImplemented 16 | 17 | 18 | class Downloader(Command): 19 | def __init__(self, config: Config, do_load: bool, recursive: Optional[bool] = None): 20 | self._log = logging.getLogger('crosspm') 21 | self._config = config # type: Config 22 | self.cache = config.cache 23 | self.solid = config.solid 24 | self.common_parser = Parser('common', {}, config) 25 | self._root_package = Package('', 0, {self._config.name_column: ''}, self, None, 26 | self.common_parser) 27 | self.recursive = config.recursive if recursive is None else recursive 28 | 29 | self.do_load = do_load 30 | 31 | def update_progress(self, msg, progress): 32 | self._log.info('\r{0} [{1:10}] {2}%'.format(msg, '#' * int(float(progress) / 10.0), int(progress))) 33 | 34 | # Get list of all packages needed to resolve all the dependencies. 35 | # List of Package class instances. 36 | def get_dependency_packages(self, list_or_file_path=None, property_validate=True): 37 | """ 38 | :param list_or_file_path: 39 | :param property_validate: for `root` packages we need check property, bad if we find packages from `lock` file, 40 | we can skip validate part 41 | """ 42 | if list_or_file_path is None: 43 | list_or_file_path = self._config.deps_lock_file_path 44 | if not os.path.isfile(list_or_file_path) or self._config.lock_on_success: 45 | list_or_file_path = self._config.deps_file_path 46 | 47 | _packages = OrderedDict() 48 | if isinstance(list_or_file_path, str): 49 | self._log.info('Reading dependencies ... [%s]', list_or_file_path) 50 | for i, _src in enumerate(self._config.sources()): 51 | if i > 0: 52 | self._log.info('') 53 | self._log.info('Next source ...') 54 | _found_packages = _src.get_packages(self, list_or_file_path, property_validate) 55 | _packages.update( 56 | OrderedDict([(k, v) for k, v in _found_packages.items() if _packages.get(k, None) is None])) 57 | if not self._config.no_fails: 58 | if isinstance(list_or_file_path, (list, tuple)): 59 | list_or_file_path = [x for x in list_or_file_path if 60 | _packages.get(x[self._config.name_column], None) is None] 61 | elif isinstance(list_or_file_path, dict) and isinstance(list_or_file_path.get('raw', None), list): 62 | list_or_file_path['raw'] = [x for x in list_or_file_path['raw'] if 63 | _packages.get(x[self._config.name_column], None) is None] 64 | 65 | return _packages 66 | 67 | def get_usedby_packages(self, list_or_file_path=None, property_validate=True): 68 | """ 69 | 70 | :param list_or_file_path: 71 | :param property_validate: for `root` packages we need check property, bad if we find packages from `lock` file, 72 | we can skip validate part 73 | :return: 74 | """ 75 | if list_or_file_path is None: 76 | list_or_file_path = self._config.deps_lock_file_name 77 | if not os.path.isfile(list_or_file_path): 78 | list_or_file_path = self._config.deps_file_name 79 | 80 | _packages = OrderedDict() 81 | if isinstance(list_or_file_path, str): 82 | self._log.info('Reading dependencies ... [%s]', list_or_file_path) 83 | for i, _src in enumerate(self._config.sources()): 84 | if i > 0: 85 | self._log.info('') 86 | self._log.info('Next source ...') 87 | _found_packages = _src.get_usedby(self, list_or_file_path, property_validate) 88 | _packages.update( 89 | OrderedDict([(k, v) for k, v in _found_packages.items() if _packages.get(k, None) is None])) 90 | if not self._config.no_fails: 91 | if isinstance(list_or_file_path, (list, tuple)): 92 | list_or_file_path = [x for x in list_or_file_path if 93 | _packages.get(x[self._config.name_column], None) is None] 94 | elif isinstance(list_or_file_path, dict) and isinstance(list_or_file_path.get('raw', None), list): 95 | list_or_file_path['raw'] = [x for x in list_or_file_path['raw'] if 96 | _packages.get(x[self._config.name_column], None) is None] 97 | 98 | return _packages 99 | 100 | # Download packages or just unpack already loaded (it's up to adapter to decide) 101 | def download_packages(self, depslock_file_path=None): 102 | deps_content = self._config.deps_content \ 103 | if not self._config.deps_lock_content \ 104 | else self._config.deps_lock_content 105 | if depslock_file_path is None: 106 | depslock_file_path = self._config.deps_lock_file_path 107 | if os.path.isfile(depslock_file_path) and not self._config.lock_on_success: 108 | self.search_dependencies(depslock_file_path, deps_content=deps_content) 109 | else: 110 | self.search_dependencies(self._config.deps_file_path, deps_content=deps_content) 111 | 112 | if self.do_load: 113 | self._log.info('Unpack ...') 114 | 115 | total = len(self._root_package.all_packages) 116 | for i, _pkg in enumerate(self._root_package.all_packages): 117 | self.update_progress('Download/Unpack:', float(i) / float(total) * 100.0) 118 | if _pkg.download(): # self.packed_path): 119 | _pkg.unpack() # self.unpacked_path) 120 | 121 | self.update_progress('Download/Unpack:', 100) 122 | self._log.info('') 123 | self._log.info('Done!') 124 | 125 | if self._config.lock_on_success: 126 | from crosspm.helpers.locker import Locker 127 | Locker(self._config, do_load=self.do_load, recursive=self.recursive).\ 128 | lock_packages(self._root_package.packages) 129 | 130 | return self._root_package.all_packages 131 | 132 | def entrypoint(self, *args, **kwargs): 133 | self.download_packages(*args, **kwargs) 134 | 135 | def search_dependencies(self, depslock_file_path, deps_content=None): 136 | self._log.info('Check dependencies ...') 137 | self._root_package.find_dependencies(depslock_file_path, property_validate=True, deps_content=deps_content, ) 138 | self._log.info('') 139 | self.set_duplicated_flag() 140 | self._log.info('Dependency tree:') 141 | self._root_package.print(0, self._config.output('tree', [{self._config.name_column: 0}])) 142 | self.check_unique(self._config.no_fails) 143 | self.check_not_found() 144 | 145 | def check_not_found(self): 146 | _not_found = self.get_not_found_packages() 147 | if _not_found: 148 | raise CrosspmException( 149 | CROSSPM_ERRORCODE_PACKAGE_NOT_FOUND, 150 | 'Some package(s) not found: {}'.format(', '.join(_not_found)) 151 | ) 152 | 153 | def get_not_found_packages(self): 154 | return self._root_package.get_none_packages() 155 | 156 | def add_package(self, pkg_name, package): 157 | _added = False 158 | if package is not None: 159 | _added = True 160 | return _added, package 161 | 162 | def set_duplicated_flag(self): 163 | """ 164 | For all package set flag duplicated, if it's not unique package 165 | :return: 166 | """ 167 | package_by_name = defaultdict(list) 168 | 169 | for package1 in self._root_package.all_packages: 170 | if package1 is None: 171 | continue 172 | pkg_name = package1.package_name 173 | param_list = self._config.get_fails('unique', {}) 174 | params1 = package1.get_params(param_list) 175 | for package2 in package_by_name[pkg_name]: 176 | params2 = package2.get_params(param_list) 177 | for x in param_list: 178 | # START HACK for cached archive 179 | param1 = params1[x] 180 | param2 = params2[x] 181 | if isinstance(param1, list): 182 | param1 = [str(x) for x in param1] 183 | if isinstance(param2, list): 184 | param2 = [str(x) for x in param2] 185 | # END 186 | 187 | if str(param1) != str(param2): 188 | package1.duplicated = True 189 | package2.duplicated = True 190 | package_by_name[pkg_name].append(package1) 191 | 192 | def check_unique(self, no_fails): 193 | if no_fails: 194 | return 195 | not_unique = set(x.package_name for x in self._root_package.all_packages if x and x.duplicated) 196 | if not_unique: 197 | raise CrosspmException( 198 | CROSSPM_ERRORCODE_MULTIPLE_DEPS, 199 | 'Multiple versions of package "{}" found in dependencies.\n' 200 | 'See dependency tree in log (package with exclamation mark "!")'.format( 201 | ', '.join(not_unique)), 202 | ) 203 | 204 | def get_raw_packages(self): 205 | """ 206 | Get all packages 207 | :return: list of all packages 208 | """ 209 | return self._root_package.all_packages 210 | 211 | def get_tree_packages(self): 212 | """ 213 | Get all packages, with hierarchy 214 | :return: list of first level packages, with child 215 | """ 216 | return self._root_package.packages 217 | -------------------------------------------------------------------------------- /crosspm/helpers/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | CROSSPM_ERRORCODES = ( 4 | CROSSPM_ERRORCODE_SUCCESS, 5 | CROSSPM_ERRORCODE_UNKNOWN_ERROR, 6 | CROSSPM_ERRORCODE_WRONG_ARGS, 7 | CROSSPM_ERRORCODE_FILE_DEPS_NOT_FOUND, 8 | CROSSPM_ERRORCODE_WRONG_SYNTAX, 9 | CROSSPM_ERRORCODE_MULTIPLE_DEPS, 10 | CROSSPM_ERRORCODE_NO_FILES_TO_PACK, 11 | CROSSPM_ERRORCODE_SERVER_CONNECT_ERROR, 12 | CROSSPM_ERRORCODE_PACKAGE_NOT_FOUND, 13 | CROSSPM_ERRORCODE_PACKAGE_BRANCH_NOT_FOUND, 14 | CROSSPM_ERRORCODE_VERSION_PATTERN_NOT_MATCH, 15 | CROSSPM_ERRORCODE_UNKNOWN_OUT_TYPE, 16 | CROSSPM_ERRORCODE_FILE_IO, 17 | CROSSPM_ERRORCODE_CONFIG_NOT_FOUND, 18 | CROSSPM_ERRORCODE_CONFIG_IO_ERROR, 19 | CROSSPM_ERRORCODE_CONFIG_FORMAT_ERROR, 20 | CROSSPM_ERRORCODE_ADAPTER_ERROR, 21 | CROSSPM_ERRORCODE_UNKNOWN_ARCHIVE, 22 | ) = range(18) 23 | 24 | 25 | class CrosspmException(Exception): 26 | def __init__(self, error_code, msg=''): 27 | super().__init__(msg) 28 | self.error_code = error_code 29 | self.msg = msg 30 | 31 | 32 | class CrosspmExceptionWrongArgs(CrosspmException): 33 | def __init__(self, msg=''): 34 | super().__init__(CROSSPM_ERRORCODE_WRONG_ARGS, msg) 35 | -------------------------------------------------------------------------------- /crosspm/helpers/locker.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from typing import Optional, Dict 3 | from crosspm.helpers.package import Package 4 | from crosspm.helpers.downloader import Downloader 5 | from crosspm.helpers.output import Output 6 | from crosspm.helpers.config import Config 7 | 8 | 9 | class Locker(Downloader): 10 | def __init__(self, config: Config, do_load: bool, recursive: Optional[bool] = None): 11 | # TODO: revise logic to allow recursive search without downloading 12 | super(Locker, self).__init__(config, do_load, recursive) 13 | 14 | def lock_packages(self, packages: Optional[Dict[str, Package]] = None): 15 | """ 16 | Lock packages. Downloader search packages 17 | """ 18 | 19 | if packages: 20 | self._root_package.packages = packages 21 | 22 | if len(self._root_package.packages) == 0: 23 | self.search_dependencies(self._config.deps_file_path, self._config.deps_content) 24 | 25 | output_params = { 26 | 'out_format': 'lock', 27 | 'output': self._config.deps_lock_file_path, 28 | } 29 | Output(config=self._config).write_output(output_params, self._root_package.packages) 30 | self._log.info('Done!') 31 | 32 | def entrypoint(self, *args, **kwargs): 33 | self.lock_packages(*args, **kwargs) 34 | -------------------------------------------------------------------------------- /crosspm/helpers/package.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import fnmatch 3 | import hashlib 4 | import logging 5 | import os 6 | import shutil 7 | from collections import OrderedDict 8 | from typing import Dict 9 | 10 | from artifactory import ArtifactoryPath 11 | 12 | from crosspm.helpers.archive import Archive 13 | 14 | # import only for TypeHint 15 | try: 16 | from crosspm.helpers.downloader import Downloader # noqa: F401 17 | except ImportError: 18 | pass 19 | 20 | 21 | class Package: 22 | def __init__(self, name, pkg, params, downloader, adapter, parser, params_found=None, params_found_raw=None, 23 | stat=None, in_cache=False): 24 | self.name = name 25 | self.package_name = name 26 | self.packed_path = '' 27 | self.unpacked_path = '' 28 | self.duplicated = False 29 | self.packages: Dict[str, Package] = OrderedDict() 30 | 31 | self.pkg = pkg # type: ArtifactoryPath 32 | # Someone use this internal object, do not remove them :) 33 | self._pkg = self.pkg 34 | 35 | if isinstance(pkg, int): 36 | if pkg == 0: 37 | self._root = True 38 | 39 | self._raw = [] 40 | self._root = False 41 | self._params_found = {} 42 | self._params_found_raw = {} 43 | self._not_cached = True 44 | self._log = logging.getLogger('crosspm') 45 | 46 | self._params = params 47 | self._adapter = adapter 48 | self._parser = parser 49 | self._downloader = downloader # type: Downloader 50 | self._in_cache = in_cache 51 | 52 | if params_found: 53 | self._params_found = params_found 54 | if params_found_raw: 55 | self._params_found_raw = params_found_raw 56 | self.stat = stat 57 | 58 | def download(self, force=False): 59 | """ 60 | Download file containing this package. 61 | :param force: Force download even if it seems file already exists 62 | :return: Full path with filename of downloaded package file. 63 | """ 64 | exists, dest_path = self._downloader.cache.exists_packed(package=self, pkg_path=self.packed_path, 65 | check_stat=not self._in_cache) 66 | unp_exists, unp_path = self._downloader.cache.exists_unpacked(package=self, pkg_path=self.unpacked_path) 67 | 68 | # Если архива нет, то и кешу доверять не стоит 69 | if not exists: 70 | unp_exists = False 71 | 72 | if exists and not self.packed_path: 73 | self.packed_path = dest_path 74 | 75 | if force or not exists: 76 | # _packed_path = self._packed_path 77 | dest_path_tmp = dest_path + ".tmp" 78 | if os.path.exists(dest_path_tmp): 79 | os.remove(dest_path_tmp) 80 | self._adapter.download_package(self._pkg, dest_path_tmp) 81 | os.rename(dest_path_tmp, dest_path) 82 | self.packed_path = dest_path 83 | # if not _packed_path: 84 | self._not_cached = True 85 | else: 86 | if unp_exists and not self.unpacked_path: 87 | self.unpacked_path = unp_path 88 | self._not_cached = False 89 | 90 | if self._not_cached and unp_exists: 91 | shutil.rmtree(unp_path, ignore_errors=True) 92 | 93 | return self.packed_path 94 | 95 | def get_file(self, file_name, unpack_force=True): 96 | if unpack_force: 97 | self.unpack() 98 | _dest_file = os.path.normpath(self.get_file_path(file_name)) 99 | _dest_file = _dest_file if os.path.isfile(_dest_file) else None 100 | return _dest_file 101 | 102 | def get_file_path(self, file_name): 103 | _dest_file = os.path.join(self.unpacked_path, file_name) 104 | return _dest_file 105 | 106 | def find_dependencies(self, depslock_file_path, property_validate=True, deps_content=None): 107 | """ 108 | Find all dependencies by package 109 | :param depslock_file_path: 110 | :param property_validate: for `root` packages we need check property, bad if we find packages from `lock` file, 111 | :param deps_content: HACK for use --dependencies-content and existed dependencies.txt.lock file 112 | we can skip validate part 113 | :return: 114 | """ 115 | self._raw = [x for x in 116 | self._downloader.common_parser.iter_packages_params(depslock_file_path, deps_content=deps_content)] 117 | self.packages = self._downloader.get_dependency_packages({'raw': self._raw}, 118 | property_validate=property_validate) 119 | 120 | def find_usedby(self, depslock_file_path, property_validate=True): 121 | """ 122 | Find all dependencies by package 123 | :param depslock_file_path: 124 | :param property_validate: for `root` packages we need check property, bad if we find packages from `lock` file, 125 | we can skip validate part 126 | :return: 127 | """ 128 | if depslock_file_path is None: 129 | self._raw = [self._params] 130 | self._raw[0]['repo'] = None 131 | self._raw[0]['server'] = None 132 | else: 133 | self._raw = [x for x in self._downloader.common_parser.iter_packages_params(depslock_file_path)] 134 | self.packages = self._downloader.get_usedby_packages({'raw': self._raw}, 135 | property_validate=property_validate) 136 | 137 | def unpack(self, force=False): 138 | if self._downloader.solid(self): 139 | self.unpacked_path = self.packed_path 140 | else: 141 | exists, dest_path = self._downloader.cache.exists_unpacked(package=self, pkg_path=self.unpacked_path) 142 | if exists and not self.unpacked_path: 143 | self.unpacked_path = dest_path 144 | 145 | # if force or not exists: 146 | 147 | # if not dest_path: 148 | # dest_path = self._downloader.unpacked_path 149 | # temp_path = os.path.realpath(os.path.join(dest_path, self._name)) 150 | # _exists = os.path.exists(temp_path) 151 | if not self._not_cached: 152 | self.unpacked_path = dest_path if exists else '' # temp_path if exists else '' 153 | if force or self._not_cached or (not exists): 154 | Archive.extract(self.packed_path, dest_path) # temp_path) 155 | self.unpacked_path = dest_path # temp_path 156 | self._not_cached = False 157 | 158 | def pack(self, src_path): 159 | Archive.create(self.packed_path, src_path) 160 | 161 | def print(self, level=0, output=None): 162 | def do_print(left): 163 | res_str = '' 164 | for out_item in output: 165 | for k, v in out_item.items(): 166 | cur_str = self.get_params(merged=True).get(k, '') 167 | if not res_str: 168 | cur_str = self._params.get(k, '') 169 | if not res_str: 170 | res_str = '{}{}'.format(left, cur_str) 171 | continue 172 | cur_format = ' {}' 173 | if v > 0: 174 | cur_format = '{:%s}' % (v if len(cur_str) <= v else v + len(left)) 175 | res_str += cur_format.format(cur_str) 176 | break 177 | self._log.info(res_str) 178 | 179 | _sign = ' ' 180 | if not self._root: 181 | if self.duplicated: 182 | _sign = '!' 183 | elif self.unpacked_path: 184 | _sign = '+' 185 | elif self.packed_path: 186 | _sign = '>' 187 | else: 188 | _sign = '-' 189 | _left = '{}{}'.format(' ' * 4 * level, _sign) 190 | do_print(_left) 191 | for _pkg_name in self.packages: 192 | _pkg = self.packages[_pkg_name] 193 | if not _pkg: 194 | _left = '{}-'.format(' ' * 4 * (level + 1)) 195 | self._log.info('{}{}'.format(_left, _pkg_name)) 196 | else: 197 | _pkg.print(level + 1, output) 198 | if self._root: 199 | self._log.info('') 200 | 201 | def get_params(self, param_list=None, get_path=False, merged=False, raw=False): 202 | """ 203 | Get Package params 204 | :param param_list: name or list of parameters 205 | :param get_path: 206 | :param merged: if version splited, True return version in string 207 | :param raw: 208 | :return: 209 | """ 210 | # Convert parameter name to list 211 | if param_list and isinstance(param_list, str): 212 | param_list = [param_list] 213 | if param_list and isinstance(param_list, (list, tuple)): 214 | result = {k: v for k, v in self._params_found.items() if k in param_list} 215 | result.update({k: v for k, v in self._params.items() if (k in param_list and k not in result)}) 216 | else: 217 | result = {k: v for k, v in self._params_found.items()} 218 | result.update({k: v for k, v in self._params.items() if k not in result}) 219 | if get_path: 220 | result['path'] = self.unpacked_path 221 | if merged: 222 | result.update(self._parser.merge_valued(result)) 223 | if raw: 224 | result.update({k: v for k, v in self._params_found_raw.items()}) 225 | return result 226 | 227 | def set_full_unique_name(self): 228 | self.name = self._parser.get_full_package_name(self) 229 | return self.name 230 | 231 | def get_name_and_path(self, name_only=False): 232 | if name_only: 233 | return self.name 234 | return self.name, self.unpacked_path 235 | 236 | def get_none_packages(self): 237 | """ 238 | Get packages with None (not founded), recursively 239 | """ 240 | not_found = set() 241 | for package_name, package in self.packages.items(): 242 | if package is None: 243 | not_found.add(package_name) 244 | else: 245 | if package.packages: 246 | not_found = not_found | package.get_none_packages() 247 | return not_found 248 | 249 | @property 250 | def all_packages(self): 251 | packages = [] 252 | for package in self.packages.values(): 253 | if package: 254 | packages.extend(package.all_packages) 255 | packages.extend(self.packages.values()) 256 | return packages 257 | 258 | def ext(self, check_ext): 259 | if self._pkg: 260 | if not isinstance(check_ext, (list, tuple)): 261 | check_ext = [check_ext] 262 | name = self._adapter.get_package_filename(self._pkg) 263 | if any((fnmatch.fnmatch(name, x) or fnmatch.fnmatch(name, '*%s' % x)) for x in check_ext): 264 | return True 265 | return False 266 | 267 | @property 268 | def md5(self): 269 | try: 270 | return ArtifactoryPath.stat(self.pkg).md5 271 | except AttributeError: 272 | return md5sum(self.packed_path) 273 | 274 | 275 | def md5sum(filename): 276 | """ 277 | Calculates md5 hash of a file 278 | """ 279 | md5 = hashlib.md5() 280 | with open(filename, 'rb') as f: 281 | for chunk in iter(lambda: f.read(128 * md5.block_size), b''): 282 | md5.update(chunk) 283 | return md5.hexdigest() 284 | -------------------------------------------------------------------------------- /crosspm/helpers/promoter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import fnmatch 4 | import logging 5 | 6 | # third-party 7 | import requests 8 | from crosspm.helpers.config import CROSSPM_DEPENDENCY_LOCK_FILENAME 9 | from crosspm.helpers.exceptions import * 10 | 11 | 12 | # log = logging.getLogger(__name__) 13 | 14 | 15 | # TODO: REWORK PROMOTER COMPLETELY to match new architecture!!! 16 | class Promoter: 17 | def __init__(self, config, depslock_path=''): 18 | self._log = logging.getLogger('crosspm') 19 | self._config = config 20 | self._cache_path = config.crosspm_cache_root 21 | if not os.path.exists(self._cache_path): 22 | os.makedirs(self._cache_path) 23 | 24 | self.packed_path = os.path.realpath(os.path.join(self._cache_path, 'archive')) 25 | self.unpacked_path = os.path.realpath(os.path.join(self._cache_path, 'cache')) 26 | self.temp_path = os.path.realpath(os.path.join(self._cache_path, 'tmp')) 27 | 28 | if not depslock_path: 29 | depslock_path = config.deps_lock_file_path \ 30 | if config.deps_lock_file_path \ 31 | else CROSSPM_DEPENDENCY_LOCK_FILENAME 32 | self._depslock_path = os.path.realpath(depslock_path) 33 | 34 | @staticmethod 35 | def get_version_int(version_str): 36 | parts = version_str.split('.') 37 | try: 38 | parts = list(map(int, parts)) 39 | if len(parts) < 4: 40 | parts.insert(2, 0) 41 | return parts 42 | 43 | except ValueError: 44 | return [0] * 4 45 | 46 | except IndexError: 47 | return [0] * 4 48 | 49 | def parse_dir_list(self, data_dict, dir_list_data): 50 | for item in dir_list_data["files"]: 51 | if item["folder"]: 52 | continue 53 | 54 | parts = item["uri"].split('/') 55 | 56 | name = parts[1] 57 | branch = parts[2] 58 | version = parts[3] 59 | version_int = self.get_version_int(version) 60 | 61 | data_dict.setdefault(name, {}) 62 | data_dict[name].setdefault(branch, []) 63 | data_dict[name][branch].append((version, version_int)) 64 | 65 | @staticmethod 66 | def join_package_path(server_url, part_a, repo, path): 67 | server_url = server_url if not server_url.endswith('/') else server_url[: -1] 68 | part_a = part_a if not part_a.startswith('/') else part_a[1:] 69 | part_a = part_a if not part_a.endswith('/') else part_a[: -1] 70 | repo = repo if not repo.startswith('/') else repo[1:] 71 | repo = repo if not repo.endswith('/') else repo[: -1] 72 | path = path if not path.startswith('/') else path[1:] 73 | 74 | return '{server_url}/{part_a}/{repo}/{path}'.format(**locals()) 75 | 76 | def promote_packages(self, list_or_file_path=None): 77 | if list_or_file_path is None: 78 | list_or_file_path = self._depslock_path 79 | 80 | data_tree = {} 81 | deps_list = [] # pm_common.get_dependencies(os.path.join(os.getcwd(), CROSSPM_DEPENDENCY_FILENAME)) 82 | out_file_data_str = '' 83 | out_file_format = '{:20s} {:20s} {}\n' 84 | out_file_path = os.path.join(os.getcwd(), CROSSPM_DEPENDENCY_LOCK_FILENAME) 85 | 86 | # clean file 87 | with open(out_file_path, 'w') as out_f: 88 | out_f.write(out_file_format.format('# package', '# version', '# branch\n')) 89 | 90 | for i, _src in enumerate(self._config.sources()): 91 | if i > 0: 92 | self._log.warning('') 93 | self._log.warning('Next source ...') 94 | 95 | for current_repo in _src['repo']: 96 | request_url = self.join_package_path( 97 | _src['server'], 98 | 'api/storage', 99 | current_repo, 100 | '?list&deep=1&listFolders=0&mdTimestamps=1&includeRootPath=1', 101 | ) 102 | 103 | self._log.info('GET request: %s', request_url) 104 | 105 | r = requests.get( 106 | request_url, 107 | verify=False, 108 | auth=tuple(_src['auth']), 109 | ) 110 | 111 | r.raise_for_status() 112 | 113 | data_response = r.json() 114 | 115 | self.parse_dir_list(data_tree, data_response) 116 | 117 | for (d_name, d_version, d_branch,) in deps_list: 118 | if d_name not in data_tree: 119 | raise CrosspmException( 120 | CROSSPM_ERRORCODE_PACKAGE_NOT_FOUND, 121 | 'unknown package [{}]'.format(d_name), 122 | ) 123 | 124 | if d_branch not in data_tree[d_name]: 125 | raise CrosspmException( 126 | CROSSPM_ERRORCODE_PACKAGE_BRANCH_NOT_FOUND, 127 | 'unknown branch [{}] of package [{}]'.format( 128 | d_branch, 129 | d_name, 130 | )) 131 | 132 | libs = data_tree[d_name][d_branch] 133 | 134 | versions = [v for v in libs if fnmatch.fnmatch(v[0], '.'.join(d_version))] 135 | 136 | if not versions: 137 | raise CrosspmException( 138 | CROSSPM_ERRORCODE_VERSION_PATTERN_NOT_MATCH, 139 | 'pattern [{ver}] does not match any version of package [{pkg}] for branch [{br}] '.format( 140 | ver='.'.join(d_version), 141 | pkg=d_name, 142 | br=d_branch, 143 | )) 144 | 145 | current_version = max(versions, key=lambda x: x[1]) 146 | 147 | out_file_data_str += out_file_format.format( 148 | d_name, 149 | current_version[0], 150 | d_branch 151 | ) 152 | 153 | with open(out_file_path, 'w') as out_f: 154 | out_f.write(out_file_data_str) 155 | -------------------------------------------------------------------------------- /crosspm/helpers/python.py: -------------------------------------------------------------------------------- 1 | def get_object_from_string(object_path): 2 | """ 3 | Return python object from string 4 | :param object_path: e.g os.path.join 5 | :return: python object 6 | """ 7 | # split like crosspm.template.GUS => crosspm.template, GUS 8 | try: 9 | module_name, object_name = object_path.rsplit('.', maxsplit=1) 10 | module_ = __import__(module_name, globals(), locals(), ['App'], 0) 11 | variable_ = getattr(module_, object_name) 12 | except Exception: 13 | variable_ = None 14 | return variable_ 15 | -------------------------------------------------------------------------------- /crosspm/helpers/source.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # from crosspm.helpers.exceptions import * 3 | 4 | 5 | class Source: 6 | def __init__(self, adapter, parser, data): 7 | self._adapter = adapter 8 | self._parser = parser 9 | if 'repo' not in data: 10 | data['repo'] = '' 11 | if isinstance(data['repo'], str): 12 | data['repo'] = [data['repo']] 13 | self.args = {k: v for k, v in data.items() if k not in ['type', 'parser']} 14 | 15 | @property 16 | def repos(self): 17 | return self.args['repo'] 18 | 19 | def get_packages(self, downloader, list_or_file_path, property_validate=True): 20 | return self._adapter.get_packages(self, self._parser, downloader, list_or_file_path, property_validate) 21 | 22 | def get_usedby(self, downloader, list_or_file_path, property_validate=True): 23 | return self._adapter.get_usedby(self, self._parser, downloader, list_or_file_path, property_validate) 24 | 25 | def __getattr__(self, item): 26 | return self.args.get(item, None) 27 | 28 | def __getitem__(self, item): 29 | return self.args.get(item, None) 30 | -------------------------------------------------------------------------------- /crosspm/helpers/usedby.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from crosspm.helpers.locker import Locker 4 | 5 | 6 | class Usedby(Locker): 7 | def __init__(self, config, do_load, recursive): 8 | # Ignore do_load flag 9 | super(Usedby, self).__init__(config, False, recursive) 10 | 11 | def usedby_packages(self, deps_file_path=None, depslock_file_path=None, packages=None): 12 | """ 13 | Lock packages. Downloader search packages 14 | """ 15 | if deps_file_path is None: 16 | deps_file_path = self._deps_path 17 | if depslock_file_path is None: 18 | depslock_file_path = self._depslock_path 19 | if deps_file_path == depslock_file_path: 20 | depslock_file_path += '.lock' 21 | 22 | if packages is None: 23 | self.search_dependencies(deps_file_path) 24 | else: 25 | self._root_package.packages = packages 26 | self._log.info('Done!') 27 | 28 | def search_dependencies(self, depslock_file_path): 29 | self._log.info('Check dependencies ...') 30 | self._root_package.find_usedby(depslock_file_path, property_validate=True) 31 | self._log.info('') 32 | self._log.info('Dependency tree:') 33 | self._root_package.print(0, self._config.output('tree', [{self._config.name_column: 0}])) 34 | 35 | def entrypoint(self, *args, **kwargs): 36 | self.usedby_packages(*args, **kwargs) 37 | -------------------------------------------------------------------------------- /crosspm/template/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | 4 | template_dir = os.path.dirname(os.path.realpath(__file__)) 5 | GUS = os.path.join(template_dir, 'gus.j2') 6 | ALL_YAML = os.path.join(template_dir, 'all_yaml.j2') 7 | -------------------------------------------------------------------------------- /crosspm/template/all_yaml.j2: -------------------------------------------------------------------------------- 1 | {% for package in packages.values() %} 2 | - 'name': '{{package.name}}' 3 | 'packed_path': '{{package.packed_path}}' 4 | 'unpacked_path': '{{package.unpacked_path}}' 5 | 'duplicated': '{{package.unpacked_path}}' 6 | 'url': '{{package.pkg}}' 7 | 'md5': '{{package.md5}}' 8 | 'params': 9 | {% for name, value in package.get_params(merged=True).items() %} 10 | '{{name}}': '{{value}}' 11 | {% endfor %} 12 | {% endfor %} 13 | -------------------------------------------------------------------------------- /crosspm/template/gus.j2: -------------------------------------------------------------------------------- 1 | artifacts: 2 | {% for package in packages.values() %} 3 | - src_path : "{{package.pkg.path_in_repo}}" 4 | dest_path: "{{package.pkg.name}}" 5 | md5hash: "{{package.md5}}" 6 | version: "{{package.get_params(merged=True)['version']}}" 7 | src_repo_name: "{{package.pkg.repo}}" 8 | {% endfor %} 9 | -------------------------------------------------------------------------------- /docs/FAQ.md: -------------------------------------------------------------------------------- 1 | Frequently Asked Questions 2 | ========================== 3 | 4 | * [Frequently Asked Questions](#frequently-asked-questions) 5 | * [Как запретить рекурсивное выкачивание пакетов?](#как-запретить-рекурсивное-выкачивание-пакетов) 6 | * [При вызове crosspm в командной строке](#при-вызове-crosspm-в-командной-строке) 7 | * [Старый способ - через указание в конфиге](#старый-способ---через-указание-в-конфиге) 8 | * [CrossPM вылетает с ошибкой при поиске в репозитории с анонимным доступом (type: artifactory-aql)](#crosspm-вылетает-с-ошибкой-при-поиске-в-репозитории-с-анонимным-доступом-type-artifactory-aql) 9 | * [Как отфильтровать пакеты, чтобы выбирались только БЕЗ feature в версии?](#как-отфильтровать-пакеты-чтобы-выбирались-только-без-feature-в-версии) 10 | * [А как отфильтровать пакеты, чтобы выбирались ТОЛЬКО feature версии?](#а-как-отфильтровать-пакеты-чтобы-выбирались-только-feature-версии) 11 | * [Как отфильтровать пакеты по их свойствам?](#как-отфильтровать-пакеты-по-их-свойствам) 12 | * [Как скачать два пакеты с разной версией, но один именем?](#как-скачать-два-пакеты-с-разной-версией-но-один-именем) 13 | * [Один модуль ищется 5 минут, как ускорить поиск? (type: artifactory)](#один-модуль-ищется-5-минут-как-ускорить-поиск-type-artifactory) 14 | * [Как вынести пароль из файла конфигурации?](#как-вынести-пароль-из-файла-конфигурации) 15 | * [Как сделать проверку что все используемые пакеты имеют одинаковую версию? Например, что все используют последнюю версию openssl?](#как-сделать-проверку-что-все-используемые-пакеты-имеют-одинаковую-версию-например-что-все-используют-последнюю-версию-openssl) 16 | * [Хочу использовать кэш, но crosspm странно себя ведет - берет кэш пакета с другой версией?](#хочу-использовать-кэш-но-crosspm-странно-себя-ведет---берет-кэш-пакета-с-другой-версией) 17 | * [Не хочу создавать dependencies-файл, хочу указать сразу в команде crosspm список пакетов](#не-хочу-создавать-dependencies-файл-хочу-указать-сразу-в-команде-crosspm-список-пакетов) 18 | * [Как просто и безопасно передавать учётные данные в рамках crosspm.](#как-просто-и-безопасно-передавать-учётные-данные-в-рамках-crosspm) 19 | * [crosspm пишет все сообщения в stderr](#crosspm-пишет-все-сообщения-в-stderr) 20 | 21 | 22 | 23 | ## Как запретить рекурсивное выкачивание пакетов? 24 | 25 | ### При вызове crosspm в командной строке 26 | Используйте флаг `--recursive=False` при вызове `crosspm`. 27 | ```python 28 | # сделать lock-файл с пакетами только первого уровня, не рекурсивно 29 | crosspm lock # поведение по умолчанию - только первый уровень 30 | crosspm lock --recursive=False 31 | crosspm lock --recursive False 32 | # проверить правильность и наличие пакетов, рекурсивно 33 | crosspm lock --recursive 34 | crosspm lock --recursive=True 35 | 36 | # скачать без зависимостей, только пакеты указанные в dependencies 37 | crosspm download # поведение по умолчанию - только первый уровень 38 | crosspm download --recursive=False 39 | crosspm download --recursive False 40 | # скачать все дерево пакетов вместе с зависимостями 41 | crosspm download --recursive 42 | crosspm download --recursive=True 43 | ``` 44 | 45 | 46 | ### Старый способ - через указание в конфиге 47 | Можно задать неправильные имена файлов dependencies, тогда он не найдет их и не пойдет глубже 48 | ```yaml 49 | cpm: 50 | dependencies: no-dependencies.txt 51 | dependencies-lock: no-dependencies.txt.lock 52 | ``` 53 | И вызывать с указанием правильного файла: 54 | ```bash 55 | crosspm download --depslock-path=./dependencies.txt.lock 56 | ``` 57 | 58 | 59 | ## CrossPM вылетает с ошибкой при поиске в репозитории с анонимным доступом (type: artifactory-aql) 60 | Это ограничение запросов AQL в Artifactory - их возможно использовать только с авторизацией. Добавьте в конфиг любые учетные данные для доступа к этим репозиториям 61 | 62 | 63 | ## Как отфильтровать пакеты, чтобы выбирались только БЕЗ feature в версии? 64 | Если формат версии в конфиге определен так: 65 | ```yaml 66 | columns: 67 | version: "{int}.{int}.{int}[-{str}]" 68 | ``` 69 | то в dependencies можно указать, что необязательной части шаблона быть не должно. Для этого нужно добавить разделитель из необязательной части в конце маски версии. Несколько вариантов для примера: 70 | ``` 71 | PackageName *- R14.1 >=snapshot 72 | PackageName *.*.*- R14.1 >=snapshot 73 | PackageName 14.*.*- R14.1 >=snapshot 74 | ``` 75 | 76 | 77 | ## А как отфильтровать пакеты, чтобы выбирались ТОЛЬКО feature версии? 78 | Если формат версии в конфиге определен так: 79 | ```yaml 80 | columns: 81 | version: "{int}.{int}.{int}[-{str}]" 82 | ``` 83 | то в dependencies можно указать, что необязательная часть шаблона должна быть. Для этого нужно добавить эту необязательную часть в конце маски версии. В таком случае CrossPM не будет брать версии без feature. Несколько вариантов для примера: 84 | ``` 85 | Agent *-* R14.1 >=snapshot 86 | Agent *-develop R14.1 >=snapshot 87 | Agent *.*.*-* R14.1 >=snapshot 88 | Agent 14.*.*-* R14.1 >=snapshot 89 | Agent 14.*.*-CERT* R14.1 >=snapshot 90 | ``` 91 | 92 | 93 | ## Как отфильтровать пакеты по их свойствам? 94 | Например, формат версий и столбцы для файла dependencies в конфиге вы определили так (через точку с запятой в поле properties отделяются шаблоны для раздичных свойств артефакта, их может быть несколько): 95 | ```yaml 96 | columns: '*package, version, contract.db, contract.be' 97 | 98 | parsers: 99 | common: 100 | columns: 101 | version: '{int}.{int}.{int}[.{int}][-{str}]' 102 | 103 | repo: 104 | path: '{server}/{repo}/{package}_{version}_[all|amd64].deb' 105 | properties: 'contract.db={contract.db};contract.be={contract.be}' 106 | ``` 107 | Под контрактом мы понимаем любые теги в свойстве артефакта с именами, например, "contract.db" (контракт базы) и "contract.be" (контракт бекенда). Например, тут можно указать совместимость с другими артефактами по этому свойству - билд-контракты. Тогда CrossPM сможет искать пакеты из dependencies и фильтровать их по одному или всем свойствам, указанным в отдельных столбцах (свойств может быть больше одного): 108 | ``` 109 | # crosspm scheme (see crosspm.yaml): 110 | # packages version contract.db contract.be 111 | # --- DB: 112 | db-config 2.3.0.* hash1 rest2 113 | db-schema 2.4.12.* hash1 rest2 114 | # --- BE: 115 | backend 2.4.* hash1 rest5 116 | backend-doc 2.4.* hash2 rest5 117 | # --- UI: 118 | ui 2.* hash1 * 119 | documentation 2.* * * 120 | ``` 121 | Как обычно, звёздочка заменяет поиск по любому значению свойства. В данном абстрактном примере артефакты базы должны быть совместимы с артефактом UI по билд-контракту contract.db=hash1. Сам контракт может быть установлен для артефакта в момент сборки, либо проставлен вручную позже. 122 | 123 | 124 | ## Как скачать два пакета с разной версией, но одним именем? 125 | 126 | Нужно создать "псевдоуникальное имя пакета". Пример файла **dependencies.txt.lock** 127 | ``` 128 | PackageRelease17 Package 17.0.* 129 | PackageRelease161 Package 16.1.* 130 | OtherPackage OtherPackage * 131 | ``` 132 | И добавить в конфиг строки 133 | 134 | ```yaml 135 | columns: "*uniquename, package, version" # Указание в какой колонке указано имя пакета 136 | 137 | output: # Для упрощения дебага в stdout 138 | tree: 139 | - uniquename: 25 140 | - version: 0 141 | ``` 142 | 143 | 144 | 145 | ## Один модуль ищется 5 минут, как ускорить поиск? (type: artifactory) 146 | Данная проблема могла наблюдаться в адаптере **artifactory**, нужно поменять его на **artifactory-aql** 147 | ```yaml 148 | type: artifactory-aql 149 | ``` 150 | 151 | 152 | ## Как вынести пароль из файла конфигурации? 153 | С помощью [разделения на два файла конфигурации и использования import](config/IMPORT) 154 | 155 | 156 | 157 | ## Как сделать проверку что все используемые пакеты имеют одинаковую версию? Например, что все используют последнюю версию openssl? 158 | 159 | Если нужно проверить, что все пакеты используют одну версию пакетов, **но** загрузить только файлы, указанные непосредственно в **dependencies.txt**, можно поступить следующим образом: 160 | 1. [Разделить файлы конфигурации](config/IMPORT) на **crosspm.main.yaml**, **crosspm.download.yaml**, **crosspm.lock.yaml** 161 | 2. В **lock**-конфигурации указать существующий **dependencies.txt.lock** 162 | 3. В **download**-конфигурации указать НЕ существующий **no-dependencies.txt.lock** 163 | 4. Запустить crosspm с указанием разных конфигураций: 164 | 165 | ```bash 166 | # CrossPM: lock and recursive check packages 167 | # Попытается сделать lock-файл для пакетов из dep.txt, при этом проверит его зависимости на использование одной версии пакетов 168 | crosspm lock \ 169 | dependencies.txt dependencies.txt.lock \ 170 | --recursive \ 171 | --options cl="gcc-5.1",arch="x86_64",os="debian-8" \ 172 | --config=".\crosspm.lock.yaml" 173 | (( $? )) && exit 1 174 | 175 | # CrossPM: downloading packages 176 | # Скачает только пакеты, которые указаны в dependencies.txt 177 | crosspm download \ 178 | --config=".\crosspm.download.yaml" \ 179 | --lock-on-success \ 180 | --deps-path=".\dependencies.txt" \ 181 | --out-format="cmd" \ 182 | --output=".\klmn.cmd" \ 183 | --options cl="gcc-5.1",arch="x86_64",os="debian-8" \ 184 | (( $? )) && exit 1 185 | ``` 186 | 187 | 188 | ## Хочу использовать кэш, но crosspm странно себя ведет - берет кэш пакета с другой версией? 189 | На данный момент, если в пути до файла нет версии, например `repo/projectname/projectname/projectname.version.zip` 190 | То архив разархивируется в папку `PROJECTNAME`. 191 | В случае, когда выкачивается архив с новой версией `PROJECTNAME`, то он её не разархивирует, т.к. уже существует кэш с именем `PROJECTNAME`. 192 | 193 | Решение - использовать кастомные пути до распакованных\запакованных файлов: 194 | 195 | ```yaml 196 | cache: 197 | storage: 198 | packed: '{package}/{branch}/{version}/{compiler}/{arch}/{osname}/{package}.{version}.tar.gz' 199 | unpacked: '{package}/{branch}/{version}/{compiler}/{arch}/{osname}' 200 | ``` 201 | 202 | 203 | ## Не хочу создавать dependencies-файл, хочу указать сразу в команде crosspm список пакетов 204 | Для этого нужно запустить одну из команд: 205 | 206 | ```bash 207 | # download - без указания --depslock-path 208 | crosspm download --config config.yaml --dependencies-lock-content "boost 1.64.388 1.64-pm-icu" -o os=win,cl=vc140,arch=x86_64 209 | 210 | # lock - БЕЗ указания файлов в начале. Создает по итогам файл dependencies.txt.lock 211 | crosspm lock --config config.yaml --dependencies-content "boost 1.64.388 1.64-pm-icu" -o os=win,cl=vc140,arch=x86_64 212 | 213 | # usedby - БЕЗ указания файлов в начале 214 | crosspm usedby -c config.yaml --dependencies-content "packagename 1.2.* master" 215 | ``` 216 | 217 | 218 | ## Как просто и безопасно передавать учётные данные в рамках crosspm. 219 | 220 | Для этого нужно задать переменные в конфигурационном файле. 221 | 222 | Подробнее можно [почитать тут](usage/USAGE-CREDS.md) 223 | 224 | ## crosspm пишет все сообщения в stderr 225 | При выполнении команды `crosspm download` печатает сообщения в `stderr` (стандартный потом вывода ошибок). Из-за этого сложно понять правильно работает `crosspm` или там есть ошибки. 226 | 227 | `crosspm` ведет себя так для поддержки совместимости с cmakepm (старой версии `crosspm`). `Cmake` ищет в `stdout` пути до пакетов в последних строках - вида `PACKAGENAME_ROOT=e:\cache\package\1.2.123`. Поэтому все диагностические сообщения пишем в другой поток - `stderr`. 228 | 229 | Чтобы сделать поведение "как у других программ" - запустите `crosspm` с флагом `--stdout`. Тогда диагностические сообщения будут выводиться в `stdout`, а ошибки - в `stderr` 230 | ```bash 231 | crosspm download --stdout 232 | 233 | # ИЛИ можно задать переменную окружения CROSSPM_STDOUT с любым значением 234 | set CROSSPM_STDOUT=1 235 | # set CROSSPM_STDOUT=true 236 | crosspm download # аналогично --stdout 237 | ``` 238 | В конфигурационный файл задать этот параметр нельзя, т.к. логирование происходит до прочтения конфигурационного файла. 239 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | Cross Package Manager (CrossPM) 2 | ======= 3 | 4 | [![build](https://travis-ci.org/devopshq/crosspm.svg?branch=master)](https://travis-ci.org/devopshq/crosspm) 5 | [![codacy](https://api.codacy.com/project/badge/Grade/7a9ed2e6bb3e445f9e4a776e9b7f7886)](https://www.codacy.com/app/devopshq/crosspm/dashboard) 6 | [![pypi](https://img.shields.io/pypi/v/crosspm.svg)](https://pypi.python.org/pypi/crosspm) 7 | [![license](https://img.shields.io/pypi/l/crosspm.svg)](https://github.com/devopshq/crosspm/blob/master/LICENSE) 8 | 9 | 10 | * [Cross Package Manager (CrossPM)](#cross-package-manager-crosspm) 11 | * [Introduction](#introduction) 12 | * [Installation](#installation) 13 | * [Usage](#usage) 14 | * [Other pages](#other-pages) 15 | * [Development](#development) 16 | 17 | 18 | Introduction 19 | ------------ 20 | 21 | CrossPM (Cross Package Manager) is a universal extensible package manager. 22 | It lets you download and as a next step - manage packages of different types from different repositories. 23 | 24 | Out-of-the-box modules: 25 | 26 | - Adapters 27 | - Artifactory 28 | - [Artifactory-AQL](https://www.jfrog.com/confluence/display/RTF/Artifactory+Query+Language) (supported since artifactory 3.5.0): 29 | - files (simple repository on your local filesystem) 30 | 31 | - Package file formats 32 | - zip 33 | - tar.gz 34 | - nupkg (treats like simple zip archive for now) 35 | 36 | Modules planned to implement: 37 | 38 | - Adapters 39 | - git 40 | - smb 41 | - sftp/ftp 42 | 43 | - Package file formats 44 | - nupkg (nupkg dependencies support) 45 | - 7z 46 | 47 | We also need your feedback to let us know which repositories and package formats do you need, 48 | so we could plan its implementation. 49 | 50 | The biggest feature of CrossPM is flexibility. It is fully customizable, i.e. repository structure, package formats, 51 | packages version templates, etc. 52 | 53 | To handle all the power it have, you need to write configuration file (**crosspm.yaml**) 54 | and manifest file with the list of packages you need to download. 55 | 56 | Configuration file format is YAML, as you could see from its filename, so you free to use yaml hints and tricks, 57 | as long, as main configuration parameters remains on their levels :) 58 | 59 | CrossPM uses Python 3.4-3.6 60 | 61 | Installation 62 | ------------ 63 | To install CrossPM, simply: 64 | ``` 65 | pip install crosspm 66 | ``` 67 | 68 | Usage 69 | ----- 70 | To see commandline parameters help, run: 71 | ``` 72 | crosspm --help 73 | ``` 74 | 75 | You'll see something like this: 76 | ``` 77 | Usage: 78 | crosspm download [options] 79 | crosspm lock [DEPS] [DEPSLOCK] [options] 80 | crosspm usedby [DEPS] [options] 81 | crosspm pack [options] 82 | crosspm cache [size | age | clear [hard]] 83 | crosspm -h | --help 84 | crosspm --version 85 | 86 | Options: 87 | Output file. 88 | Source directory path. 89 | -h, --help Show this screen. 90 | --version Show version. 91 | -L, --list Do not load packages and its dependencies. Just show what's found. 92 | -v LEVEL, --verbose=LEVEL Set output verbosity: ({verb_level}) [default: ]. 93 | -l LOGFILE, --log=LOGFILE File name for log output. Log level is '{log_default}' if set when verbose doesn't. 94 | -c FILE, --config=FILE Path to configuration file. 95 | -o OPTIONS, --options OPTIONS Extra options. 96 | --deps-path=FILE Path to file with dependencies [./{deps_default}] 97 | --depslock-path=FILE Path to file with locked dependencies [./{deps_lock_default}] 98 | --dependencies-content=CONTENT Content for dependencies.txt file 99 | --dependencies-lock-content=CONTENT Content for dependencies.txt.lock file 100 | --lock-on-success Save file with locked dependencies next to original one if download succeeds 101 | --out-format=TYPE Output data format. Available formats:({out_format}) [default: {out_format_default}] 102 | --output=FILE Output file name (required if --out_format is not stdout) 103 | --output-template=FILE Template path, e.g. nuget.packages.config.j2 (required if --out_format=jinja) 104 | --no-fails Ignore fails config if possible. 105 | --recursive=VALUE Process all packages recursively to find and lock all dependencies 106 | --prefer-local Do not search package if exist in cache 107 | --stdout Print info and debug message to STDOUT, error to STDERR. Otherwise - all messages to STDERR 108 | ``` 109 | 110 | Other pages 111 | -------- 112 | - [Usage (RU)](usage/USAGE) 113 | - [CrossPM in Python as module](usage/USAGE-PYTHON) 114 | - [CrossPM in Cmake](usage/USAGE-CMAKE) 115 | - [CrossPM used by (RU)](usage/USAGE-USEDBY) 116 | - [CrossPM Config](config/CONFIG) 117 | - [Import (RU)](config/IMPORT) 118 | - [Environment variables (RU)](config/Environment-variables) 119 | - [cpmconfig - Centralized distribution of configuration files](cpmconfig) 120 | - [Output - get information about downloaded packages (RU)](config/OUTPUT) 121 | - [Output - template (RU)](config/output-template) 122 | - [FAQ (RU)](FAQ) 123 | - [Changelog](https://github.com/devopshq/crosspm/blob/master/CHANGELOG.md) 124 | 125 | Development 126 | -------- 127 | 128 | We use modified script for generate Table of content: https://github.com/ekalinin/github-markdown-toc 129 | ```bash 130 | # Prepare 131 | export GH_TOC_TOKEN="$(pwd)/token.txt" # or other path to token.txt with your github token 132 | BASE=$(pwd) # crosspm git folder 133 | 134 | # Process one file, eg FAQ.md 135 | cd ./docs 136 | $BASE/gh-md-toc --insert FAQ.md 137 | 138 | # Process all files 139 | bash toc.sh 140 | ``` 141 | -------------------------------------------------------------------------------- /docs/config/CONFIG.md: -------------------------------------------------------------------------------- 1 | CrossPM Config 2 | ======= 3 | 4 | 5 | * [CrossPM Config](#crosspm-config) 6 | * [import](#import) 7 | * [cpm](#cpm) 8 | * [cpm:dependencies and cpm:dependencies-lock](#cpmdependencies-and-cpmdependencies-lock) 9 | * [cpm:lock-on-success](#cpmlock-on-success) 10 | * [cpm: prefer-local](#cpm-prefer-local) 11 | * [cpm: recursive](#cpm-recursive) 12 | * [cache](#cache) 13 | * [cache:storage](#cachestorage) 14 | * [cache:clear - NOT used](#cacheclear---not-used) 15 | * [columns](#columns) 16 | * [parser column](#parser-column) 17 | * [repo column](#repo-column) 18 | * [values](#values) 19 | * [options](#options) 20 | * [parsers](#parsers) 21 | * [parsers:path](#parserspath) 22 | * [defaults](#defaults) 23 | * [solid:ext](#solidext) 24 | * [fails:unique](#failsunique) 25 | * [common](#common) 26 | * [sources](#sources) 27 | * [output:tree](#outputtree) 28 | * [Full example](#full-example) 29 | 30 | 31 | # import 32 | If defined, imports yaml config parts from other files. Must be the first parameter in config file. 33 | 34 | ```yaml 35 | import: 36 | - cred.yaml 37 | - template-config.yaml 38 | ``` 39 | Read more about [`import` hacks (RU)](OUTPUT) 40 | 41 | # `cpm` 42 | Main configuration such as manifest file name and cache path. 43 | 44 | ```yaml 45 | cpm: 46 | # Short description of your configuration file 47 | description: Simple example configuration 48 | 49 | dependencies: dependencies.txt 50 | dependencies-lock: dependencies.txt.lock 51 | lock-on-success: true 52 | prefer-local: True 53 | ``` 54 | 55 | ## `cpm:dependencies` and `cpm:dependencies-lock` 56 | - `cpm:dependencies` - manifest file name (not path - just filename) 57 | - `cpm:dependencies-lock` - manifest with locked dependencies (without masks and conditions) file name (not path - just filename). 58 | Equals to "dependencies" if not set. 59 | 60 | 61 | ## `cpm:lock-on-success` 62 | If set to `true` (or yes or 1) - dependencies lock file will be generated after successfull **download**. 63 | Lock file will be saved near original dependencies file. 64 | 65 | ## `cpm: prefer-local` 66 | Search in local cache before query repo. 67 | Work only with: 68 | - FIXED package version, without mask 69 | - FIXED extenstion 70 | - download-mode only (in lock it does not work) 71 | - only in artifactory-aql adapters 72 | 73 | ## `cpm: recursive` 74 | If set to `true` (or yes or 1) - crosspm will process all packages recursively to find and lock all dependencies. 75 | 76 | # `cache` 77 | Parameters for cache handling 78 | 79 | ```yaml 80 | cache: 81 | cmdline: cache 82 | env: CROSSPM_CACHE_ROOT 83 | # default: 84 | # path: 85 | storage: 86 | packed: '{package}/{branch}/{version}/{compiler}/{arch}/{osname}/{package}.{version}.tar.gz' 87 | unpacked: '{package}/{branch}/{version}/{compiler}/{arch}/{osname}' 88 | ``` 89 | Several way to set where crosspm stored files (archive and unpacked package) 90 | - `cache:cmdline` - Command line option name with path to cache folder. 91 | - `cache:env` - Environment variable name with path to cache folder. Used if command line option is not set 92 | - `cache:default` - Default path to cache folder. Used if command line option and environment variable are not set 93 | - `cache:path` - `cmdline`, `env` and `default` are ignored if `path` set. 94 | 95 | ## cache:storage 96 | Local storage setting, how `crosspm` will be stored files: 97 | - `storage:packed` - path to packed file 98 | - `storage:unpacked` - path to unpacked file 99 | 100 | ## `cache:clear` - NOT used 101 | 102 | ```yaml 103 | cache: 104 | ... 105 | # Parameters for cleaning cache. 106 | clear: 107 | # Delete files or folders older than "days". 108 | days: 10 109 | # Delete older files and folders if cache size is bigger than "size". Could be in b, Kb, Mb, Gb. Bytes (b) is a default. 110 | size: 300 mb 111 | # Call cache check and clear before download 112 | auto: true 113 | ``` 114 | 115 | # `columns` 116 | 117 | ```yaml 118 | columns: "*package, version, branch" 119 | ``` 120 | 121 | 122 | Let's keep in mind that any value we use in path, properties and columns description, called `column` in CrossPM. 123 | 124 | Manifest file columns definition. Asterisk here points to name column (column of manifest file with package name). CrossPM uses it for building list with unique packages (i.e. by package name) 125 | 126 | Некоторые колонки в `crosspm` имеют специальное значение 127 | 128 | ## parser column 129 | Чтобы явно задать какой пакет каким парсером искать - используйте `parser`-колонку в dependencies-файле. 130 | 131 | Этот подход позволяет ускорить поиск, если у вас много `parser`. 132 | 133 | config.yaml content: 134 | ```yaml 135 | # Часть пропущена 136 | columns: "*package, version, parser" 137 | parsers: 138 | release_packages: 139 | columns: 140 | version: '{int}.{int}.{int}' 141 | path: '{server}/{repo}/{name}/{name}.{version}.[msi|exe]' 142 | feature_packages: 143 | columns: 144 | version: '{int}.{int}.{int}' 145 | path: '{server}/{repo}/{name}/{name}.{version}.[msi|exe]' 146 | ``` 147 | 148 | dependencies.txt content: 149 | ``` 150 | # package version parser 151 | package1 1.0.* release_packages 152 | package2 1.0.* release_packages 153 | 154 | # Указывайте несколько через запятую, без пробела: 155 | package2 1.0.* feature_packages,release_packages 156 | 157 | # Варианты для поиска по всем парсерам: *, -, пустая колонка 158 | package3 1.0.* - 159 | package3 1.0.* * 160 | package3 1.0.* 161 | ``` 162 | 163 | ## repo column 164 | Можно указать в каких репозиториях искать пакет - используйте колонку `repo`. Все используемые `repo` должны быть явно заданы в [sources](#sources). 165 | 166 | Указание `repo` не ускоряет поиск, т.к. запрос идет по определенным в `sources` репозиториям. Но позволяет точнее определить пакет. 167 | 168 | `config.yaml` content: 169 | ```yaml 170 | columns: "*package, version, repo" 171 | sources: 172 | - repo: repo.snapshot 173 | parser: artifactory-parser 174 | - repo: repo.release 175 | parser: artifactory-parser 176 | ``` 177 | 178 | `dependencies.txt` content: 179 | ``` 180 | # package version repo 181 | package1 1.0.* repo.snapshot 182 | package1 1.0.* repo.release 183 | 184 | # Указывайте несколько через запятую, без пробела: 185 | package1 1.0.* repo.snapshot,repo.release 186 | 187 | # Поиск во всех репозиториях: 188 | package1 1.0.* * 189 | package1 1.0.* - 190 | ``` 191 | 192 | # `values` 193 | Lists or dicts of available values for some columns (if we need it). 194 | ```yaml 195 | values: 196 | quality: 197 | 1: banned 198 | 2: snapshot 199 | 3: integration 200 | 4: stable 201 | 5: release 202 | ``` 203 | 204 | # `options` 205 | Here we can define commandline options and environment variable names from which we will get some of columns values. We can define default values for those columns here too. Each option must be configured with this parameters: 206 | - `cmdline` - Command line option name with option's value 207 | - `env` - Environment variable name with option's value. Used if command line option is not set 208 | - `default` - Default option's value. Used if command line option and environment variable are not set 209 | 210 | # `parsers` 211 | Rules for parsing columns, paths, properties, etc. 212 | ```yaml 213 | parsers: 214 | common: 215 | columns: 216 | version: "{int}.{int}.{int}[.{int}][-{str}]" 217 | sort: 218 | - version 219 | - '*' 220 | index: -1 221 | usedby: 222 | AQL: 223 | "@dd.{package}.version": "{version}" 224 | "@dd.{package}.operator": "=" 225 | 226 | property-parser: 227 | "deb.name": "package" 228 | "deb.version": "version" 229 | "qaverdict": "qaverdict" 230 | 231 | artifactory: 232 | path: "{server}/{repo}/{package}/{branch}/{version}/{compiler|any}/{arch|any}/{osname}/{package}.{version}[.zip|.tar.gz|.nupkg]" 233 | properties: "some.org.quality = {quality}" 234 | ``` 235 | - `columns` - Dictionary with column name as a key and template as a value. 236 | Means that version column contains three numeric parts divided by a dot, followed by numeric or string or numeric and string parts with dividers or nothing at all: 237 | ```yaml 238 | version: "{int}.{int}.{int}[.{int}][-{str}]" 239 | ``` 240 | - `sort` - List of column names in sorting order. Used for sorting packages if more than one version found 241 | for defined parameters. Asterisk can be one of values of a list representing all columns not mentioned here. 242 | - `index` - Used for picking one element from sorted list. It's just a list index as in python. 243 | - `properties` - Extra properties. i.e. object properties in Artifactory. 244 | 245 | ## `parsers:path` 246 | Path template for searching packages in repository. Here {} is column, [|] is variation. 247 | 248 | ```yaml 249 | path: "{server}/{repo}/{package}/{compiler|any}/{osname}/{package}.{version}[.zip|.tar.gz] 250 | ``` 251 | these paths will be searched: 252 | 253 | ```yaml 254 | https://repo.some.org/artifactory/libs-release.snapshot/boost/gcc4/linux/boost.1.60.204.zip 255 | https://repo.some.org/artifactory/libs-release.snapshot/boost/gcc4/linux/boost.1.60.204.tar.gz 256 | https://repo.some.org/artifactory/libs-release.snapshot/boost/any/linux/boost.1.60.204.zip 257 | https://repo.some.org/artifactory/libs-release.snapshot/boost/any/linux/boost.1.60.204.tar.gz 258 | ``` 259 | 260 | Для того, чтобы задать любой путь, можно использовать `*`. Это пригодится, когда вам нужно выкачать пакеты, в оригинале не обращающие внимания на пути (`nupkg`, `deb`) 261 | ```yaml 262 | # deb 263 | parsers: 264 | common: 265 | columns: 266 | version: '{int}.{int}.{int}[.{int}]' 267 | artifactory-deb: 268 | path: '{server}/{repo}/pool/*/{package}.{version}.deb' 269 | 270 | # nupkg 271 | parsers: 272 | common: 273 | columns: 274 | version: '{int}.{int}.{int}[.{int}][-{str}]' 275 | artifactory-nupkg: 276 | path: '{server}/{repo}/pool/*/{package}.{version}.nupkg' 277 | ``` 278 | 279 | 280 | # `defaults` 281 | Default values for columns not defined in "options". 282 | ```yaml 283 | defaults: 284 | branch: master 285 | quality: stable 286 | # default support python format, like this: 287 | otherparams: "{package}/{version}" 288 | ``` 289 | 290 | # `solid:ext` 291 | Set of rules pointing to packages which doesn't need to be unpacked. File name extension (i.e. ".tgz", ".tar.gz", or more real example ".deb"). 292 | ```yaml 293 | solid: 294 | ext: *.deb 295 | ``` 296 | 297 | # `fails:unique` 298 | Here we can define some rules for failing CrossPM jobs. `fails:unique` - List of columns for generating unique index. 299 | ```yaml 300 | fails: 301 | unique: 302 | - package 303 | - version 304 | ``` 305 | 306 | 307 | # `common` 308 | Common parameters for all or several of sources. 309 | 310 | ```yaml 311 | common: 312 | server: https://repo.some.org/artifactory 313 | parser: artifactory 314 | type: jfrog-artifactory 315 | auth_type: simple 316 | auth: 317 | - username 318 | - password 319 | ``` 320 | 321 | # `sources` 322 | 323 | Sources definition. Here we define parameters for repositories access. 324 | - `type` - Source type. Available types list depends on existing adapter modules. 325 | - `parser` - Available parsers defined in parsers. 326 | - `server` - Root URL of repository server. 327 | - `repo` - Subpath to specific part of repository on server. 328 | - `auth_type` - Authorization type. For example "simple". 329 | - `auth` - Authorization data. For "simple" here we define login and password. 330 | 331 | 332 | ```yaml 333 | sources: 334 | - repo: 335 | - libs-release.snapshot 336 | - libs-release/extlibs 337 | - type: jfrog-artifactory 338 | parser: artifactory 339 | server: https://repo.some.org/artifactory 340 | repo: project.snapshot/temp-packages 341 | auth_type: simple 342 | auth: 343 | - username2 344 | - password2 345 | ``` 346 | 347 | # `output:tree` 348 | Report output format definition. `tree` - columns and widths for tree output, printed in the end of CrossPM job. 349 | 350 | ```yaml 351 | output: 352 | tree: 353 | - package: 25 354 | - version: 0 355 | ``` 356 | 357 | 358 | # Full example 359 | We'll add some more examples soon. Here is one of configuration file examples for now. 360 | 361 | **crosspm.yaml** 362 | ```yaml 363 | import: 364 | - cred.yaml 365 | 366 | cpm: 367 | description: Simple example configuration 368 | dependencies: dependencies.txt 369 | dependencies-lock: dependencies.txt.lock 370 | lock-on-success: true 371 | 372 | # search in local cache before query repo. 373 | # Work only with: 374 | # - FIXED package version, without mask 375 | # - FIXED extenstion 376 | # - download-mode only (in lock it does not work) 377 | # - only in artifactory-aql adapters 378 | prefer-local: True 379 | 380 | cache: 381 | cmdline: cache 382 | env: CROSSPM_CACHE_ROOT 383 | storage: 384 | packed: '{package}/{branch}/{version}/{compiler}/{arch}/{osname}/{package}.{version}.tar.gz' 385 | unpacked: '{package}/{branch}/{version}/{compiler}/{arch}/{osname}' 386 | 387 | columns: "*package, version, branch" 388 | 389 | values: 390 | quality: 391 | 1: banned 392 | 2: snapshot 393 | 3: integration 394 | 4: stable 395 | 5: release 396 | 397 | options: 398 | compiler: 399 | cmdline: cl 400 | env: CROSSPM_COMPILER 401 | default: vc110 402 | 403 | arch: 404 | cmdline: arch 405 | env: CROSSPM_ARCH 406 | default: x86 407 | 408 | osname: 409 | cmdline: os 410 | env: CROSSPM_OS 411 | default: win 412 | 413 | parsers: 414 | common: 415 | columns: 416 | version: "{int}.{int}.{int}[.{int}][-{str}]" 417 | sort: 418 | - version 419 | - '*' 420 | index: -1 421 | usedby: 422 | AQL: 423 | "@dd.{package}.version": "{version}" 424 | "@dd.{package}.operator": "=" 425 | 426 | property-parser: 427 | "deb.name": "package" 428 | "deb.version": "version" 429 | "qaverdict": "qaverdict" 430 | 431 | artifactory: 432 | path: "{server}/{repo}/{package}/{branch}/{version}/{compiler|any}/{arch|any}/{osname}/{package}.{version}[.zip|.tar.gz|.nupkg]" 433 | properties: "some.org.quality = {quality}" 434 | 435 | defaults: 436 | branch: master 437 | quality: stable 438 | # default support python format, like this: 439 | otherparams: "{package}/{version}" 440 | 441 | solid: 442 | ext: *.deb 443 | 444 | fails: 445 | unique: 446 | - package 447 | - version 448 | 449 | common: 450 | server: https://repo.some.org/artifactory 451 | parser: artifactory 452 | type: jfrog-artifactory 453 | auth_type: simple 454 | auth: 455 | - username 456 | - password 457 | 458 | sources: 459 | - repo: 460 | - libs-release.snapshot 461 | - libs-release/extlibs 462 | 463 | - type: jfrog-artifactory 464 | parser: artifactory 465 | server: https://repo.some.org/artifactory 466 | repo: project.snapshot/temp-packages 467 | auth_type: simple 468 | auth: 469 | - username2 470 | - password2 471 | 472 | output: 473 | tree: 474 | - package: 25 475 | - version: 0 476 | ``` 477 | -------------------------------------------------------------------------------- /docs/config/Environment-variables.md: -------------------------------------------------------------------------------- 1 | Переменные окружения 2 | ======= 3 | 4 | * [Переменные окружения](#переменные-окружения) 5 | * [CROSSPM_CONFIG_PATH](#crosspm_config_path) 6 | 7 | 8 | Есть только одна переменная окружения с фиксированным именем (остальные задаются через файл конфигурации) 9 | ### CROSSPM_CONFIG_PATH 10 | - Значение этой переменной указывает на расположение файла конфигурации. Если в переменной указан полный путь вместе с именем файла - берется этот файл, если другой не задан в параметрах командной строки. Если в переменной указан только путь без имени файла, то по указанному пути ищется файл **crosspm.yaml** 11 | - По указанному в этой переменной пути, также происходит поиск импортируемых файлов конфигурации, указанных в разделе import основного файла конфигурации. 12 | - В этой переменной может быть указано несколько путей, разделенных знаком ";". В таком случае, если одно из этих значений - полный путь с именем файла, то crosspm попытается использовать именно его. 13 | - При указании полного пути с именем файла, нет необходимости добавлять этот путь еще раз для поиска импортируемых файлов - crosspm сделает это сам. 14 | - Глобальный файл конфигурации так же ищется по путям, указанным в этой переменной. Его имя файла должно быть одним из двух вариантов: **crosspm_global.yaml** или **global.yaml** 15 | -------------------------------------------------------------------------------- /docs/config/IMPORT.md: -------------------------------------------------------------------------------- 1 | Import 2 | ======= 3 | 4 | * [Import](#import) 5 | * [Вынесение паролей из файла конфигурации](#вынесение-паролей-из-файла-конфигурации) 6 | * [Использование почти похожих файлов-конфигураций](#использование-почти-похожих-файлов-конфигураций) 7 | 8 | 9 | Файл конфигурации возможно разделить на несколько и импортировать недостающие части. Приведём примеры, когда это необходимо. 10 | 11 | Механизм **import** является внутренней реализацией CrossPM, но в остальном поддерживается обычный yaml-формат. [Подробнее про yaml и anchor, которые используются на этой странице](https://learnxinyminutes.com/docs/yaml/) 12 | 13 | #### Вынесение паролей из файла конфигурации 14 | Чтобы не сохранять чувствительную информацию (логин\пароль от хранилища артефактов) в VCS, но иметь возможность использовать CrossPM во время сборки на сервере Continious Integration, нужно сделать следующее: 15 | 16 | Создать файл **cred.yaml** со следующим содержимым: 17 | ```yaml 18 | .aliases: 19 | - &auth_name1 20 | - myusername1 21 | - myp@ssw0rd 22 | - &auth_name2 23 | - myusername2 24 | - myp@ssw0rd 25 | ``` 26 | 27 | Создать файл конфигурации **crosspm.yaml**, где импортировать указанный выше файл и использовать определенный **anchor**: 28 | ```yaml 29 | # Пропущена часть конфигурации, которая нас не интересует 30 | # -------------- 31 | sources: 32 | - repo: libs-cpp-snapshot 33 | parser: crosspackage 34 | auth: *auth_name1 35 | - repo: nuget-repo 36 | parser: nuger 37 | auth: *auth_name2 38 | # -------------- 39 | # Пропущена часть конфигурации, которая нас не интересует 40 | ``` 41 | 42 | Теперь добавляем в **.gitignore** файл **cred.yaml** и коммитим **crosspm.yaml**. 43 | 44 | Добавляем в сервер CI пред-сборочный шаг по созданию **cred.yaml** из секретных переменных (реализация зависит от используемого CI-сервера) 45 | 46 | #### Использование почти похожих файлов-конфигураций 47 | Бывают случаи, когда вам необходимо создать примерно два одинаковых файл-конфигурации, но с отличающимися полями. Для примера - возмём два разных используемых имени **dependencies-файла** 48 | 49 | Создаём **crosspm.main.yaml**: 50 | ```yaml 51 | # cpm: 52 | # dependencies: dependencies 53 | # dependencies-lock: dependencies 54 | # prefer-local: True 55 | 56 | # -------------- 57 | # Тут определён файл весь конфигурации, кроме секции cpm 58 | # Так же допустимо использовать yaml-anchor для чувствительной информации 59 | ``` 60 | 61 | Создаём **crosspm.dependencies.yaml** 62 | ```yaml 63 | import: 64 | - cred.yaml 65 | - crosspm.main.yaml 66 | 67 | cpm: 68 | dependencies: dependencies.txt 69 | dependencies-lock: dependencies.txt.lock 70 | prefer-local: True 71 | ``` 72 | 73 | Создаём **crosspm.no-dependencies.yaml** 74 | ```yaml 75 | import: 76 | - cred.yaml 77 | - crosspm.main.yaml 78 | 79 | cpm: 80 | dependencies: no-dependencies.txt 81 | dependencies-lock: no-dependencies.txt.lock 82 | prefer-local: True 83 | ``` 84 | 85 | Запускаем **crosspm** с указанием используемого файла конфигурации: 86 | ```bash 87 | crosspm download ^ 88 | --options cl="vc140",arch="x86_64",os="win" ^ 89 | --config=".\crosspm.dependencies.yaml" 90 | 91 | crosspm download ^ 92 | --options cl="vc140",arch="x86_64",os="win" ^ 93 | --config=".\crosspm.no-dependencies.yaml" 94 | ``` 95 | -------------------------------------------------------------------------------- /docs/config/OUTPUT.md: -------------------------------------------------------------------------------- 1 | Output 2 | ====== 3 | 4 | * [Output](#output) 5 | * [shell](#shell) 6 | * [cmd](#cmd) 7 | * [json](#json) 8 | * [python](#python) 9 | * [stdout](#stdout) 10 | * [lock](#lock) 11 | 12 | 13 | Как скачать пакеты понятно, как теперь их дальше использовать? Для этого в **crosspm** есть несколько видов **output**. 14 | 15 | Далее будем указывать варианты вызова из командой строки, но такие же варианты допустимо указывать в [файле конфигурации](CONFIG), в секции **output** 16 | 17 | Возможно так же создание файла с большим количеством данных, смотри подробнее [output-template](output-template) 18 | 19 | ## shell 20 | Для использования в linux можно делать вывод формата **shell**: 21 | ```bash 22 | crosspm download \ 23 | --config="./crosspm.yaml" \ 24 | --out-format="shell" \ 25 | --output="./paths.sh" 26 | (( $? )) && exit 1 27 | 28 | cat ./paths.sh 29 | 30 | source './paths.sh' 31 | 32 | # Далее будут доступны переменные вида PACKAGE_ROOT, где PACKAGE - имя пакет в dependencies.txt.lock 33 | echo $PACKAGENAME1_ROOT 34 | echo $PACKAGENAME2_ROOT 35 | python $PACKANAME1_ROOT/somepath/inside/file.py 36 | ``` 37 | 38 | Контент файла **paths.sh**: 39 | ```bash 40 | PACKAGENAME1_ROOT=/crosspm/cache/pachagename1/branch/version 41 | PACKAGENAME2_ROOT=/crosspm/cache/pachagename2/branch/version 42 | ``` 43 | 44 | ## cmd 45 | Для использования в Windows: 46 | ```bash 47 | crosspm download ^ 48 | --config=".\crosspm.yaml" ^ 49 | --out-format="cmd" ^ 50 | --output=".\paths.cmd" ^ 51 | IF %ERRORLEVEL% NEQ 0 GOTO badexit 52 | 53 | CALL ".\paths.cmd" 54 | # Далее будут доступны переменные вида PACKAGE_ROOT, где PACKAGE - имя пакет в dependencies.txt.lock 55 | "%SOMEPACKAGE_ROOT%\converter.exe" ^ 56 | --rules="%OTHERPACKAGE_ROOT\rules" 57 | ``` 58 | Контент файла **paths.cmd**: 59 | ```bash 60 | set SOMEPACKAGE_ROOT=E:\crosspm\cache\packagename2\branch\version 61 | set OTHERPACKAGE_ROOT=E:\crosspm\cache\otherpackage\branch\version 62 | ``` 63 | 64 | 65 | ## json 66 | ```bash 67 | crosspm download \ 68 | --config="./crosspm.yaml" \ 69 | --out-format="json" \ 70 | --output="./paths.json" 71 | (( $? )) && exit 1 72 | 73 | cat ./paths.sh 74 | ``` 75 | 76 | Контент файла **paths.json**: 77 | ```json 78 | { 79 | "PACKAGES_ROOT": { 80 | "OPENSSL_ROOT": "/test/path/package2", 81 | "BOOST_ROOT": "/test/path/package11" 82 | } 83 | } 84 | ``` 85 | 86 | 87 | ## python 88 | [Лучшая практика использования в python](../usage/USAGE-PYTHON) 89 | 90 | ## stdout 91 | Используется для вывода в stdout, смотри тесты 92 | 93 | ## lock 94 | Создаёт lock-файл, используется для `crosspm lock` команды 95 | -------------------------------------------------------------------------------- /docs/config/output-template.md: -------------------------------------------------------------------------------- 1 | Output template 2 | =============== 3 | 4 | * [Output template](#output-template) 5 | * [Доступные встроенные шаблоны](#доступные-встроенные-шаблоны) 6 | * [crosspm.template.ALL_YAML](#crosspmtemplateall_yaml) 7 | * [crosspm.template.GUS - Global Update Server](#crosspmtemplategus---global-update-server) 8 | * [Вызов для своего шаблона](#вызов-для-своего-шаблона) 9 | * [Распространение шаблонов через pypi](#распространение-шаблонов-через-pypi) 10 | 11 | 12 | `crosspm` умеет выводить информацию о скачанных\найденных пакетов в разные форматы, с помощью `jinja2`-шаблонов 13 | 14 | Генерация шаблонов доступна в командах `lock` и `download` 15 | 16 | 17 | ## Доступные встроенные шаблоны 18 | ### crosspm.template.ALL_YAML 19 | Можно вывести всю инфомацию о пакете в `yaml`-файл по формату ниже. 20 | 21 | Для этого нужно вызывать: 22 | ```bash 23 | crosspm download --deps-path dependencies.txt --config config.yaml --out-format jinja --output yaml.yaml --output-template crosspm.template.ALL_YAML 24 | ``` 25 | 26 | ```yaml 27 | - 'name': 'boost' 28 | 'packed_path': 'C:\crosspm\archive\boost\1.64-pm-icu\1.64.388\vc140\x86_64\win\boost.1.64.388.tar.gz' 29 | 'unpacked_path': 'C:\crosspm\cache\boost\1.64-pm-icu\1.64.388\vc140\x86_64\win' 30 | 'duplicated': 'C:\crosspm\cache\boost\1.64-pm-icu\1.64.388\vc140\x86_64\win' 31 | 'url': 'https://repo.artifactory.com/artifactory/libs-cpp-release.snapshot/boost/1.64-pm-icu/1.64.388/vc140/x86_64/win/boost.1.64.388.tar.gz' 32 | 'md5': 'c3140673778ba01eed812304da3a5af9' 33 | 'params': 34 | 'osname': 'win' 35 | 'branch': '1.64-pm-icu' 36 | 'package': 'boost' 37 | 'server': 'https://repo.artifactory.com/artifactory' 38 | 'compiler': 'vc140' 39 | 'filename': 'boost.1.64.388.tar.gz' 40 | 'version': '1.64.388' 41 | 'repo': 'libs-cpp-release.snapshot' 42 | 'arch': 'x86_64' 43 | - 'name': 'log4cplus' 44 | 'packed_path': 'C:\crosspm\archive\log4cplus\1.2.0-pm\1.2.0.149\vc140\x86_64\win\log4cplus.1.2.0.149.tar.gz' 45 | ... 46 | ``` 47 | ### crosspm.template.GUS - Global Update Server 48 | Можно вызвать crosspm, чтобы он нашел нужные пакеты и сгенерировал `GUS` manifest.yaml. 49 | 50 | ```bash 51 | crosspm download --deps-path dependencies.txt --config config.yaml --out-format jinja --output yaml.yaml --output-template crosspm.template.GUS 52 | ``` 53 | 54 | ```yaml 55 | artifacts: 56 | - src_path : "/boost/1.64-pm-icu/1.64.388/vc140/x86_64/win/boost.1.64.388.tar.gz" 57 | dest_path: "boost.1.64.388.tar.gz" 58 | md5hash: "c3140673778ba01eed812304da3a5af9" 59 | version: "1.64.388" 60 | src_repo_name: "libs-cpp-release.snapshot" 61 | - src_path : "/log4cplus/1.2.0-pm/1.2.0.149/vc140/x86_64/win/log4cplus.1.2.0.149.tar.gz" 62 | dest_path: "log4cplus.1.2.0.149.tar.gz" 63 | md5hash: "90bb506b44c4e388c565a0af435f8c71" 64 | version: "1.2.0.149" 65 | src_repo_name: "libs-cpp-release.snapshot" 66 | - src_path : "/poco/1.7.9-pm/1.7.9.315/vc140/x86_64/win/poco.1.7.9.315.tar.gz" 67 | dest_path: "poco.1.7.9.315.tar.gz" 68 | md5hash: "1e9c92f79df6cfd143c7d99c3d7cc868" 69 | version: "1.7.9.315" 70 | src_repo_name: "libs-cpp-release.snapshot" 71 | ``` 72 | 73 | ## Вызов для своего шаблона 74 | Можно сделать любой свой шаблон и генерировать на выход файлы 75 | 76 | В шаблон передаётся переменная `packages`, которая содержит верхний уровень всех пакетов (подробнее смотри [usage python](../usage/USAGE-PYTHON)) 77 | 78 | Например, мы хотим получить следующий формат для передачи в скрипт установки `deb`-пакетов 79 | 80 | Для этого, мы создаём файл `deb.j2` со следующим содержимым 81 | 82 | ``` 83 | {% for package in packages.values() %} 84 | {{package.pkg.name}} = {{package.get_params(merged=True)['version']}} 85 | {% endfor %} 86 | ``` 87 | 88 | Вызываем `crosspm`: 89 | 90 | ```bash 91 | download --deps-path dependencies.txt --out-format jinja --output depends.txt --output-template /path/to/template/deb.j2 92 | ``` 93 | 94 | Получаем на выходе: 95 | 96 | ``` 97 | package-name1 = 1.2.123 98 | package-name2 = 1.2.111 99 | ``` 100 | 101 | 102 | ## Распространение шаблонов через pypi 103 | Допустимо шаблоны распространять через `pypi`-пакеты (смотри подробнее [cpmconfig](../cpmconfig)) 104 | 105 | Для этого, нужно вызвать `crosspm` со следующими параметрами 106 | 107 | ```bash 108 | download --deps-path dependencies.txt --out-format jinja --output depends.txt --output-template module1.module2.VARIABLE_NAME 109 | ``` 110 | 111 | где: 112 | - `module1.module2` - имя модулей (с файлами `__init__.py`) 113 | - `VARIABLE_NAME` - имя переменной, в которой хранится полный (или относительный текущей директории) путь до jinja-шаблона 114 | 115 | Т.е. в обычном python-файле у вас должно работать следующее: 116 | ```python 117 | from module1.module2 import VARIABLE_NAME 118 | print(VARIABLE_NAME) 119 | #C:\Python34\Lib\site-packages\module1\module2\my_template_name.j2 120 | ``` 121 | 122 | Пример распространения и настройки смотри в `crosspm.template` 123 | -------------------------------------------------------------------------------- /docs/cpmconfig.md: -------------------------------------------------------------------------------- 1 | В разработке... 2 | -------------------------------------------------------------------------------- /docs/usage/USAGE-CMAKE.md: -------------------------------------------------------------------------------- 1 | 2 | * [Usage](#usage) 3 | 4 | Usage 5 | ======= 6 | You can use Crosspm in cmake. It will help to download the libraries recursively and connect them to the project. 7 | The root of the project must contain the dependencies.txt.lock with libraries to download from the repository. 8 | An example of dependencies.txt.lock: 9 | 10 | ```python 11 | zlib 1.2.8.199 1.2.8-pm 12 | ``` 13 | Place a file crosspm.cmake to your project. The file is available after downloading crosspm. 14 | crosspm.cmake downloads the libraries specified in dependencies.txt.lock and then looks there for the _pm_package file, 15 | which contains the import logic of a specific library into the calling project. 16 | An example of _pm_package.cmake: 17 | 18 | ```shell 19 | 20 | function(pm_add_lib ZLIB_ROOT) 21 | message("ZLIB_ROOT = ${ZLIB_ROOT}") 22 | 23 | if(NOT TARGET ZLIB) 24 | set(TARGET_IMPORTED_NAME ZLIB) 25 | add_library(${TARGET_IMPORTED_NAME} STATIC IMPORTED GLOBAL) 26 | set_property(TARGET ${TARGET_IMPORTED_NAME} PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${ZLIB_ROOT}/include") 27 | 28 | if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANGXX) 29 | if(UNIX) 30 | set_property(TARGET ${TARGET_IMPORTED_NAME} PROPERTY IMPORTED_LOCATION "${ZLIB_ROOT}/lib/libz.a") 31 | elseif(WIN32) 32 | set_property(TARGET ${TARGET_IMPORTED_NAME} PROPERTY IMPORTED_LOCATION "${ZLIB_ROOT}/lib/libzlibstatic.a") 33 | endif() 34 | elseif(MSVC) 35 | set_property(TARGET ${TARGET_IMPORTED_NAME} PROPERTY IMPORTED_LOCATION_DEBUG "${ZLIB_ROOT}/lib/zlibstaticd.lib") 36 | set_property(TARGET ${TARGET_IMPORTED_NAME} PROPERTY IMPORTED_LOCATION_RELEASE "${ZLIB_ROOT}/lib/zlibstatic.lib") 37 | endif() 38 | endif() 39 | endfunction() 40 | 41 | ``` 42 | 43 | You just have to add in CMakeLists.txt: 44 | 45 | ```python 46 | include(crosspm) 47 | pm_download_dependencies() 48 | ... 49 | target_link_libraries( 50 | ZLIB 51 | ) 52 | ``` 53 | -------------------------------------------------------------------------------- /docs/usage/USAGE-CREDS.md: -------------------------------------------------------------------------------- 1 | 2 | * [Варианты использования учётных данных для подключения к artifactory](#варианты-использования-учётных-данных-для-подключения-к-artifactory) 3 | * [Примеры использования.](#примеры-использования) 4 | 5 | ## Варианты использования учётных данных для подключения к artifactory 6 | 7 | Для использования необходимо указать в конфигурационном файле следующие параметры. 8 | В разделе `options` укажите блок `auth` и/или блоки `user`, `password` как показано ниже: 9 | ```yaml 10 | options: 11 | <... пропущен ненужный блок ...> 12 | # можно указать все блоки, а использовать или auth или user, password 13 | auth: 14 | cmdline: auth_libs1 15 | env: DOWNLOAD_CI_AUTH 16 | secret: true 17 | 18 | user: 19 | cmdline: user 20 | env: DOWNLOAD_CI_USER 21 | secret: false 22 | 23 | password: 24 | cmdline: password 25 | env: DOWNLOAD_CI_PASSWORD 26 | secret: true 27 | ``` 28 | 29 | - `user`, `password`, `auth` - произвольные названия переменных для авторизации. 30 | - `cmdline` - название аргумента при задании переменной с помощью коммандной строки. 31 | - `env` - название системной переменной для использования. Если переменная, указанная в cmdline не задана при запуске, 32 | но задана системная переменная, то будет использована значение системной переменной. 33 | - `secret` - отображать или нет в лог значение переменной. Если поставить значение `true`, 34 | то параметр не будет выводиться в лог. Для примера выше значения `auth` и `password` не будут видны в логе, а значение `user` будет: 35 | ``` 36 | boost: {'version': ['*', '*', '*', None, None], 'branch': '1.64-pm-icu', 'user': 'env_test_user', 37 | 'compiler': 'vc110', 'arch': 'x86', 'osname': 'win', 'user1': None, 'user2': None, 38 | 'server': 'https://repo.example.com/artifactory'} 39 | ``` 40 | 41 | После того как переменные для авторизации указаны в разделе `options` можно использовать их в разделах `common` или `sourses`. 42 | Переменные задаются по имени в скобках `'{имяпеременной}'`. 43 | Варианты того как можно задать переменные: 44 | ```yaml 45 | common: 46 | server: https://repo.example.com/artifactory 47 | parser: artifactory 48 | type: jfrog-artifactory-aql 49 | auth_type: simple 50 | auth: '{auth}' 51 | 52 | sources: 53 | - repo: 54 | - some-repo.snapshot 55 | ``` 56 | 57 | ```yaml 58 | common: 59 | server: https://repo.example.com/artifactory 60 | parser: artifactory 61 | type: jfrog-artifactory-aql 62 | auth_type: simple 63 | 64 | sources: 65 | - repo: 66 | - another-repo.snapshot 67 | auth: '{user}:{password}' 68 | ``` 69 | 70 | ```yaml 71 | common: 72 | server: https://repo.example.com/artifactory 73 | parser: artifactory 74 | type: jfrog-artifactory-aql 75 | auth_type: simple 76 | 77 | sources: 78 | - repo: 79 | - first-repo.snapshot 80 | auth: 81 | - '{user1}' 82 | - '{password1}' 83 | - repo: 84 | - second-repo.snapshot 85 | auth: 86 | - '{user2}:{password2}' 87 | auth: 88 | - 'user:{password}' 89 | - repo: 90 | - third-repo.snapshot 91 | auth: 92 | - '{auth}' 93 | ``` 94 | 95 | ## Примеры использования. 96 | 97 | - Можно задать системные переменные, указанные в конфигурационном файле. Например: 98 | 99 | `export DOWNLOAD_CI_AUTH='user:password'` 100 | 101 | `export DOWNLOAD_CI_USER='user'` 102 | `export DOWNLOAD_CI_PASSWORD='password'` 103 | 104 | - Можно задать в командной строке. Например: 105 | 106 | `crosspm download -o user=user,password=password` 107 | или 108 | `crosspm download -o auth_libs1=user:password` 109 | -------------------------------------------------------------------------------- /docs/usage/USAGE-PYTHON.md: -------------------------------------------------------------------------------- 1 | 2 | * [Usage as Python module](#usage-as-python-module) 3 | * [Usage](#usage) 4 | * [Package](#package) 5 | * [Logging](#logging) 6 | 7 | Usage as Python module 8 | ====================== 9 | 10 | ### Usage 11 | Use `crosspm` as python-module: 12 | ```python 13 | from crosspm import CrossPM 14 | config_name = 'myconfig.yaml' 15 | dependencies = 'dependencies.txt.lock' 16 | 17 | # Если у вас есть уже подготовленный файл dependencies.txt.lock 18 | argv = 'download -c "{}" --depslock-path="{}" --no-fails'.format(config_name, dependencies) 19 | 20 | # Если нужно указать зависимости сразу в коде 21 | # dependencies = "packagename 1.2.123 master" 22 | # argv = 'download -c "{}" --dependencies-lock-content="{}" --no-fails'.format(config_name, dependencies) 23 | 24 | 25 | # run download and get result 26 | # crosspm_raw_packages - LIST of all downloaded packages (with duplicate) 27 | err, crosspm_raw_packages = CrossPM(argv, return_result='raw').run() 28 | 29 | crosspm_unique_packages = set(crosspm_raw_packages) 30 | 31 | # crosspm_tree_packages - LIST of first-level packages, with child in package.packages variable 32 | err, crosspm_tree_packages = CrossPM(argv, return_result='tree').run() 33 | ``` 34 | 35 | ### Package 36 | You can get some Package field, like this: 37 | 38 | ```python 39 | package = crosspm_tree_packages[0] 40 | # class Package have this public interface: 41 | print('Package has this dependencies') 42 | dependencies_package = package.packages # access to all child packages 43 | print(len(dependencies_package)) 44 | 45 | name = package.name # Package name 46 | packed = package.packed_path # Path to archive (tar.gz\zip\nupkg 47 | unpacked = package.unpacked_path # Path to unpacked folder 48 | dup = package.duplicated # The package has a duplicate package with different version 49 | 50 | pkg = package.pkg # type: ArtifactoryPath from dohq-artifactory 51 | md5 = package.md5 # Artifactory or local file md5hash 52 | 53 | package.get_name_and_path() # LEGACY - access to "name, unpacked_path" 54 | package.get_file_path('some.exe') # Path to unpacked file (inside package) 55 | package.get_file("some.exe") # Like "get_file_path" or None if file not exist 56 | 57 | # Get package params like arch, branch, compiler, osname, version, repo, etc 58 | params = package.get_params() 59 | """ Return dict, like this 60 | NB: version is list 61 | { 62 | "arch": "x86", 63 | "branch": "1.9-pm", 64 | "compiler": "vc120", 65 | "osname": "win", 66 | "package": "libiconv", 67 | "repo": "libs-cpp-release", 68 | "version": ["1", "9", "131", "feature"] 69 | } 70 | """ 71 | 72 | # Get version in string format 73 | params = package.get_params(merged=True) 74 | """ 75 | NB: version is list 76 | { 77 | "arch": "x86", 78 | "branch": "1.9-pm", 79 | "compiler": "vc120", 80 | "osname": "win", 81 | "package": "libiconv", 82 | "repo": "libs-cpp-release", 83 | "version": "1.9.131-feature", 84 | } 85 | """ 86 | 87 | # Get version in string format 88 | params = package.get_params(param_list='version', merged=True) 89 | """ 90 | NB: version is list 91 | { 92 | "version": "1.9.131-feature", 93 | } 94 | """ 95 | params = package.get_params(param_list=['arch','version'], merged=True) 96 | """ 97 | NB: version is list 98 | { 99 | "arch": "x86", 100 | "version": "1.9.131-feature", 101 | } 102 | """ 103 | 104 | ``` 105 | 106 | ### Logging 107 | `crosspm` log message to stderr by default. For now, we have this hack for disable standart logging. 108 | 109 | ```python 110 | def init_logging(): 111 | logger_format_string = '%(message)s' 112 | logging.basicConfig(level=logging.DEBUG, format=logger_format_string, stream=sys.stdout) 113 | init_logging() 114 | cpm = CrossPM(argv_usedby, return_result='raw') 115 | cpm.set_logging_level = lambda: (0, None) 116 | err, crosspm_packages = cpm.run() 117 | ``` 118 | 119 | **NOTE**: If you set your logging setting and not suppressing `crosspm`-logging, you will have two logs, one in `stderr`, other in `stdout` 120 | -------------------------------------------------------------------------------- /docs/usage/USAGE-USEDBY.md: -------------------------------------------------------------------------------- 1 | 2 | * [Использование CrossPM для поиска "пакетов, использующих данный пакет"](#использование-crosspm-для-поиска-пакетов-использующих-данный-пакет) 3 | * [Использование](#использование) 4 | * [Примечания](#примечания) 5 | * [Пример поиска deb-пакетов](#пример-поиска-deb-пакетов) 6 | * [Пример поиска tar-gz crosspm package](#пример-поиска-tar-gz-crosspm-package) 7 | 8 | ## Использование CrossPM для поиска "пакетов, использующих данный пакет" 9 | 10 | Иногда есть необходимость найти "пакеты, испольщующие данный пакет". Например, если вы нашли баг в пакете, нужно забанить так же пакеты, которые используют этот "плохой" пакет 11 | 12 | За поиск отвечает секция `usedby` конфигурации 13 | ```yaml 14 | parsers: 15 | common: 16 | usedby: 17 | AQL: 18 | "@dd.{package}.version": "{version}" 19 | "@dd.{package}.operator": "=" 20 | 21 | property-parser: 22 | "deb.name": "package" 23 | "deb.version": "version" 24 | "qaverdict": "qaverdict" 25 | 26 | path-parser: 'https://(?P.*?)/artifactory/(?P.*?)/(?P.*?)/(?P.*?)/(?P.*?)/(?P.*?)/(?P.*?)/(?P.*?)/.*.tar.gz' 27 | 28 | # может быть и в секции конкретного parser 29 | artifactory: 30 | usedby: ... 31 | ``` 32 | 33 | `AQL` - тут находится правила для непосредственного поиска. Быстрее всего выполнить поиск по `Artifactory Properties` (**они должны быть выставлены предварительно сторонним инструментом) 34 | 35 | При нахождении пакета, мы можем попробовать найти и пакеты, которые зависят от него тоже, но для этого надо вытащить колонки из артефакта в Артифактории 36 | - `property-parser` - берем колонки из `property` Артифактория 37 | - `path-parser` - **в разработке** берем колонки из пути к файлу в Артифактории 38 | 39 | ## Использование 40 | Для поиска нужно создать файл `dependencies.txt` с примерно следующим содержимым и запустить поиск 41 | 42 | ``` 43 | # package version branch 44 | packagename 1.2.3 master 45 | ``` 46 | 47 | ```bash 48 | crosspm usedby -c config.yaml 49 | ``` 50 | 51 | ### Примечания 52 | - **ВАЖНО** - `crosspm` не включает указанный в `dependencies.txt` пакет, чтобы и его найти - нужно использовать команду `crosspm lock` и соединить результаты, тогда будет "полная" картина 53 | - Удобно получать список пакетов через [python-интерфейс](./USAGE-PYTHON) у `crosspm` или через стандартные [jinja-шаблоны](../config/output-template) 54 | 55 | ## Пример поиска deb-пакетов 56 | Допустим, у нас есть пакеты, с примерной иерархией зависимостей: 57 | ``` 58 | package1 1.0.1 59 | - package2 2.0.2 60 | - package3 3.0.3 61 | - package31 31.0.1 62 | - package32 32.0.2 63 | ``` 64 | 65 | Мы проставляем при загрузке в артифакторий дополнительные свойства, по формату (пример для `package3`) 66 | ``` 67 | dd.package31.version: 31.0.1 68 | dd.package31.operator: = 69 | dd.package32.version: 32.0.2 70 | dd.package32.operator: = 71 | ``` 72 | 73 | Теперь, если мы находит баг в пакете `package31`, мы делаем поиск (смотри выше пример cli-запуска) 74 | ```bash 75 | >> cat dependencies.txt 76 | # package version 77 | package31 31.0.1 78 | 79 | # Примерный вывод в stdout 80 | >> crosspm usedby 81 | Dependencies tree: 82 | - 83 | - package3 3.0.3 84 | - package1 1.0.1 85 | ``` 86 | 87 | ## Пример поиска tar-gz crosspm package 88 | Допустим, у нас есть пакеты, с примерной иерархией зависимостей (с помощью файлов `dependencies.txt.lock`, но проставление свойств должно быть реализовано сторонним инструментом) 89 | ``` 90 | package1 1.0.1 91 | - package2 2.0.2 92 | - package3 3.0.3 93 | - package31 31.0.1 94 | - package32 32.0.2 95 | ``` 96 | 97 | Мы проставляем при загрузке в артифакторий дополнительные свойства, по формату (пример для `package3`) 98 | ``` 99 | ud.package31.version: 31.0.1 100 | ud.package32.version: 32.0.2 101 | ``` 102 | 103 | Теперь, если мы находит баг в пакете `package31`, мы делаем поиск (смотри выше пример cli-запуска) 104 | ```bash 105 | >> cat dependencies.txt 106 | # package version 107 | package31 31.0.1 108 | 109 | # Примерный вывод в stdout 110 | >> crosspm usedby 111 | Dependencies tree: 112 | - 113 | - package3 3.0.3 114 | - package1 1.0.1 115 | ``` 116 | -------------------------------------------------------------------------------- /docs/usage/USAGE.md: -------------------------------------------------------------------------------- 1 | 2 | * [Usage](./docs/usage/USAGE.md#usage) 3 | * [crosspm lock](./docs/usage/USAGE.md#crosspm-lock) 4 | 5 | 6 | 7 | 8 | Usage 9 | ======= 10 | Идея CrossPM (как пакетного менеджера) примерно такая же, как и других популярных пакетных менеджеров (npm, pyenv, etc). 11 | 12 | Можно придерживаться примерно следующего алгоритма: 13 | 1. Создаете файл **dependencies.txt** - список пакетов, которые нужно скачать. Поддерживаются маски в версиях, свойства артефактов. 14 | 2. Ищем пакеты и фиксируем их командой `crosspm lock` - создает файл **dependencies.txt.lock**. Он хранит список пакетов последней версии, которые нашел `crosspm`. Условия для поиска задаются в [файле конфигурации crosspm.yaml\config.yaml](../config/CONFIG) 15 | 3. Скачиваем пакеты `crosspm download` - пакеты из **lock**-файла выкачиются рекурсивно (если есть **lock**-файл в скаченном архиве, он скачает все пакеты у пакета) 16 | 4. Используем скачанные пакеты: используем сразу в [Cmake](../usage/USAGE-CMAKE), импортируем переменные из [sh, cmd, json, или stdout](../config/OUTPUT), настраиваем [свой формат файлов](../config/output-template.md) 17 | 18 | `crosspm download` может включает в себя `lock` команду, т.е. если вы используете файл с масками, то он ищет последние при загрузке. 19 | 20 | ### crosspm lock 21 | Команда `crosspm lock` фиксирует текущие версии удовлетворяющие маскам, указанным в файле `dependencies.txt`. Фиксирует версии в выходной файл, обычно, `dependencies.txt.lock`. 22 | Разработчик запускает команду, сохраняет новый файл `dependencies.txt.lock` в гит-репозитория для повторяемости сборки. 23 | 24 | Имея, например, следующий файл манифест **dependencies.txt**: 25 | ```bash 26 | boost * 1.55-pm 27 | poco *- 1.46-pm 28 | openssl 1.0.20- 1.0.1i-pm 29 | log4cplus 1.1.6 1.1-pm 30 | pyyaml 3.10.115 3.10-python-3.4 31 | protobuf 0.>=1.* default 32 | gtest 0.0.6 default 33 | gmock 0.0.5 default 34 | ``` 35 | Выполним команду **lock**: 36 | ```bash 37 | crosspm lock dependencies.txt dependencies.txt.lock 38 | ``` 39 | Будет получен следующий файл **dependencies.txt.lock**: 40 | ```bash 41 | boost 1.55.0 1.55-pm 42 | poco 1.46.11075-develop-vc110-x86 1.46-pm 43 | openssl 1.0.20-develop 1.0.1i-pm 44 | log4cplus 1.1.6 1.1-pm 45 | pyyaml 3.10.115 3.10-python-3.4 46 | protobuf 1.10.343 default 47 | gtest 0.0.6 default 48 | gmock 0.0.5 default 49 | ``` 50 | -------------------------------------------------------------------------------- /gh-md-toc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Steps: 5 | # 6 | # 1. Download corresponding html file for some README.md: 7 | # curl -s $1 8 | # 9 | # 2. Discard rows where no substring 'user-content-' (github's markup): 10 | # awk '/user-content-/ { ... 11 | # 12 | # 3.1 Get last number in each row like ' ... sitemap.js.*<\/h/)+2, RLENGTH-5) 21 | # 22 | # 5. Find anchor and insert it inside "(...)": 23 | # substr($0, match($0, "href=\"[^\"]+?\" ")+6, RLENGTH-8) 24 | # 25 | 26 | gh_toc_version="0.6.0" 27 | 28 | gh_user_agent="gh-md-toc v$gh_toc_version" 29 | 30 | # 31 | # Download rendered into html README.md by its url. 32 | # 33 | # 34 | gh_toc_load() { 35 | local gh_url=$1 36 | 37 | if type curl &>/dev/null; then 38 | curl --user-agent "$gh_user_agent" -s "$gh_url" 39 | elif type wget &>/dev/null; then 40 | wget --user-agent="$gh_user_agent" -qO- "$gh_url" 41 | else 42 | echo "Please, install 'curl' or 'wget' and try again." 43 | exit 1 44 | fi 45 | } 46 | 47 | # 48 | # Converts local md file into html by GitHub 49 | # 50 | # ➥ curl -X POST --data '{"text": "Hello world github/linguist#1 **cool**, and #1!"}' https://api.github.com/markdown 51 | #

Hello world github/linguist#1 cool, and #1!

'" 52 | gh_toc_md2html() { 53 | local gh_file_md=$1 54 | URL=https://api.github.com/markdown/raw 55 | if [ -z "$GH_TOC_TOKEN" ]; then 56 | TOKEN=$GH_TOC_TOKEN 57 | else 58 | TOKEN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/token.txt" 59 | fi 60 | if [ -f "$TOKEN" ]; then 61 | URL="$URL?access_token=$(cat $TOKEN)" 62 | fi 63 | # echo $URL 1>&2 64 | OUTPUT="$(curl -s --user-agent "$gh_user_agent" \ 65 | --data-binary @"$gh_file_md" -H "Content-Type:text/plain" \ 66 | $URL)" 67 | # echo $OUTPUT 1>&2 68 | 69 | if [ "$?" != "0" ]; then 70 | echo "XXNetworkErrorXX" 71 | fi 72 | if [ "$(echo "${OUTPUT}" | awk '/API rate limit exceeded/')" != "" ]; then 73 | echo "XXRateLimitXX" 74 | else 75 | echo "${OUTPUT}" 76 | fi 77 | } 78 | 79 | 80 | # 81 | # Is passed string url 82 | # 83 | gh_is_url() { 84 | case $1 in 85 | https* | http*) 86 | echo "yes";; 87 | *) 88 | echo "no";; 89 | esac 90 | } 91 | 92 | # 93 | # TOC generator 94 | # 95 | gh_toc(){ 96 | local gh_src=$1 97 | local gh_src_copy=$1 98 | local gh_ttl_docs=$2 99 | local need_replace=$3 100 | echo "***** Processing: ${gh_src}" 101 | 102 | if [ "$gh_src" = "" ]; then 103 | echo "Please, enter URL or local path for a README.md" 104 | exit 1 105 | fi 106 | 107 | 108 | # Show "TOC" string only if working with one document 109 | if [ "$gh_ttl_docs" = "1" ]; then 110 | 111 | echo "Table of Contents" 112 | echo "=================" 113 | echo "" 114 | gh_src_copy="" 115 | 116 | fi 117 | 118 | if [ "$(gh_is_url "$gh_src")" == "yes" ]; then 119 | gh_toc_load "$gh_src" | gh_toc_grab "$gh_src_copy" 120 | if [ "${PIPESTATUS[0]}" != "0" ]; then 121 | echo "Could not load remote document." 122 | echo "Please check your url or network connectivity" 123 | exit 1 124 | fi 125 | if [ "$need_replace" = "yes" ]; then 126 | echo 127 | echo "!! '$gh_src' is not a local file" 128 | echo "!! Can't insert the TOC into it." 129 | echo 130 | fi 131 | else 132 | local rawhtml=$(gh_toc_md2html "$gh_src") 133 | if [ "$rawhtml" == "XXNetworkErrorXX" ]; then 134 | echo "Parsing local markdown file requires access to github API" 135 | echo "Please make sure curl is installed and check your network connectivity" 136 | exit 1 137 | fi 138 | if [ "$rawhtml" == "XXRateLimitXX" ]; then 139 | echo "Parsing local markdown file requires access to github API" 140 | echo "Error: You exceeded the hourly limit. See: https://developer.github.com/v3/#rate-limiting" 141 | TOKEN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/token.txt" 142 | echo "or place github auth token here: $TOKEN" 143 | exit 1 144 | fi 145 | local toc=`echo "$rawhtml" | gh_toc_grab "$gh_src_copy"` 146 | echo "$toc" 147 | if [ "$need_replace" = "yes" ]; then 148 | local ts="<\!--ts-->" 149 | local te="<\!--te-->" 150 | local dt=`date +'%F_%H%M%S'` 151 | local ext=".orig.${dt}" 152 | local toc_path="${gh_src}.toc.${dt}" 153 | # http://fahdshariff.blogspot.ru/2012/12/sed-mutli-line-replacement-between-two.html 154 | # clear old TOC 155 | sed -i${ext} "/${ts}/,/${te}/{//!d;}" "$gh_src" 156 | # create toc file 157 | echo "${toc}" > "${toc_path}" 158 | # insert toc file 159 | if [[ "`uname`" == "Darwin" ]]; then 160 | sed -i "" "/${ts}/r ${toc_path}" "$gh_src" 161 | else 162 | sed -i "/${ts}/r ${toc_path}" "$gh_src" 163 | fi 164 | echo 165 | echo "!! TOC was added into: '$gh_src'" 166 | echo "!! Origin version of the file: '${gh_src}${ext}'" 167 | echo "!! TOC added into a separate file: '${toc_path}'" 168 | rm "${gh_src}${ext}" -v 169 | rm "${toc_path}" -v 170 | echo 171 | fi 172 | fi 173 | } 174 | 175 | # 176 | # Grabber of the TOC from rendered html 177 | # 178 | # $1 — a source url of document. 179 | # It's need if TOC is generated for multiple documents. 180 | # 181 | gh_toc_grab() { 182 | # if closed is on the new line, then move it on the prev line 183 | # for example: 184 | # was: The command foo1 185 | # 186 | # became: The command foo1 187 | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n<\/h/<\/h/g' | 188 | # find strings that corresponds to template 189 | grep -E -o '//g' | sed 's/<\/code>//g' | 192 | # now all rows are like: 193 | # ... .*<\/h/)+2, RLENGTH-5) 200 | href = substr($0, match($0, "href=\"[^\"]+?\"")+6, RLENGTH-7) 201 | print sprintf("%*s", level*3, " ") "* [" text "](" gh_url href ")" }' | 202 | sed 'y/+/ /; s/%/\\x/g')" 203 | } 204 | 205 | # 206 | # Returns filename only from full path or url 207 | # 208 | gh_toc_get_filename() { 209 | echo "${1##*/}" 210 | } 211 | 212 | # 213 | # Options hendlers 214 | # 215 | gh_toc_app() { 216 | local app_name=$(basename $0) 217 | local need_replace="no" 218 | 219 | if [ "$1" = '--help' ] || [ $# -eq 0 ] ; then 220 | echo "GitHub TOC generator ($app_name): $gh_toc_version" 221 | echo "" 222 | echo "Usage:" 223 | echo " $app_name [--insert] src [src] Create TOC for a README file (url or local path)" 224 | echo " $app_name - Create TOC for markdown from STDIN" 225 | echo " $app_name --help Show help" 226 | echo " $app_name --version Show version" 227 | return 228 | fi 229 | 230 | if [ "$1" = '--version' ]; then 231 | echo "$gh_toc_version" 232 | echo 233 | echo "os: `lsb_release -d | cut -f 2`" 234 | echo "kernel: `cat /proc/version`" 235 | echo "shell: `$SHELL --version`" 236 | echo 237 | for tool in curl wget grep awk sed; do 238 | printf "%-5s: " $tool 239 | echo `$tool --version | head -n 1` 240 | done 241 | return 242 | fi 243 | 244 | if [ "$1" = "-" ]; then 245 | if [ -z "$TMPDIR" ]; then 246 | TMPDIR="/tmp" 247 | elif [ -n "$TMPDIR" -a ! -d "$TMPDIR" ]; then 248 | mkdir -p "$TMPDIR" 249 | fi 250 | local gh_tmp_md 251 | gh_tmp_md=$(mktemp $TMPDIR/tmp.XXXXXX) 252 | while read input; do 253 | echo "$input" >> "$gh_tmp_md" 254 | done 255 | gh_toc_md2html "$gh_tmp_md" | gh_toc_grab "" 256 | return 257 | fi 258 | 259 | if [ "$1" = '--insert' ]; then 260 | need_replace="yes" 261 | shift 262 | fi 263 | 264 | for md in "$@" 265 | do 266 | echo "" 267 | gh_toc "$md" "$#" "$need_replace" 268 | done 269 | 270 | echo "" 271 | echo "Created by [gh-md-toc](https://github.com/ekalinin/github-markdown-toc)" 272 | } 273 | 274 | # 275 | # Entry point 276 | # 277 | gh_toc_app "$@" 278 | -------------------------------------------------------------------------------- /requirements-ci.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | -r requirements-test.txt 3 | coverage==4.5 4 | codacy-coverage==1.3.11 5 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | -r requirements-test.txt 3 | pre-commit 4 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | flake8==3.7.9 2 | pytest>=5.2 3 | pytest-flask>=1.0.0 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | docopt==0.6.2 2 | dohq-artifactory>=0.8.3 3 | Jinja2>=3.1.3 4 | patool==1.12 5 | pyunpack==0.2 6 | PyYAML>=6.0 7 | requests>=2.30.0,<3.0.0 8 | urllib3>=2.2.0 9 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest 3 | 4 | [flake8] 5 | max-line-length = 120 6 | exclude = tests,.eggs 7 | ignore = F405,F403 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | from setuptools import setup 5 | from crosspm import config 6 | 7 | build_number = os.getenv('GITHUB_RUN_NUMBER', '0') 8 | branch = os.getenv('GITHUB_REF_NAME', '') 9 | is_gh_actions = os.getenv('CI') == 'true' 10 | version = config.__version__.split('.') 11 | develop_status = '4 - Beta' 12 | url = 'http://devopshq.github.io/crosspm' 13 | 14 | if is_gh_actions: 15 | version = version[0:3] 16 | if branch == 'master': 17 | develop_status = '5 - Production/Stable' 18 | version.append(build_number) 19 | else: 20 | version.append('{}{}'.format('dev' if branch == 'develop' else branch, build_number)) 21 | else: 22 | if len(version) < 4: 23 | version.append('dev0') 24 | 25 | version = '.'.join(version) 26 | if is_gh_actions: 27 | with open('crosspm/config.py', 'w', encoding="utf-8") as f: 28 | f.write("__version__ = '{}'".format(version)) 29 | 30 | with open('README.md', encoding="utf-8") as f: 31 | long_description = f.read() 32 | 33 | setup( 34 | name='crosspm', 35 | version=version, 36 | description='Cross Package Manager', 37 | license='MIT', 38 | author='Alexander Kovalev', 39 | author_email='ak@alkov.pro', 40 | url=url, 41 | long_description=long_description, 42 | download_url='https://github.com/devopshq/crosspm.git', 43 | entry_points={'console_scripts': ['crosspm=crosspm.__main__:main']}, 44 | python_requires='>=3.8.0', 45 | classifiers=[ 46 | 'Development Status :: {}'.format(develop_status), 47 | 'Environment :: Console', 48 | 'Intended Audience :: Developers', 49 | 'Topic :: Software Development :: Build Tools', 50 | 'License :: OSI Approved :: MIT License', 51 | 'Natural Language :: English', 52 | 'Programming Language :: Python :: 3', 53 | 'Programming Language :: Python :: 3.8', 54 | 'Programming Language :: Python :: 3.9', 55 | 'Programming Language :: Python :: 3.10', 56 | 'Programming Language :: Python :: 3.11', 57 | ], 58 | keywords=[ 59 | 'development', 60 | 'dependency', 61 | 'requirements', 62 | 'manager', 63 | 'versioning', 64 | 'packet', 65 | ], 66 | packages=[ 67 | 'crosspm', 68 | 'crosspm.helpers', 69 | 'crosspm.adapters', 70 | 'crosspm.template', 71 | ], 72 | setup_requires=[ 73 | 'wheel==0.34.2', 74 | 'pypandoc==1.5', 75 | ], 76 | tests_require=[ 77 | "pytest>=5.2", 78 | "pytest-flask>=1.0.0", 79 | "PyYAML>=6.0", 80 | ], 81 | install_requires=[ 82 | "requests>=2.30.0,<3.0.0", 83 | 'urllib3>=2.2.0', 84 | 'docopt==0.6.2', 85 | "PyYAML>=6.0", 86 | "dohq-artifactory>=0.8.3", 87 | "Jinja2>=3.1.3", 88 | 'patool==1.12', # need for pyunpack 89 | 'pyunpack==0.2', 90 | ], 91 | package_data={ 92 | '': [ 93 | './template/*.j2', 94 | '*.cmake', 95 | '../LICENSE', 96 | # '../README.*', 97 | ], 98 | }, 99 | ) 100 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devopshq/crosspm/f081203edc5c4df99b0f44b7e79ed2da31aa630a/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | import pytest 4 | 5 | from crosspm.helpers.output import Output 6 | from crosspm.helpers.package import Package 7 | 8 | 9 | def pytest_configure(config): 10 | config.addinivalue_line( 11 | "markers", "artifactoryaql: testing artifactoryaql" 12 | ) 13 | 14 | 15 | @pytest.fixture(scope="function") 16 | def package(): 17 | params = {'arch': 'x86', 'osname': 'win', 'package': 'package'} 18 | params_found = {'repo': 'lib-cpp-release', 'version': '1.2.3'} 19 | _package = Package('package', None, params, None, None, None, params_found, None, None) 20 | _package.unpacked_path = "/test/path" 21 | yield _package 22 | 23 | 24 | @pytest.fixture(scope="function") 25 | def package_root(): 26 | """ 27 | Create root with dependencies: 28 | root 29 | - package1 30 | - package11 31 | - package12 32 | - package2 33 | """ 34 | params = {'arch': 'x86', 'osname': 'win', 'package': 'root'} 35 | _root = Package('root', None, params, None, None, None, None, None, None) 36 | 37 | params = {'arch': 'x86', 'osname': 'win', 'package': 'package1'} 38 | _package1 = Package('package1', None, params, None, None, None, None, None, None) 39 | 40 | params = {'arch': 'x86', 'osname': 'win', 'package': 'package11'} 41 | _package11 = Package('package11', None, params, None, None, None, None, None, None) 42 | 43 | params = {'arch': 'x86', 'osname': 'win', 'package': 'package12'} 44 | _package12 = Package('package12', None, params, None, None, None, None, None, None) 45 | 46 | params = {'arch': 'x86', 'osname': 'win', 'package': 'package2'} 47 | _package2 = Package('package2', None, params, None, None, None, None, None, None) 48 | 49 | _package1.packages = OrderedDict([('package11', _package11), ('package12', _package12)]) 50 | _root.packages = OrderedDict([('package2', _package2), ('package1', _package1)]) 51 | for _package in _root.all_packages: 52 | _package.unpacked_path = "/test/path/{}".format(_package.name) 53 | yield _root 54 | 55 | 56 | @pytest.fixture 57 | def output(): 58 | _output = Output(data=None, name_column='package', config=None) 59 | yield _output 60 | -------------------------------------------------------------------------------- /tests/data/config_deps.yaml: -------------------------------------------------------------------------------- 1 | cpm: 2 | dependencies: test1_dependencies.txt 3 | 4 | common: 5 | server: https://stub/artifactory 6 | parser: artifactory 7 | type: jfrog-artifactory-aql 8 | 9 | parsers: 10 | artifactory: 11 | path: '{server}/{repo}/{package}.{version}[.tar.gz|.zip]' 12 | properties: '' 13 | 14 | sources: 15 | - repo: 16 | - stub 17 | -------------------------------------------------------------------------------- /tests/data/config_deps_depslock.yaml: -------------------------------------------------------------------------------- 1 | cpm: 2 | dependencies: test3_dependencies.txt 3 | dependencies-lock: test4_dependencies.txt.lock 4 | 5 | common: 6 | server: https://stub/artifactory 7 | parser: artifactory 8 | type: jfrog-artifactory-aql 9 | 10 | parsers: 11 | artifactory: 12 | path: '{server}/{repo}/{package}.{version}[.tar.gz|.zip]' 13 | properties: '' 14 | 15 | sources: 16 | - repo: 17 | - stub 18 | -------------------------------------------------------------------------------- /tests/data/config_depslock.yaml: -------------------------------------------------------------------------------- 1 | cpm: 2 | dependencies-lock: test2_dependencies.txt.lock 3 | 4 | common: 5 | server: https://stub/artifactory 6 | parser: artifactory 7 | type: jfrog-artifactory-aql 8 | 9 | parsers: 10 | artifactory: 11 | path: '{server}/{repo}/{package}.{version}[.tar.gz|.zip]' 12 | properties: '' 13 | 14 | sources: 15 | - repo: 16 | - stub 17 | -------------------------------------------------------------------------------- /tests/data/config_en_cp1251.yaml: -------------------------------------------------------------------------------- 1 | cpm: 2 | description: English description -------------------------------------------------------------------------------- /tests/data/config_en_utf8_bom.yaml: -------------------------------------------------------------------------------- 1 | # Comment 2 | cpm: 3 | description: My config for english test -------------------------------------------------------------------------------- /tests/data/config_import_with_comment.yaml: -------------------------------------------------------------------------------- 1 | ####################### 2 | ####IMPORT SETTINGS#### 3 | ####################### 4 | 5 | import: 6 | - test_cred.yaml 7 | -------------------------------------------------------------------------------- /tests/data/config_import_without_comment.yaml: -------------------------------------------------------------------------------- 1 | import: 2 | - test_cred.yaml 3 | -------------------------------------------------------------------------------- /tests/data/config_recursive_off.yaml: -------------------------------------------------------------------------------- 1 | cpm: 2 | recursive: false 3 | 4 | common: 5 | server: https://stub/artifactory 6 | parser: artifactory 7 | type: jfrog-artifactory-aql 8 | 9 | parsers: 10 | artifactory: 11 | path: '{server}/{repo}/{package}.{version}[.tar.gz|.zip]' 12 | properties: '' 13 | 14 | sources: 15 | - repo: 16 | - stub 17 | -------------------------------------------------------------------------------- /tests/data/config_recursive_on.yaml: -------------------------------------------------------------------------------- 1 | cpm: 2 | recursive: true 3 | 4 | common: 5 | server: https://stub/artifactory 6 | parser: artifactory 7 | type: jfrog-artifactory-aql 8 | 9 | parsers: 10 | artifactory: 11 | path: '{server}/{repo}/{package}.{version}[.tar.gz|.zip]' 12 | properties: '' 13 | 14 | sources: 15 | - repo: 16 | - stub 17 | -------------------------------------------------------------------------------- /tests/data/config_ru_utf8_bom.yaml: -------------------------------------------------------------------------------- 1 | # Комментарий 2 | cpm: 3 | description: Описание на русском -------------------------------------------------------------------------------- /tests/data/config_stub.yaml: -------------------------------------------------------------------------------- 1 | cpm: 2 | description: test stub 3 | 4 | common: 5 | server: https://stub/artifactory 6 | parser: artifactory 7 | type: jfrog-artifactory-aql 8 | 9 | parsers: 10 | artifactory: 11 | path: '{server}/{repo}/{package}.{version}[.tar.gz|.zip]' 12 | properties: '' 13 | 14 | sources: 15 | - repo: 16 | - stub 17 | -------------------------------------------------------------------------------- /tests/data/test_cred.yaml: -------------------------------------------------------------------------------- 1 | .aliases: 2 | - &auth1 3 | - test_user 4 | - test-passwd 5 | -------------------------------------------------------------------------------- /tests/test_artifactoryaql.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import pytest 3 | 4 | from crosspm.adapters.artifactoryaql import Adapter 5 | from crosspm.helpers.source import Source 6 | 7 | 8 | class TestAdapter: 9 | @pytest.fixture() 10 | def source(self): 11 | source = Source("", "", {}) 12 | return source 13 | 14 | @pytest.fixture() 15 | def auth(self): 16 | list_or_file_path = {'raw': [{'auth': 'concrete_user:concrete_password'}]} 17 | return list_or_file_path 18 | 19 | @pytest.fixture() 20 | def user_password(self): 21 | list_or_file_path = {'raw': [{'user': 'concrete_user', 'password': 'concrete_password'}]} 22 | return list_or_file_path 23 | 24 | def test_split_auth(self): 25 | assert ['concrete_user', 'concrete_password'] == Adapter("").split_auth('concrete_user:concrete_password') 26 | 27 | def test_search_auth_with_auth_parameter(self, source, auth): 28 | source.args['auth'] = '{auth}' 29 | Adapter("").search_auth(auth, source) 30 | assert source.args['auth'] == ['concrete_user', 'concrete_password'] 31 | 32 | def test_search_auth_with_user_password_parameter(self, source, user_password): 33 | source.args['auth'] = '{user}:{password}' 34 | Adapter("").search_auth(user_password, source) 35 | assert source.args['auth'] == ['concrete_user', 'concrete_password'] 36 | 37 | def test_search_auth_with_password_parameter(self, source, user_password): 38 | source.args['auth'] = 'concrete_user:{password}' 39 | Adapter("").search_auth(user_password, source) 40 | assert source.args['auth'] == ['concrete_user', 'concrete_password'] 41 | 42 | def test_search_auth_without_parameter(self, source, user_password): 43 | source.args['auth'] = 'concrete_user:concrete_password' 44 | Adapter("").search_auth(user_password, source) 45 | assert source.args['auth'] == ['concrete_user', 'concrete_password'] 46 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from pathlib import Path 4 | 5 | import pytest 6 | 7 | from crosspm.helpers.config import Config 8 | 9 | DATA = Path(__file__).parent / 'data' 10 | 11 | 12 | @pytest.mark.parametrize("filename", [ 13 | ('config_en_cp1251'), 14 | ('config_en_utf8_bom'), 15 | ('config_ru_utf8_bom'), 16 | ]) 17 | def test_config(filename): 18 | c = Config.load_yaml(str(DATA / '{}.yaml'.format(filename))) 19 | assert isinstance(c, dict) 20 | 21 | 22 | @pytest.mark.parametrize("filename", [ 23 | ('config_import_without_comment'), 24 | ('config_import_with_comment'), 25 | ]) 26 | def test_config_comment_before_import_handle(filename, monkeypatch): 27 | monkeypatch.chdir('{}/data'.format(Path(__file__).parent)) 28 | result = Config.load_yaml(str(DATA / '{}.yaml'.format(filename))) 29 | assert '.aliases' in result 30 | 31 | 32 | @pytest.mark.parametrize("filename, cli_param, result", [ 33 | ('config_recursive_on', None, True), 34 | ('config_recursive_off', None, False), 35 | ('config_recursive_on', False, False), 36 | ('config_recursive_off', False, False), 37 | ('config_recursive_on', True, True), 38 | ('config_recursive_off', True, True), 39 | ('config_stub', None, False), 40 | ('config_stub', True, True), 41 | ('config_stub', False, False), 42 | ]) 43 | def test_config_recursive(filename, cli_param, result): 44 | config = Config(config_file_name=str(DATA / f'{filename}.yaml'), recursive=cli_param) 45 | assert config.recursive == result 46 | 47 | 48 | @pytest.mark.parametrize("filename, deps, deps_lock, result_deps, result_deps_lock", [ 49 | ('config_deps', "", "", "test1_dependencies.txt", "test1_dependencies.txt.lock"), 50 | ('config_depslock', "", "", "dependencies.txt", "test2_dependencies.txt.lock"), 51 | ('config_deps_depslock', "", "", "test3_dependencies.txt", "test4_dependencies.txt.lock"), 52 | ('config_stub', "", "", "dependencies.txt", "dependencies.txt.lock"), 53 | ('config_deps', "derps.txt", "", "derps.txt", "derps.txt.lock"), 54 | ('config_depslock', "derps.txt", "", "derps.txt", "derps.txt.lock"), 55 | ('config_deps_depslock', "derps.txt", "", "derps.txt", "derps.txt.lock"), 56 | ('config_stub', "derps.txt", "", "derps.txt", "derps.txt.lock"), 57 | ('config_deps', "", "derps.txt.lock", "test1_dependencies.txt", "derps.txt.lock"), 58 | ('config_depslock', "", "derps.txt.lock", "dependencies.txt", "derps.txt.lock"), 59 | ('config_deps_depslock', "", "derps.txt.lock", "test3_dependencies.txt", "derps.txt.lock"), 60 | ('config_stub', "derps.txt", "derps.txt.lock", "derps.txt", "derps.txt.lock"), 61 | ('config_deps', "derps.txt", "derps.txt.lock", "derps.txt", "derps.txt.lock"), 62 | ('config_depslock', "derps.txt", "derps.txt.lock", "derps.txt", "derps.txt.lock"), 63 | ('config_deps_depslock', "derps.txt", "derps.txt.lock", "derps.txt", "derps.txt.lock"), 64 | ('config_stub', "derps.txt", "derps.txt.lock", "derps.txt", "derps.txt.lock"), 65 | ]) 66 | def test_config_deps_depslock(filename, deps, deps_lock, result_deps, result_deps_lock): 67 | config = Config(config_file_name=str(DATA / f'{filename}.yaml'), deps_file_path=deps, deps_lock_file_path=deps_lock) 68 | assert config.deps_file_name == result_deps 69 | assert config.deps_lock_file_name == result_deps_lock 70 | -------------------------------------------------------------------------------- /tests/test_cpm.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | 4 | import pytest 5 | 6 | from crosspm.cpm import CrossPM 7 | 8 | 9 | # 10 | # ARGS parse 11 | # 12 | 13 | @pytest.mark.parametrize("expect, raw", [ 14 | 15 | (["command", "--config", "/home/src/config.yml"], 16 | "command --config /home/src/config.yml"), 17 | 18 | (["command"], 19 | "command"), 20 | 21 | (["command", "--command", "my", "args"], 22 | "command --command my args"), 23 | 24 | (["command", "--recursive=True"], 25 | "command --recursive"), 26 | 27 | (["command", "--recursive=True", "--other", "command"], 28 | "command --recursive --other command"), 29 | 30 | (["command", "--recursive=True"], 31 | "command --recursive=True"), 32 | 33 | (["command", "--recursive=False"], 34 | "command --recursive=False"), 35 | 36 | (["command", "--recursive", "True"], 37 | "command --recursive True"), 38 | 39 | (["command", "--recursive", "False"], 40 | "command --recursive False"), 41 | 42 | (["command", "--recursive", "false"], 43 | "command --recursive false"), 44 | 45 | ]) 46 | def test_prepare_args_any(expect, raw): 47 | assert CrossPM.prepare_args(raw, windows=False) == expect, "Error when pass as string, on Linux-system" 48 | assert CrossPM.prepare_args(raw, windows=True) == expect, "Error when pass as string, on Windows-system" 49 | 50 | 51 | @pytest.mark.parametrize("expect, raw", [ 52 | (["command", "--config", "e:\\project\\config.yaml"], 53 | "command --config e:\\project\\config.yaml"), 54 | (['download', '--options', '"with quotes"'], 55 | 'download --options "with quotes"'), 56 | ]) 57 | def test_prepare_args_windows(expect, raw): 58 | assert CrossPM.prepare_args(raw, windows=True) == expect, "Error when pass as string, on Windows-system" 59 | 60 | 61 | @pytest.mark.parametrize("expect, raw", [ 62 | (['download', '--options', 'with quotes'], 63 | 'download --options "with quotes"'), 64 | ]) 65 | def test_prepare_args_linux(expect, raw): 66 | assert CrossPM.prepare_args(raw, windows=False) == expect, "Error when pass as string, on Linux-system" 67 | 68 | 69 | def test_lock_recursive_unknown(): 70 | try: 71 | _ = CrossPM("lock --recursive Trololo") 72 | except Exception as e: 73 | assert "Trololo" in str(e) 74 | 75 | 76 | # --stdout 77 | 78 | def test_stdout_flag(): 79 | cpm = CrossPM("download") 80 | assert cpm.stdout is False 81 | 82 | cpm = CrossPM("download --stdout") 83 | assert cpm.stdout is True 84 | 85 | os.environ['CROSSPM_STDOUT'] = '1' 86 | cpm = CrossPM("download") 87 | assert cpm.stdout is True 88 | -------------------------------------------------------------------------------- /tests/test_helpers_python.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | 4 | from crosspm.helpers.python import get_object_from_string 5 | 6 | 7 | def test_get_object_from_string(): 8 | obj_ = get_object_from_string('os.path') 9 | assert os.path is obj_ 10 | -------------------------------------------------------------------------------- /tests/test_output.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from io import StringIO 3 | 4 | from crosspm.helpers.output import LIST, PLAIN, DICT 5 | 6 | 7 | def test_output_format_shell_one(package, output): 8 | expected_result = """ 9 | PACKAGE_ROOT='/test/path' 10 | 11 | """ 12 | result = output.output_format_shell(packages={'package': package}) 13 | assert expected_result == result 14 | 15 | 16 | def test_output_format_shell_dependencies(package_root, output): 17 | assert True 18 | expected_result = """ 19 | PACKAGE2_ROOT='/test/path/package2' 20 | PACKAGE11_ROOT='/test/path/package11' 21 | PACKAGE12_ROOT='/test/path/package12' 22 | PACKAGE1_ROOT='/test/path/package1' 23 | 24 | """ 25 | result = output.output_format_shell(packages=package_root.packages) 26 | assert expected_result == result 27 | 28 | 29 | def test_output_format_shell_cmd_one(package, output): 30 | expected_result = """ 31 | set PACKAGE_ROOT=/test/path 32 | 33 | """ 34 | result = output.output_format_cmd(packages={'package': package}) 35 | assert expected_result == result 36 | 37 | 38 | def test_output_format_cmd_dependencies(package_root, output): 39 | assert True 40 | expected_result = """ 41 | set PACKAGE2_ROOT=/test/path/package2 42 | set PACKAGE11_ROOT=/test/path/package11 43 | set PACKAGE12_ROOT=/test/path/package12 44 | set PACKAGE1_ROOT=/test/path/package1 45 | 46 | """ 47 | result = output.output_format_cmd(packages=package_root.packages) 48 | assert expected_result == result 49 | 50 | 51 | def test_output_format_python_list(package_root, output): 52 | expected_result = """# -*- coding: utf-8 -*- 53 | 54 | PACKAGES_ROOT = [""" 55 | output._output_config['type'] = LIST 56 | result = output.output_format_python(packages=package_root.packages) 57 | assert result.startswith(expected_result) 58 | 59 | 60 | def test_output_format_python_dict(package_root, output): 61 | expected_result = """# -*- coding: utf-8 -*- 62 | 63 | PACKAGES_ROOT = {""" 64 | output._output_config['type'] = DICT 65 | result = output.output_format_python(packages=package_root.packages) 66 | assert result.startswith(expected_result) 67 | 68 | 69 | def test_output_format_python_plain(package_root, output): 70 | expected_result = """# -*- coding: utf-8 -*- 71 | 72 | PACKAGE11_ROOT = '/test/path/package11' 73 | PACKAGE12_ROOT = '/test/path/package12' 74 | PACKAGE1_ROOT = '/test/path/package1' 75 | PACKAGE2_ROOT = '/test/path/package2' 76 | """ 77 | output._output_config['type'] = PLAIN 78 | result = output.output_format_python(packages=package_root.packages) 79 | assert result == expected_result 80 | 81 | 82 | def test_output_format_json(package_root, output): 83 | expected_result = """{ 84 | "PACKAGES_ROOT": { 85 | "PACKAGE2_ROOT": "/test/path/package2", 86 | "PACKAGE11_ROOT": "/test/path/package11", 87 | "PACKAGE12_ROOT": "/test/path/package12", 88 | "PACKAGE1_ROOT": "/test/path/package1" 89 | } 90 | }""" 91 | result = output.output_format_json(packages=package_root.packages) 92 | assert result == expected_result 93 | 94 | 95 | def test_output_format_stdout(package_root, output): 96 | expected_result = """PACKAGE2_ROOT: /test/path/package2 97 | PACKAGE11_ROOT: /test/path/package11 98 | PACKAGE12_ROOT: /test/path/package12 99 | PACKAGE1_ROOT: /test/path/package1 100 | """ 101 | old_stdout = sys.stdout 102 | sys.stdout = mystdout = StringIO() 103 | 104 | output.output_format_stdout(packages=package_root.packages) 105 | 106 | sys.stdout = old_stdout 107 | mystdout.seek(0) 108 | assert expected_result == mystdout.read() 109 | 110 | 111 | def test_output_jinja2(): 112 | # Создавать темповый файл с 113 | # for package in packages - {{package.name}} 114 | # 115 | # И явно проверять result, какой хотим 116 | # Удалить темповый файл 117 | assert True 118 | 119 | 120 | def test_output_format_lock(): 121 | assert True 122 | -------------------------------------------------------------------------------- /tests/test_package.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | from unittest import TestCase 4 | 5 | 6 | def test_package_tree(package_root): 7 | packages = [x.name for x in package_root.all_packages] 8 | assert packages == ['package11', 'package12', 'package2', 'package1'] 9 | 10 | 11 | def test_public_interface(package): 12 | """ 13 | Test public interface, which documented in docs/USAGE-PYTHON.md and people can use this attribute in python-code 14 | """ 15 | assert hasattr(package, 'name') 16 | assert hasattr(package, 'packed_path') 17 | assert hasattr(package, 'unpacked_path') 18 | assert hasattr(package, 'duplicated') 19 | 20 | assert hasattr(package, 'packages') 21 | assert isinstance(package.packages, dict) 22 | 23 | # Legacy get_name_and_path 24 | assert package.get_name_and_path() == (package.name, package.unpacked_path) 25 | 26 | 27 | def test_get_file(package): 28 | assert os.path.normpath(package.get_file_path('some.exe')) == os.path.normpath('/test/path/some.exe') 29 | assert package.get_file('not-exist.exe', unpack_force=False) is None 30 | 31 | 32 | def test_get_params(package): 33 | assert isinstance(package.get_params(), dict) 34 | assert package.get_params()['osname'] == 'win' 35 | -------------------------------------------------------------------------------- /tests/test_parser.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import pytest 3 | import yaml 4 | 5 | from crosspm.helpers.parser import Parser 6 | 7 | 8 | class BaseParserTest: 9 | _parsers = {} 10 | 11 | def init_parser(self, parsers): 12 | if 'common' not in parsers: 13 | parsers['common'] = {} 14 | for k, v in parsers.items(): 15 | if k not in self._parsers: 16 | v.update({_k: _v for _k, _v in parsers['common'].items() if _k not in v}) 17 | self._parsers[k] = Parser(k, v, self) 18 | else: 19 | return False 20 | # code = CROSSPM_ERRORCODE_CONFIG_FORMAT_ERROR 21 | # msg = 'Config file contains multiple definitions of the same parser: [{}]'.format(k) 22 | # self._log.exception(msg) 23 | # raise CrosspmException(code, msg) 24 | if len(self._parsers) == 0: 25 | return False 26 | # code = CROSSPM_ERRORCODE_CONFIG_FORMAT_ERROR 27 | # msg = 'Config file does not contain parsers! Unable to process any further.' 28 | # self._log.exception(msg) 29 | # raise CrosspmException(code, msg) 30 | return True 31 | 32 | 33 | class TestParser(BaseParserTest): 34 | config = r""" 35 | parsers: 36 | common: 37 | columns: 38 | version: "{int}.{int}.{int}[.{int}][-{str}]" 39 | sort: 40 | - version 41 | - '*' 42 | index: -1 43 | 44 | 45 | artifactory: 46 | path: "{server}/{repo}/{package}/{branch}/{version}/{compiler|any}/{arch|any}/{osname}/{package}.{version}[.zip|.tar.gz|.nupkg]" 47 | properties: "" 48 | usedby: 49 | AQL: 50 | "@dd.{package}.version": "{version}" 51 | "@dd.{package}.operator": "=" 52 | "path": 53 | "$match": "*vc140/x86_64/win*" 54 | 55 | property-parser: 56 | "deb.name": "package" 57 | "deb.version": "version" 58 | "qaverdict": "qaverdict" 59 | 60 | path-parser: 'https://repo.example.com/artifactory/libs-cpp-release.snapshot/(?P.*?)/(?P.*?)/(?P.*?)/(?P.*?)/(?P.*?)/(?P.*?)/.*.tar.gz' 61 | """ 62 | 63 | @pytest.fixture(scope='class', autouse=True) 64 | def init(self): 65 | config = yaml.safe_load(self.config) 66 | assert self.init_parser(config.get('parsers', {})) is True 67 | 68 | def test__init(self): 69 | assert isinstance(self._parsers.get('artifactory', None), Parser) 70 | 71 | def test__parse_by_mask__package(self): 72 | parser = self._parsers.get('artifactory', None) 73 | 74 | assert \ 75 | parser.parse_by_mask('package', 'boost', False, True) == \ 76 | 'boost' 77 | 78 | def test__parse_by_mask__version(self): 79 | parser = self._parsers.get('artifactory', None) 80 | assert \ 81 | parser.parse_by_mask('version', '1.2.3', False, True) == \ 82 | ['1', '2', '3', None, None] 83 | 84 | assert \ 85 | parser.parse_by_mask('version', '1.2.3.4', False, True) == \ 86 | ['1', '2', '3', '4', None] 87 | 88 | assert \ 89 | parser.parse_by_mask('version', '1.2.3.4-feature', False, True) == \ 90 | ['1', '2', '3', '4', 'feature'] 91 | 92 | # TODO: Make this test pass 93 | # assert parser.parse_by_mask('version', '1.2.3-feature', False, True) == ['1', '2', '3', None, 'feature'] 94 | 95 | def test__parse_by_mask__version_with_types(self): 96 | parser = self._parsers.get('artifactory', None) 97 | assert \ 98 | parser.parse_by_mask('version', '1.2.3', True, True) == \ 99 | [('1', 'int'), ('2', 'int'), ('3', 'int'), (None, 'int'), (None, 'str')] 100 | 101 | assert \ 102 | parser.parse_by_mask('version', '1.2.3.4', True, True) == \ 103 | [('1', 'int'), ('2', 'int'), ('3', 'int'), ('4', 'int'), (None, 'str')] 104 | 105 | assert \ 106 | parser.parse_by_mask('version', '1.2.3.4-feature', True, True) == \ 107 | [('1', 'int'), ('2', 'int'), ('3', 'int'), ('4', 'int'), ('feature', 'str')] 108 | 109 | # TODO: Make this test pass 110 | # assert \ 111 | # parser.parse_by_mask('version', '1.2.3-feature', True, True) == \ 112 | # [('1', 'int'), ('2', 'int'), ('3', 'int'), (None, 'int'), ('feature', 'str')] 113 | 114 | def test__merge_with_mask(self): 115 | pass 116 | 117 | @pytest.mark.artifactoryaql 118 | def test_split_fixed_pattern_with_file_name_with_mask(self): 119 | parser = self._parsers.get('common', None) 120 | 121 | path = "https://repo.example.com/artifactory/libs-cpp-release.snapshot/boost/1.60-pm/*.*.*/vc110/x86/win/boost.*.*.*.tar.gz" 122 | path_fixed = "https://repo.example.com/artifactory/libs-cpp-release.snapshot/boost/1.60-pm" 123 | path_pattern = "*.*.*/vc110/x86/win" 124 | file_name_pattern = "boost.*.*.*.tar.gz" 125 | 126 | _path_fixed, _path_pattern, _file_name_pattern = parser.split_fixed_pattern_with_file_name(path) 127 | 128 | assert path_fixed == _path_fixed 129 | assert path_pattern == _path_pattern 130 | assert file_name_pattern == _file_name_pattern 131 | 132 | @pytest.mark.artifactoryaql 133 | def test_split_fixed_pattern_with_file_name_with_lock(self): 134 | parser = self._parsers.get('common', None) 135 | 136 | path = "https://repo.example.com/artifactory/libs-cpp-release.snapshot/zlib/1.2.8-pm/1.2.8.199/vc110/x86/win/zlib.1.2.8.199.tar.gz" 137 | path_fixed = "https://repo.example.com/artifactory/libs-cpp-release.snapshot/zlib/1.2.8-pm/1.2.8.199/vc110/x86/win" 138 | path_pattern = "" 139 | file_name_pattern = "zlib.1.2.8.199.tar.gz" 140 | 141 | _path_fixed, _path_pattern, _file_name_pattern = parser.split_fixed_pattern_with_file_name(path) 142 | 143 | assert path_fixed == _path_fixed 144 | assert path_pattern == _path_pattern 145 | assert file_name_pattern == _file_name_pattern 146 | 147 | @pytest.mark.artifactoryaql 148 | def test_split_fixed_pattern_with_file_name_with_partial_lock(self): 149 | parser = self._parsers.get('common', None) 150 | 151 | path = "https://repo.example.com/artifactory/libs-cpp-release.snapshot/build-tools/master/0.6.*/vc120/x86/win/build-tools.0.6.*.tar.gz" 152 | path_fixed = "https://repo.example.com/artifactory/libs-cpp-release.snapshot/build-tools/master" 153 | path_pattern = "0.6.*/vc120/x86/win" 154 | file_name_pattern = "build-tools.0.6.*.tar.gz" 155 | 156 | _path_fixed, _path_pattern, _file_name_pattern = parser.split_fixed_pattern_with_file_name(path) 157 | 158 | assert path_fixed == _path_fixed 159 | assert path_pattern == _path_pattern 160 | assert file_name_pattern == _file_name_pattern 161 | 162 | @pytest.mark.artifactoryaql 163 | def test_split_fixed_pattern_with_file_name_with_masked_filename_only(self): 164 | parser = self._parsers.get('common', None) 165 | 166 | path = "https://repo.example.com/artifactory/libs-cpp-release.snapshot/build-tools/master/vc120/x86/win/build-tools.0.6.*.tar.gz" 167 | path_fixed = "https://repo.example.com/artifactory/libs-cpp-release.snapshot/build-tools/master/vc120/x86/win" 168 | path_pattern = "" 169 | file_name_pattern = "build-tools.0.6.*.tar.gz" 170 | 171 | _path_fixed, _path_pattern, _file_name_pattern = parser.split_fixed_pattern_with_file_name(path) 172 | 173 | assert path_fixed == _path_fixed 174 | assert path_pattern == _path_pattern 175 | assert file_name_pattern == _file_name_pattern 176 | 177 | def test_split_fixed_pattern(self): 178 | parser = self._parsers.get('common', None) 179 | 180 | path = "https://repo.example.com/artifactory/libs-cpp-release.snapshot/boost/1.60-pm/*.*.*/vc110/x86/win/boost.*.*.*.tar.gz" 181 | path_fixed = "https://repo.example.com/artifactory/libs-cpp-release.snapshot/boost/1.60-pm/" 182 | path_pattern = "*.*.*/vc110/x86/win/boost.*.*.*.tar.gz" 183 | 184 | _path_fixed, _path_pattern = parser.split_fixed_pattern(path) 185 | 186 | assert path_fixed == _path_fixed 187 | assert path_pattern == _path_pattern 188 | 189 | def test_has_rule(self): 190 | parser = self._parsers.get('artifactory', None) 191 | 192 | assert parser.has_rule('path') is True 193 | assert parser.has_rule('properties') is False 194 | 195 | # def test_get_full_package_name(self): 196 | # parser = self._parsers.get('artifactory', None) 197 | # 198 | # package = Package('test_package', None, None, None, None, parser=parser) 199 | # name = parser.get_full_package_name(package) 200 | # 201 | # assert name == "test_package" 202 | 203 | def test_list_flatter(self): 204 | parser = self._parsers.get('common', None) 205 | 206 | src = [[ 207 | 'https://repo.example.com/artifactory/cybsi.snapshot/cybsi/*/*.*.*/[any|any]/[any|any]/cybsi/cybsi.*.*.*[.zip|.tar.gz|.nupkg]']] 208 | result = [ 209 | 'https://repo.example.com/artifactory/cybsi.snapshot/cybsi/*/*.*.*/[any|any]/[any|any]/cybsi/cybsi.*.*.*[.zip|.tar.gz|.nupkg]'] 210 | 211 | assert parser.list_flatter(src) == result 212 | 213 | def test_validate_atom(self): 214 | parser = self._parsers.get('artifactory', None) 215 | 216 | assert parser.validate_atom('feature-netcore-probes', '*') is True 217 | assert parser.validate_atom('1.2', '>=1.3') is False 218 | 219 | def test_values_match(self): 220 | parser = self._parsers.get('artifactory', None) 221 | 222 | assert parser.values_match('1.3', '1.3') is True 223 | assert parser.values_match('2.0', '1.3') is False 224 | 225 | # def test_iter_matched_values(self): 226 | # """ 227 | # module parser have no method get_values(column_name) (see line 566 in parser) 228 | # so we should use mock object I think 229 | # """ 230 | # parser = self._parsers.get('artifactory', None) 231 | # 232 | # column_name = 'version' 233 | # value = ['1', '2', '3', None] 234 | # 235 | # ## debug 236 | # #res = parser.iter_matched_values(column_name, value) 237 | # #print(res) 238 | # 239 | # assert parser.iter_matched_values('1.3', '1.3') is True 240 | 241 | @pytest.mark.artifactoryaql 242 | def test__get_params_with_extra(self): 243 | parser = self._parsers.get('common', None) 244 | 245 | vars_extra = {'path': [ 246 | { 247 | 'compiler': ['any'], 248 | 'arch': ['any'], 249 | } 250 | ]} 251 | 252 | params = { 253 | 'arch': 'x86', 254 | 'compiler': 'vc140', 255 | 'osname': 'win', 256 | 'version': [1, 2, 3] 257 | } 258 | 259 | expected_result = [ 260 | { 261 | 'arch': 'x86', 262 | 'compiler': 'vc140', 263 | 'osname': 'win', 264 | 'version': [1, 2, 3] 265 | }, 266 | { 267 | 'arch': 'any', 268 | 'compiler': 'vc140', 269 | 'osname': 'win', 270 | 'version': [1, 2, 3] 271 | }, 272 | { 273 | 'arch': 'x86', 274 | 'compiler': 'any', 275 | 'osname': 'win', 276 | 'version': [1, 2, 3] 277 | }, 278 | { 279 | 'arch': 'any', 280 | 'compiler': 'any', 281 | 'osname': 'win', 282 | 'version': [1, 2, 3] 283 | }, 284 | 285 | ] 286 | parser._rules_vars_extra = vars_extra 287 | result = parser.get_params_with_extra('path', params) 288 | 289 | for res in result: 290 | assert res in expected_result, "Result contain MORE element then expected" 291 | 292 | for res in expected_result: 293 | assert res in result, "Result contain LESS element then expected" 294 | 295 | def test_get_usedby_aql(self): 296 | parser = self._parsers.get('artifactory', None) # type: Parser 297 | parser.merge_valued = lambda x: x # TODO: Убрать этот грубый хак :) 298 | result = parser.get_usedby_aql({'package': 'packagename', 'version': '1.2.3'}) 299 | expect_result = { 300 | '@dd.packagename.version': '1.2.3', 301 | '@dd.packagename.operator': '=', 302 | 'path': 303 | {'$match': '*vc140/x86_64/win*'} 304 | } 305 | assert expect_result == result 306 | 307 | def test_get_usedby_aql_none(self): 308 | parser = self._parsers.get('common', None) # type: Parser 309 | result = parser.get_usedby_aql({}) 310 | assert result is None 311 | 312 | def test_get_params_from_properties(self): 313 | parser = self._parsers.get('artifactory', None) # type: Parser 314 | params = parser.get_params_from_properties({'deb.name': 'packagename', 'deb.version': '1.2.3'}) 315 | expect_params = { 316 | 'package': 'packagename', 317 | 'version': '1.2.3', 318 | 'qaverdict': '' 319 | } 320 | assert expect_params == params 321 | 322 | def test_get_params_from_path(self): 323 | parser = self._parsers.get('artifactory', None) # type: Parser 324 | params = parser.get_params_from_path( 325 | "https://repo.example.com/artifactory/libs-cpp-release.snapshot/zlib/1.2.8-pm/1.2.8.199/vc110/x86/win/zlib.1.2.8.199.tar.gz") 326 | expect_params = { 327 | 'arch': 'x86', 328 | 'branch': '1.2.8-pm', 329 | 'compiler': 'vc110', 330 | 'osname': 'win', 331 | 'package': 'zlib', 332 | 'version': '1.2.8.199', 333 | } 334 | assert expect_params == params 335 | -------------------------------------------------------------------------------- /toc.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | FILES=$(find ./docs/ -name "*.md") 4 | BASE=$(pwd) 5 | for f in ${FILES}: 6 | do 7 | dir=$(dirname ${f}) 8 | filename=$(basename ${f}) 9 | pushd ${dir} 10 | ${BASE}/gh-md-toc --insert ${filename} 11 | popd 12 | done 13 | --------------------------------------------------------------------------------